krata: implement event stream retries

crates/ctl/src/cli/list.rs

@@ -4,7 +4,7 @@ use cli_tables::Table;
 use krata::{
     events::EventStream,
     v1::{
-        common::{guest_image_spec::Image, Guest, GuestStatus},
+        common::{guest_image_spec::Image, Guest},
         control::{
             control_service_client::ControlServiceClient, ListGuestsRequest, ResolveGuestRequest,
         },
@@ -14,7 +14,7 @@ use krata::{
 use serde_json::Value;
 use tonic::{transport::Channel, Request};
 
-use crate::format::{guest_state_text, guest_status_text, kv2line, proto2dynamic, proto2kv};
+use crate::format::{guest_simple_line, guest_state_text, kv2line, proto2dynamic, proto2kv};
 
 #[derive(ValueEnum, Clone, Debug, PartialEq, Eq)]
 enum ListFormat {
@@ -76,18 +76,7 @@ impl ListCommand {
 
             ListFormat::Simple => {
                 for guest in guests {
-                    let state = guest_status_text(
+                    println!("{}", guest_simple_line(&guest));
-                        guest
-                            .state
-                            .as_ref()
-                            .map(|x| x.status())
-                            .unwrap_or(GuestStatus::Unknown),
-                    );
-                    let name = guest.spec.as_ref().map(|x| x.name.as_str()).unwrap_or("");
-                    let network = guest.state.as_ref().and_then(|x| x.network.as_ref());
-                    let ipv4 = network.map(|x| x.guest_ipv4.as_str()).unwrap_or("");
-                    let ipv6 = network.map(|x| x.guest_ipv6.as_str()).unwrap_or("");
-                    println!("{}\t{}\t{}\t{}\t{}", guest.id, state, name, ipv4, ipv6);
                 }
             }
 

crates/ctl/src/cli/mod.rs

@@ -10,9 +10,7 @@ use clap::{Parser, Subcommand};
 use krata::{
     client::ControlClientProvider,
     events::EventStream,
-    v1::control::{
+    v1::control::{control_service_client::ControlServiceClient, ResolveGuestRequest},
-        control_service_client::ControlServiceClient, ResolveGuestRequest, WatchEventsRequest,
-    },
 };
 use tonic::{transport::Channel, Request};
 
@@ -43,14 +41,8 @@ pub enum Commands {
 
 impl ControlCommand {
     pub async fn run(self) -> Result<()> {
-        let mut client = ControlClientProvider::dial(self.connection.parse()?).await?;
+        let client = ControlClientProvider::dial(self.connection.parse()?).await?;
-        let events = EventStream::open(
+        let events = EventStream::open(client.clone()).await?;
-            client
-                .watch_events(WatchEventsRequest {})
-                .await?
-                .into_inner(),
-        )
-        .await?;
 
         match self.command {
             Commands::Launch(launch) => {

crates/ctl/src/cli/watch.rs

@@ -7,7 +7,7 @@ use krata::{
 use prost_reflect::ReflectMessage;
 use serde_json::Value;
 
-use crate::format::{guest_state_text, kv2line, proto2dynamic, proto2kv};
+use crate::format::{guest_simple_line, kv2line, proto2dynamic, proto2kv};
 
 #[derive(ValueEnum, Clone, Debug, PartialEq, Eq)]
 enum WatchFormat {
@@ -45,12 +45,7 @@ impl WatchCommand {
         match self.format {
             WatchFormat::Simple => {
                 if let Some(guest) = guest {
-                    println!(
+                    println!("{}", guest_simple_line(&guest));
-                        "{} guest={} status=\"{}\"",
-                        typ,
-                        guest.id,
-                        guest_state_text(guest.state.as_ref()).replace('"', "\\\"")
-                    );
                 }
             }
 

crates/ctl/src/format.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 
 use anyhow::Result;
-use krata::v1::common::{GuestState, GuestStatus};
+use krata::v1::common::{Guest, GuestState, GuestStatus};
 use prost_reflect::{DynamicMessage, ReflectMessage, Value};
 
 pub fn proto2dynamic(proto: impl ReflectMessage) -> Result<DynamicMessage> {
@@ -84,3 +84,18 @@ pub fn guest_state_text(state: Option<&GuestState>) -> String {
     }
     text
 }
+
+pub fn guest_simple_line(guest: &Guest) -> String {
+    let state = guest_status_text(
+        guest
+            .state
+            .as_ref()
+            .map(|x| x.status())
+            .unwrap_or(GuestStatus::Unknown),
+    );
+    let name = guest.spec.as_ref().map(|x| x.name.as_str()).unwrap_or("");
+    let network = guest.state.as_ref().and_then(|x| x.network.as_ref());
+    let ipv4 = network.map(|x| x.guest_ipv4.as_str()).unwrap_or("");
+    let ipv6 = network.map(|x| x.guest_ipv6.as_str()).unwrap_or("");
+    format!("{}\t{}\t{}\t{}\t{}", guest.id, state, name, ipv4, ipv6)
+}

crates/krata/src/events.rs

@@ -1,11 +1,14 @@
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 
-use crate::v1::control::{watch_events_reply::Event, WatchEventsReply};
+use crate::v1::control::{
+    control_service_client::ControlServiceClient, watch_events_reply::Event, WatchEventsReply,
+    WatchEventsRequest,
+};
 use anyhow::Result;
-use log::trace;
+use log::{error, trace, warn};
-use tokio::{sync::broadcast, task::JoinHandle};
+use tokio::{sync::broadcast, task::JoinHandle, time::sleep};
 use tokio_stream::StreamExt;
-use tonic::Streaming;
+use tonic::{transport::Channel, Streaming};
 
 #[derive(Clone)]
 pub struct EventStream {
@@ -14,27 +17,12 @@ pub struct EventStream {
 }
 
 impl EventStream {
-    pub async fn open(mut events: Streaming<WatchEventsReply>) -> Result<Self> {
+    pub async fn open(client: ControlServiceClient<Channel>) -> Result<Self> {
         let (sender, _) = broadcast::channel(1000);
         let emit = sender.clone();
         let task = tokio::task::spawn(async move {
-            loop {
+            if let Err(error) = EventStream::process(client, emit).await {
-                let Some(result) = events.next().await else {
+                error!("failed to process event stream: {}", error);
-                    break;
-                };
-
-                let reply = match result {
-                    Ok(reply) => reply,
-                    Err(error) => {
-                        trace!("event stream processing failed: {}", error);
-                        break;
-                    }
-                };
-
-                let Some(event) = reply.event else {
-                    continue;
-                };
-                let _ = emit.send(event);
             }
         });
         Ok(Self {
@@ -43,6 +31,48 @@ impl EventStream {
         })
     }
 
+    async fn process(
+        mut client: ControlServiceClient<Channel>,
+        emit: broadcast::Sender<Event>,
+    ) -> Result<()> {
+        let mut events: Option<Streaming<WatchEventsReply>> = None;
+        loop {
+            let mut stream = match events {
+                Some(stream) => stream,
+                None => {
+                    let result = client.watch_events(WatchEventsRequest {}).await;
+                    if let Err(error) = result {
+                        warn!("failed to watch events: {}", error);
+                        sleep(Duration::from_secs(1)).await;
+                        continue;
+                    }
+                    result.unwrap().into_inner()
+                }
+            };
+
+            let Some(result) = stream.next().await else {
+                events = None;
+                continue;
+            };
+
+            let reply = match result {
+                Ok(reply) => reply,
+                Err(error) => {
+                    trace!("event stream processing failed: {}", error);
+                    events = None;
+                    continue;
+                }
+            };
+
+            let Some(event) = reply.event else {
+                events = Some(stream);
+                continue;
+            };
+            let _ = emit.send(event);
+            events = Some(stream);
+        }
+    }
+
     pub fn subscribe(&self) -> broadcast::Receiver<Event> {
         self.sender.subscribe()
     }

crates/network/src/autonet.rs

@@ -5,7 +5,7 @@ use krata::{
         common::Guest,
         control::{
             control_service_client::ControlServiceClient, watch_events_reply::Event,
-            ListGuestsRequest, WatchEventsRequest,
+            ListGuestsRequest,
         },
     },
 };
@@ -50,12 +50,11 @@ pub struct AutoNetworkChangeset {
 }
 
 impl AutoNetworkWatcher {
-    pub async fn new(mut control: ControlServiceClient<Channel>) -> Result<AutoNetworkWatcher> {
+    pub async fn new(control: ControlServiceClient<Channel>) -> Result<AutoNetworkWatcher> {
-        let watch_events_response = control.watch_events(WatchEventsRequest {}).await?;
+        let client = control.clone();
-
         Ok(AutoNetworkWatcher {
             control,
-            events: EventStream::open(watch_events_response.into_inner()).await?,
+            events: EventStream::open(client).await?,
             known: HashMap::new(),
         })
     }
@@ -136,7 +135,15 @@ impl AutoNetworkWatcher {
         let mut added: Vec<NetworkMetadata> = Vec::new();
         let mut removed: Vec<NetworkMetadata> = Vec::new();
 
-        for network in self.read().await? {
+        let networks = match self.read().await {
+            Ok(networks) => networks,
+            Err(error) => {
+                warn!("failed to read network changes: {}", error);
+                return Ok(AutoNetworkChangeset { added, removed });
+            }
+        };
+
+        for network in networks {
             seen.push(network.uuid);
             if self.known.contains_key(&network.uuid) {
                 continue;

kernel/config.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-KERNEL_VERSION="6.8.2"
+KERNEL_VERSION="6.7.2"
 KERNEL_SRC_URL="https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${KERNEL_VERSION}.tar.xz"

kernel/krata-aarch64.config

@@ -1,23 +1,23 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/arm64 6.7.8 Kernel Configuration
+# Linux/arm64 6.7.2 Kernel Configuration
 #
-CONFIG_CC_VERSION_TEXT="gcc (GCC) 13.2.1 20231205 (Red Hat 13.2.1-6)"
+CONFIG_CC_VERSION_TEXT="aarch64-linux-gnu-gcc (Debian 13.2.0-12) 13.2.0"
 CONFIG_CC_IS_GCC=y
-CONFIG_GCC_VERSION=130201
+CONFIG_GCC_VERSION=130200
 CONFIG_CLANG_VERSION=0
 CONFIG_AS_IS_GNU=y
-CONFIG_AS_VERSION=24000
+CONFIG_AS_VERSION=24200
 CONFIG_LD_IS_BFD=y
-CONFIG_LD_VERSION=24000
+CONFIG_LD_VERSION=24200
 CONFIG_LLD_VERSION=0
 CONFIG_CC_CAN_LINK=y
+CONFIG_CC_CAN_LINK_STATIC=y
 CONFIG_CC_HAS_ASM_GOTO_OUTPUT=y
 CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT=y
-CONFIG_GCC_ASM_GOTO_OUTPUT_WORKAROUND=y
 CONFIG_CC_HAS_ASM_INLINE=y
 CONFIG_CC_HAS_NO_PROFILE_FN_ATTR=y
-CONFIG_PAHOLE_VERSION=0
+CONFIG_PAHOLE_VERSION=124
 CONFIG_IRQ_WORK=y
 CONFIG_BUILDTIME_TABLE_SORT=y
 CONFIG_THREAD_INFO_IN_TASK=y
@@ -401,9 +401,7 @@ CONFIG_ARM64_ERRATUM_2067961=y
 CONFIG_ARM64_ERRATUM_2441009=y
 CONFIG_ARM64_ERRATUM_2457168=y
 CONFIG_ARM64_ERRATUM_2645198=y
-CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD=y
 CONFIG_ARM64_ERRATUM_2966298=y
-CONFIG_ARM64_ERRATUM_3117295=y
 CONFIG_CAVIUM_ERRATUM_22375=y
 CONFIG_CAVIUM_ERRATUM_23144=y
 CONFIG_CAVIUM_ERRATUM_23154=y
@@ -3475,6 +3473,7 @@ CONFIG_BCMA_POSSIBLE=y
 # CONFIG_MFD_SM501 is not set
 # CONFIG_MFD_SKY81452 is not set
 # CONFIG_MFD_SYSCON is not set
+# CONFIG_MFD_TI_AM335X_TSCADC is not set
 # CONFIG_MFD_LP3943 is not set
 # CONFIG_MFD_TI_LMU is not set
 # CONFIG_TPS6105X is not set
@@ -5509,7 +5508,11 @@ CONFIG_DEBUG_INFO_DWARF5=y
 # CONFIG_DEBUG_INFO_REDUCED is not set
 CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
 # CONFIG_DEBUG_INFO_COMPRESSED_ZLIB is not set
+# CONFIG_DEBUG_INFO_COMPRESSED_ZSTD is not set
 # CONFIG_DEBUG_INFO_SPLIT is not set
+# CONFIG_DEBUG_INFO_BTF is not set
+CONFIG_PAHOLE_HAS_SPLIT_BTF=y
+CONFIG_PAHOLE_HAS_LANG_EXCLUDE=y
 # CONFIG_GDB_SCRIPTS is not set
 CONFIG_FRAME_WARN=1280
 # CONFIG_STRIP_ASM_SYMS is not set