krata: improve guest reconciliation

crates/kratad/src/event.rs

@@ -1,4 +1,8 @@
-use std::{collections::HashMap, str::FromStr, time::Duration};
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    str::FromStr,
+    time::Duration,
+};
 
 use anyhow::Result;
 use krata::common::{GuestExitInfo, GuestState, GuestStatus};
@@ -87,11 +91,13 @@ impl DaemonEventGenerator {
                 let id = Uuid::from_str(&guest.id)?;
                 match status {
                     GuestStatus::Started => {
+                        if let Entry::Vacant(e) = self.exit_code_handles.entry(id) {
                             let handle = self
                                 .runtime
                                 .subscribe_exit_code(id, self.exit_code_sender.clone())
                                 .await?;
-                        self.exit_code_handles.insert(id, handle);
+                            e.insert(handle);
+                        }
                     }
 
                     GuestStatus::Destroyed => {

crates/kratad/src/reconcile/guest.rs

@@ -1,13 +1,16 @@
+use std::time::Duration;
+
 use anyhow::{anyhow, Result};
 use krata::{
     common::{
-        guest_image_spec::Image, Guest, GuestErrorInfo, GuestNetworkState, GuestState, GuestStatus,
+        guest_image_spec::Image, Guest, GuestErrorInfo, GuestExitInfo, GuestNetworkState,
+        GuestState, GuestStatus,
     },
     control::GuestChangedEvent,
 };
 use kratart::{launch::GuestLaunchRequest, Runtime};
-use log::{error, info, warn};
+use log::{error, info, trace, warn};
-use tokio::{sync::mpsc::Receiver, task::JoinHandle};
+use tokio::{select, sync::mpsc::Receiver, task::JoinHandle, time::sleep};
 use uuid::Uuid;
 
 use crate::{
@@ -32,22 +35,36 @@ impl GuestReconciler {
 
     pub async fn launch(self, mut notify: Receiver<Uuid>) -> Result<JoinHandle<()>> {
         Ok(tokio::task::spawn(async move {
-            if let Err(error) = self.reconcile_runtime().await {
+            if let Err(error) = self.reconcile_runtime(true).await {
                 error!("runtime reconciler failed: {}", error);
             }
 
             loop {
-                let Some(uuid) = notify.recv().await else {
+                select! {
+                    x = notify.recv() => match x {
+                        None => {
                             break;
-                };
+                        },
+
+                        Some(uuid) => {
                             if let Err(error) = self.reconcile(uuid).await {
-                    error!("guest reconciler failed: {}", error);
+                                error!("failed to reconcile guest {}: {}", uuid, error);
                             }
                         }
+                    },
+
+                    _ = sleep(Duration::from_secs(30)) => {
+                        if let Err(error) = self.reconcile_runtime(false).await {
+                            error!("runtime reconciler failed: {}", error);
+                        }
+                    }
+                };
+            }
         }))
     }
 
-    pub async fn reconcile_runtime(&self) -> Result<()> {
+    pub async fn reconcile_runtime(&self, initial: bool) -> Result<()> {
+        trace!("reconciling runtime");
         let runtime_guests = self.runtime.list().await?;
         let stored_guests = self.guests.list().await?;
         for (uuid, mut stored_guest_entry) in stored_guests {
@@ -56,6 +73,7 @@ impl GuestReconciler {
                 self.guests.remove(uuid).await?;
                 continue;
             };
+            let previous_guest = stored_guest.clone();
             let runtime_guest = runtime_guests.iter().find(|x| x.uuid == uuid);
             match runtime_guest {
                 None => {
@@ -65,24 +83,33 @@ impl GuestReconciler {
                     }
                     stored_guest.state = Some(state);
                     stored_guest.network = None;
-                    self.guests.update(uuid, stored_guest_entry).await?;
+                }
-                    if let Err(error) = self.reconcile(uuid).await {
+
-                        error!("failed to reconcile guest {}: {}", uuid, error);
+                Some(runtime) => {
+                    let mut state = stored_guest.state.as_mut().cloned().unwrap_or_default();
+                    if let Some(code) = runtime.state.exit_code {
+                        state.status = GuestStatus::Exited.into();
+                        state.exit_info = Some(GuestExitInfo { code });
+                    } else {
+                        state.status = GuestStatus::Started.into();
+                    }
+                    stored_guest.state = Some(state);
+                    stored_guest.network = Some(GuestNetworkState {
+                        ipv4: runtime.ipv4.map(|x| x.ip().to_string()).unwrap_or_default(),
+                        ipv6: runtime.ipv6.map(|x| x.ip().to_string()).unwrap_or_default(),
+                    });
                 }
             }
 
-                Some(_) => {
+            let changed = *stored_guest != previous_guest;
-                    let mut state = stored_guest.state.as_mut().cloned().unwrap_or_default();
-                    state.status = GuestStatus::Started.into();
-                    stored_guest.state = Some(state);
-                    stored_guest.network = None;
             self.guests.update(uuid, stored_guest_entry).await?;
+
+            if changed || initial {
                 if let Err(error) = self.reconcile(uuid).await {
                     error!("failed to reconcile guest {}: {}", uuid, error);
                 }
             }
         }
-        }
         Ok(())
     }
 

crates/krataguest/src/init.rs

@@ -3,9 +3,10 @@ use futures::stream::TryStreamExt;
 use ipnetwork::IpNetwork;
 use krata::ethtool::EthtoolHandle;
 use krata::launchcfg::{LaunchInfo, LaunchNetwork};
+use libc::{setsid, TIOCSCTTY};
 use log::{trace, warn};
-use nix::libc::{dup2, ioctl};
+use nix::ioctl_write_int_bad;
-use nix::unistd::{execve, fork, ForkResult, Pid};
+use nix::unistd::{dup2, execve, fork, ForkResult, Pid};
 use oci_spec::image::{Config, ImageConfiguration};
 use path_absolutize::Absolutize;
 use std::collections::HashMap;
@@ -15,7 +16,7 @@ use std::net::{Ipv4Addr, Ipv6Addr};
 use std::os::fd::AsRawFd;
 use std::os::linux::fs::MetadataExt;
 use std::os::unix::ffi::OsStrExt;
-use std::os::unix::fs::{chroot, PermissionsExt};
+use std::os::unix::fs::{chroot, symlink, PermissionsExt};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::{fs, io};
@@ -47,6 +48,8 @@ const NEW_ROOT_DEV_PATH: &str = "/newroot/dev";
 const IMAGE_CONFIG_JSON_PATH: &str = "/config/image/config.json";
 const LAUNCH_CONFIG_JSON_PATH: &str = "/config/launch.json";
 
+ioctl_write_int_bad!(set_controlling_terminal, TIOCSCTTY);
+
 pub struct GuestInit {}
 
 impl Default for GuestInit {
@@ -106,7 +109,7 @@ impl GuestInit {
         self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755")?;
         self.mount_kernel_fs("proc", "/proc", "")?;
         self.mount_kernel_fs("sysfs", "/sys", "")?;
-        std::os::unix::fs::symlink("/proc/self/fd", "/dev/fd")?;
+        symlink("/proc/self/fd", "/dev/fd")?;
         Ok(())
     }
 
@@ -139,11 +142,9 @@ impl GuestInit {
 
     fn map_console(&mut self, console: &File) -> Result<()> {
         trace!("mapping console");
-        unsafe {
+        dup2(console.as_raw_fd(), 0)?;
-            dup2(console.as_raw_fd(), 0);
+        dup2(console.as_raw_fd(), 1)?;
-            dup2(console.as_raw_fd(), 1);
+        dup2(console.as_raw_fd(), 2)?;
-            dup2(console.as_raw_fd(), 2);
-        }
         Ok(())
     }
 
@@ -516,10 +517,9 @@ impl GuestInit {
     }
 
     fn set_controlling_terminal() -> Result<()> {
-        unsafe { nix::libc::setsid() };
+        unsafe {
-        let result = unsafe { ioctl(io::stdin().as_raw_fd(), nix::libc::TIOCSCTTY, 0) };
+            setsid();
-        if result != 0 {
+            set_controlling_terminal(io::stdin().as_raw_fd(), 0)?;
-            warn!("failed to set controlling terminal, result={}", result);
         }
         Ok(())
     }