From 3b5e3a077a760b14d4fdcb4e45e7d259089cfc20 Mon Sep 17 00:00:00 2001 From: Alex Zenla Date: Sat, 23 Mar 2024 02:10:30 +0000 Subject: [PATCH] krata: improve guest reconciliation --- crates/kratad/src/event.rs | 18 +++++--- crates/kratad/src/reconcile/guest.rs | 69 +++++++++++++++++++--------- crates/krataguest/src/init.rs | 26 +++++------ 3 files changed, 73 insertions(+), 40 deletions(-) diff --git a/crates/kratad/src/event.rs b/crates/kratad/src/event.rs index 1d60691..90e0f3d 100644 --- a/crates/kratad/src/event.rs +++ b/crates/kratad/src/event.rs @@ -1,4 +1,8 @@ -use std::{collections::HashMap, str::FromStr, time::Duration}; +use std::{ + collections::{hash_map::Entry, HashMap}, + str::FromStr, + time::Duration, +}; use anyhow::Result; use krata::common::{GuestExitInfo, GuestState, GuestStatus}; @@ -87,11 +91,13 @@ impl DaemonEventGenerator { let id = Uuid::from_str(&guest.id)?; match status { GuestStatus::Started => { - let handle = self - .runtime - .subscribe_exit_code(id, self.exit_code_sender.clone()) - .await?; - self.exit_code_handles.insert(id, handle); + if let Entry::Vacant(e) = self.exit_code_handles.entry(id) { + let handle = self + .runtime + .subscribe_exit_code(id, self.exit_code_sender.clone()) + .await?; + e.insert(handle); + } } GuestStatus::Destroyed => { diff --git a/crates/kratad/src/reconcile/guest.rs b/crates/kratad/src/reconcile/guest.rs index 79bfbb5..ced7565 100644 --- a/crates/kratad/src/reconcile/guest.rs +++ b/crates/kratad/src/reconcile/guest.rs @@ -1,13 +1,16 @@ +use std::time::Duration; + use anyhow::{anyhow, Result}; use krata::{ common::{ - guest_image_spec::Image, Guest, GuestErrorInfo, GuestNetworkState, GuestState, GuestStatus, + guest_image_spec::Image, Guest, GuestErrorInfo, GuestExitInfo, GuestNetworkState, + GuestState, GuestStatus, }, control::GuestChangedEvent, }; use kratart::{launch::GuestLaunchRequest, Runtime}; -use log::{error, info, warn}; -use tokio::{sync::mpsc::Receiver, task::JoinHandle}; +use log::{error, info, trace, warn}; +use tokio::{select, sync::mpsc::Receiver, task::JoinHandle, time::sleep}; use uuid::Uuid; use crate::{ @@ -32,22 +35,36 @@ impl GuestReconciler { pub async fn launch(self, mut notify: Receiver) -> Result> { Ok(tokio::task::spawn(async move { - if let Err(error) = self.reconcile_runtime().await { + if let Err(error) = self.reconcile_runtime(true).await { error!("runtime reconciler failed: {}", error); } loop { - let Some(uuid) = notify.recv().await else { - break; + select! { + x = notify.recv() => match x { + None => { + break; + }, + + Some(uuid) => { + if let Err(error) = self.reconcile(uuid).await { + error!("failed to reconcile guest {}: {}", uuid, error); + } + } + }, + + _ = sleep(Duration::from_secs(30)) => { + if let Err(error) = self.reconcile_runtime(false).await { + error!("runtime reconciler failed: {}", error); + } + } }; - if let Err(error) = self.reconcile(uuid).await { - error!("guest reconciler failed: {}", error); - } } })) } - pub async fn reconcile_runtime(&self) -> Result<()> { + pub async fn reconcile_runtime(&self, initial: bool) -> Result<()> { + trace!("reconciling runtime"); let runtime_guests = self.runtime.list().await?; let stored_guests = self.guests.list().await?; for (uuid, mut stored_guest_entry) in stored_guests { @@ -56,6 +73,7 @@ impl GuestReconciler { self.guests.remove(uuid).await?; continue; }; + let previous_guest = stored_guest.clone(); let runtime_guest = runtime_guests.iter().find(|x| x.uuid == uuid); match runtime_guest { None => { @@ -65,21 +83,30 @@ impl GuestReconciler { } stored_guest.state = Some(state); stored_guest.network = None; - self.guests.update(uuid, stored_guest_entry).await?; - if let Err(error) = self.reconcile(uuid).await { - error!("failed to reconcile guest {}: {}", uuid, error); - } } - Some(_) => { + Some(runtime) => { let mut state = stored_guest.state.as_mut().cloned().unwrap_or_default(); - state.status = GuestStatus::Started.into(); - stored_guest.state = Some(state); - stored_guest.network = None; - self.guests.update(uuid, stored_guest_entry).await?; - if let Err(error) = self.reconcile(uuid).await { - error!("failed to reconcile guest {}: {}", uuid, error); + if let Some(code) = runtime.state.exit_code { + state.status = GuestStatus::Exited.into(); + state.exit_info = Some(GuestExitInfo { code }); + } else { + state.status = GuestStatus::Started.into(); } + stored_guest.state = Some(state); + stored_guest.network = Some(GuestNetworkState { + ipv4: runtime.ipv4.map(|x| x.ip().to_string()).unwrap_or_default(), + ipv6: runtime.ipv6.map(|x| x.ip().to_string()).unwrap_or_default(), + }); + } + } + + let changed = *stored_guest != previous_guest; + self.guests.update(uuid, stored_guest_entry).await?; + + if changed || initial { + if let Err(error) = self.reconcile(uuid).await { + error!("failed to reconcile guest {}: {}", uuid, error); } } } diff --git a/crates/krataguest/src/init.rs b/crates/krataguest/src/init.rs index e2f65cc..8bf73a2 100644 --- a/crates/krataguest/src/init.rs +++ b/crates/krataguest/src/init.rs @@ -3,9 +3,10 @@ use futures::stream::TryStreamExt; use ipnetwork::IpNetwork; use krata::ethtool::EthtoolHandle; use krata::launchcfg::{LaunchInfo, LaunchNetwork}; +use libc::{setsid, TIOCSCTTY}; use log::{trace, warn}; -use nix::libc::{dup2, ioctl}; -use nix::unistd::{execve, fork, ForkResult, Pid}; +use nix::ioctl_write_int_bad; +use nix::unistd::{dup2, execve, fork, ForkResult, Pid}; use oci_spec::image::{Config, ImageConfiguration}; use path_absolutize::Absolutize; use std::collections::HashMap; @@ -15,7 +16,7 @@ use std::net::{Ipv4Addr, Ipv6Addr}; use std::os::fd::AsRawFd; use std::os::linux::fs::MetadataExt; use std::os::unix::ffi::OsStrExt; -use std::os::unix::fs::{chroot, PermissionsExt}; +use std::os::unix::fs::{chroot, symlink, PermissionsExt}; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::{fs, io}; @@ -47,6 +48,8 @@ const NEW_ROOT_DEV_PATH: &str = "/newroot/dev"; const IMAGE_CONFIG_JSON_PATH: &str = "/config/image/config.json"; const LAUNCH_CONFIG_JSON_PATH: &str = "/config/launch.json"; +ioctl_write_int_bad!(set_controlling_terminal, TIOCSCTTY); + pub struct GuestInit {} impl Default for GuestInit { @@ -106,7 +109,7 @@ impl GuestInit { self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755")?; self.mount_kernel_fs("proc", "/proc", "")?; self.mount_kernel_fs("sysfs", "/sys", "")?; - std::os::unix::fs::symlink("/proc/self/fd", "/dev/fd")?; + symlink("/proc/self/fd", "/dev/fd")?; Ok(()) } @@ -139,11 +142,9 @@ impl GuestInit { fn map_console(&mut self, console: &File) -> Result<()> { trace!("mapping console"); - unsafe { - dup2(console.as_raw_fd(), 0); - dup2(console.as_raw_fd(), 1); - dup2(console.as_raw_fd(), 2); - } + dup2(console.as_raw_fd(), 0)?; + dup2(console.as_raw_fd(), 1)?; + dup2(console.as_raw_fd(), 2)?; Ok(()) } @@ -516,10 +517,9 @@ impl GuestInit { } fn set_controlling_terminal() -> Result<()> { - unsafe { nix::libc::setsid() }; - let result = unsafe { ioctl(io::stdin().as_raw_fd(), nix::libc::TIOCSCTTY, 0) }; - if result != 0 { - warn!("failed to set controlling terminal, result={}", result); + unsafe { + setsid(); + set_controlling_terminal(io::stdin().as_raw_fd(), 0)?; } Ok(()) }