diff --git a/crates/krata/Cargo.toml b/crates/krata/Cargo.toml index ee5ee3c..0fd4b13 100644 --- a/crates/krata/Cargo.toml +++ b/crates/krata/Cargo.toml @@ -6,6 +6,7 @@ resolver = "2" [dependencies] anyhow = { workspace = true } +bytes = { workspace = true } libc = { workspace = true } log = { workspace = true } once_cell = { workspace = true } @@ -18,6 +19,9 @@ tokio-stream = { workspace = true } tower = { workspace = true } url = { workspace = true } +[target.'cfg(unix)'.dependencies] +nix = { workspace = true, features = ["term"] } + [build-dependencies] tonic-build = { workspace = true } prost-build = { workspace = true } diff --git a/crates/krata/src/idm.rs b/crates/krata/src/idm.rs deleted file mode 100644 index cb0a716..0000000 --- a/crates/krata/src/idm.rs +++ /dev/null @@ -1 +0,0 @@ -include!(concat!(env!("OUT_DIR"), "/krata.idm.rs")); diff --git a/crates/krata/src/idm/client.rs b/crates/krata/src/idm/client.rs new file mode 100644 index 0000000..919469f --- /dev/null +++ b/crates/krata/src/idm/client.rs @@ -0,0 +1,111 @@ +use std::path::Path; + +use super::protocol::IdmPacket; +use anyhow::{anyhow, Result}; +use bytes::BytesMut; +use log::error; +use nix::sys::termios::{cfmakeraw, tcgetattr, tcsetattr, SetArg}; +use prost::Message; +use tokio::{ + fs::File, + io::{unix::AsyncFd, AsyncReadExt, AsyncWriteExt}, + select, + sync::mpsc::{channel, Receiver, Sender}, + task::JoinHandle, +}; + +const IDM_PACKET_QUEUE_LEN: usize = 100; + +pub struct IdmClient { + pub receiver: Receiver, + pub sender: Sender, + task: JoinHandle<()>, +} + +impl Drop for IdmClient { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl IdmClient { + pub async fn open>(path: P) -> Result { + let file = File::options() + .read(true) + .write(true) + .create(false) + .open(path) + .await?; + IdmClient::set_raw_port(&file)?; + let (rx_sender, rx_receiver) = channel(IDM_PACKET_QUEUE_LEN); + let (tx_sender, tx_receiver) = channel(IDM_PACKET_QUEUE_LEN); + let task = tokio::task::spawn(async move { + if let Err(error) = IdmClient::process(file, rx_sender, tx_receiver).await { + error!("failed to handle idm client processing: {}", error); + } + }); + Ok(IdmClient { + receiver: rx_receiver, + sender: tx_sender, + task, + }) + } + + fn set_raw_port(file: &File) -> Result<()> { + let mut termios = tcgetattr(file)?; + cfmakeraw(&mut termios); + tcsetattr(file, SetArg::TCSANOW, &termios)?; + Ok(()) + } + + async fn process( + file: File, + sender: Sender, + mut receiver: Receiver, + ) -> Result<()> { + let mut file = AsyncFd::new(file)?; + loop { + select! { + x = file.readable_mut() => match x { + Ok(mut guard) => { + let size = guard.get_inner_mut().read_u16_le().await?; + if size == 0 { + continue; + } + let mut buffer = BytesMut::with_capacity(size as usize); + guard.get_inner_mut().read_exact(&mut buffer).await?; + match IdmPacket::decode(buffer) { + Ok(packet) => { + sender.send(packet).await?; + }, + + Err(error) => { + error!("received invalid idm packet: {}", error); + } + } + }, + + Err(error) => { + return Err(anyhow!("failed to read idm client: {}", error)); + } + }, + x = receiver.recv() => match x { + Some(packet) => { + let data = packet.encode_to_vec(); + if data.len() > u16::MAX as usize { + error!("unable to send idm packet, packet size exceeded (tried to send {} bytes)", data.len()); + continue; + } + file.get_mut().write_u16_le(data.len() as u16).await?; + file.get_mut().write_all(&data).await?; + }, + + None => { + break; + } + } + } + } + Ok(()) + } +} diff --git a/crates/krata/src/idm/mod.rs b/crates/krata/src/idm/mod.rs new file mode 100644 index 0000000..80399ba --- /dev/null +++ b/crates/krata/src/idm/mod.rs @@ -0,0 +1,3 @@ +#[cfg(unix)] +pub mod client; +pub mod protocol; diff --git a/crates/krata/src/idm/protocol.rs b/crates/krata/src/idm/protocol.rs new file mode 100644 index 0000000..f55e9b5 --- /dev/null +++ b/crates/krata/src/idm/protocol.rs @@ -0,0 +1 @@ +include!(concat!(env!("OUT_DIR"), "/krata.internal.idm.rs")); diff --git a/crates/krata/src/lib.rs b/crates/krata/src/lib.rs index 0e63c67..fc97f33 100644 --- a/crates/krata/src/lib.rs +++ b/crates/krata/src/lib.rs @@ -6,6 +6,7 @@ pub mod v1; pub mod client; pub mod dial; pub mod events; +pub mod idm; pub mod launchcfg; #[cfg(target_os = "linux")] diff --git a/crates/kratad/Cargo.toml b/crates/kratad/Cargo.toml index 681cd5e..c49677a 100644 --- a/crates/kratad/Cargo.toml +++ b/crates/kratad/Cargo.toml @@ -8,6 +8,7 @@ resolver = "2" anyhow = { workspace = true } async-stream = { workspace = true } async-trait = { workspace = true } +bytes = { workspace = true } clap = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } diff --git a/crates/kratad/src/event.rs b/crates/kratad/src/event.rs index aecdec9..cb5c8d8 100644 --- a/crates/kratad/src/event.rs +++ b/crates/kratad/src/event.rs @@ -5,7 +5,10 @@ use std::{ }; use anyhow::Result; -use krata::v1::common::{GuestExitInfo, GuestState, GuestStatus}; +use krata::{ + idm::protocol::{idm_packet::Message, IdmPacket}, + v1::common::{GuestExitInfo, GuestState, GuestStatus}, +}; use log::error; use tokio::{ select, @@ -18,14 +21,15 @@ use tokio::{ }; use uuid::Uuid; -use kratart::Runtime; - -use crate::db::GuestStore; +use crate::{ + db::GuestStore, + idm::{DaemonIdmHandle, DaemonIdmSubscribeHandle}, +}; pub type DaemonEvent = krata::v1::control::watch_events_reply::Event; const EVENT_CHANNEL_QUEUE_LEN: usize = 1000; -const EXIT_CODE_CHANNEL_QUEUE_LEN: usize = 1000; +const IDM_CHANNEL_QUEUE_LEN: usize = 1000; #[derive(Clone)] pub struct DaemonEventContext { @@ -44,13 +48,13 @@ impl DaemonEventContext { } pub struct DaemonEventGenerator { - runtime: Runtime, guests: GuestStore, guest_reconciler_notify: Sender, feed: broadcast::Receiver, - exit_code_sender: Sender<(Uuid, i32)>, - exit_code_receiver: Receiver<(Uuid, i32)>, - exit_code_handles: HashMap>, + idm: DaemonIdmHandle, + idms: HashMap, + idm_sender: Sender<(u32, IdmPacket)>, + idm_receiver: Receiver<(u32, IdmPacket)>, _event_sender: broadcast::Sender, } @@ -58,18 +62,18 @@ impl DaemonEventGenerator { pub async fn new( guests: GuestStore, guest_reconciler_notify: Sender, - runtime: Runtime, + idm: DaemonIdmHandle, ) -> Result<(DaemonEventContext, DaemonEventGenerator)> { let (sender, _) = broadcast::channel(EVENT_CHANNEL_QUEUE_LEN); - let (exit_code_sender, exit_code_receiver) = channel(EXIT_CODE_CHANNEL_QUEUE_LEN); + let (idm_sender, idm_receiver) = channel(IDM_CHANNEL_QUEUE_LEN); let generator = DaemonEventGenerator { - runtime, guests, guest_reconciler_notify, feed: sender.subscribe(), - exit_code_receiver, - exit_code_sender, - exit_code_handles: HashMap::new(), + idm, + idms: HashMap::new(), + idm_sender, + idm_receiver, _event_sender: sender.clone(), }; let context = DaemonEventContext { sender }; @@ -89,20 +93,19 @@ impl DaemonEventGenerator { let status = state.status(); let id = Uuid::from_str(&guest.id)?; + let domid = state.domid; match status { GuestStatus::Started => { - if let Entry::Vacant(e) = self.exit_code_handles.entry(id) { - let handle = self - .runtime - .subscribe_exit_code(id, self.exit_code_sender.clone()) - .await?; - e.insert(handle); + if let Entry::Vacant(e) = self.idms.entry(domid) { + let subscribe = + self.idm.subscribe(domid, self.idm_sender.clone()).await?; + e.insert((id, subscribe)); } } GuestStatus::Destroyed => { - if let Some(handle) = self.exit_code_handles.remove(&id) { - handle.abort(); + if let Some((_, handle)) = self.idms.remove(&domid) { + handle.unsubscribe().await?; } } @@ -113,6 +116,13 @@ impl DaemonEventGenerator { Ok(()) } + async fn handle_idm_packet(&mut self, id: Uuid, packet: IdmPacket) -> Result<()> { + if let Some(Message::Exit(exit)) = packet.message { + self.handle_exit_code(id, exit.code).await?; + } + Ok(()) + } + async fn handle_exit_code(&mut self, id: Uuid, code: i32) -> Result<()> { if let Some(mut entry) = self.guests.read(id).await? { let Some(ref mut guest) = entry.guest else { @@ -135,9 +145,12 @@ impl DaemonEventGenerator { async fn evaluate(&mut self) -> Result<()> { select! { - x = self.exit_code_receiver.recv() => match x { - Some((uuid, code)) => { - self.handle_exit_code(uuid, code).await + x = self.idm_receiver.recv() => match x { + Some((domid, packet)) => { + if let Some((id, _)) = self.idms.get(&domid) { + self.handle_idm_packet(*id, packet).await?; + } + Ok(()) }, None => { Ok(()) diff --git a/crates/kratad/src/idm.rs b/crates/kratad/src/idm.rs index f0a1cfb..45c5a96 100644 --- a/crates/kratad/src/idm.rs +++ b/crates/kratad/src/idm.rs @@ -1,9 +1,66 @@ +use std::{collections::HashMap, sync::Arc}; + use anyhow::Result; +use bytes::{Buf, BytesMut}; +use krata::idm::protocol::IdmPacket; use kratart::channel::ChannelService; -use log::error; -use tokio::{sync::mpsc::Receiver, task::JoinHandle}; +use log::{error, warn}; +use prost::Message; +use tokio::{ + sync::{ + mpsc::{Receiver, Sender}, + Mutex, + }, + task::JoinHandle, +}; + +type ListenerMap = Arc>>>; + +#[derive(Clone)] +pub struct DaemonIdmHandle { + listeners: ListenerMap, + task: Arc>, +} + +#[derive(Clone)] +pub struct DaemonIdmSubscribeHandle { + domid: u32, + listeners: ListenerMap, +} + +impl DaemonIdmSubscribeHandle { + pub async fn unsubscribe(&self) -> Result<()> { + let mut guard = self.listeners.lock().await; + let _ = guard.remove(&self.domid); + Ok(()) + } +} + +impl DaemonIdmHandle { + pub async fn subscribe( + &self, + domid: u32, + sender: Sender<(u32, IdmPacket)>, + ) -> Result { + let mut guard = self.listeners.lock().await; + guard.insert(domid, sender); + Ok(DaemonIdmSubscribeHandle { + domid, + listeners: self.listeners.clone(), + }) + } +} + +impl Drop for DaemonIdmHandle { + fn drop(&mut self) { + if Arc::strong_count(&self.task) <= 1 { + self.task.abort(); + } + } +} pub struct DaemonIdm { + listeners: ListenerMap, receiver: Receiver<(u32, Vec)>, task: JoinHandle<()>, } @@ -12,22 +69,60 @@ impl DaemonIdm { pub async fn new() -> Result { let (service, receiver) = ChannelService::new("krata-channel".to_string()).await?; let task = service.launch().await?; - Ok(DaemonIdm { receiver, task }) + let listeners = Arc::new(Mutex::new(HashMap::new())); + Ok(DaemonIdm { + receiver, + task, + listeners, + }) } - pub async fn launch(mut self) -> Result> { - Ok(tokio::task::spawn(async move { - if let Err(error) = self.process().await { + pub async fn launch(mut self) -> Result { + let listeners = self.listeners.clone(); + let task = tokio::task::spawn(async move { + let mut buffers: HashMap = HashMap::new(); + if let Err(error) = self.process(&mut buffers).await { error!("failed to process idm: {}", error); } - })) + }); + Ok(DaemonIdmHandle { + listeners, + task: Arc::new(task), + }) } - async fn process(&mut self) -> Result<()> { + async fn process(&mut self, buffers: &mut HashMap) -> Result<()> { loop { - let Some(_) = self.receiver.recv().await else { + let Some((domid, data)) = self.receiver.recv().await else { break; }; + + let buffer = buffers.entry(domid).or_insert_with_key(|_| BytesMut::new()); + buffer.extend_from_slice(&data); + if buffer.len() < 2 { + continue; + } + let size = (buffer[0] as u16 | (buffer[1] as u16) << 8) as usize; + let needed = size + 2; + if buffer.len() < needed { + continue; + } + let mut packet = buffer.split_to(needed); + packet.advance(2); + match IdmPacket::decode(packet) { + Ok(packet) => { + let guard = self.listeners.lock().await; + if let Some(sender) = guard.get(&domid) { + if let Err(error) = sender.try_send((domid, packet)) { + warn!("dropped idm packet from domain {}: {}", domid, error); + } + } + } + + Err(packet) => { + warn!("received invalid packet from domain {}: {}", domid, packet); + } + } } Ok(()) } diff --git a/crates/kratad/src/lib.rs b/crates/kratad/src/lib.rs index c8f726d..9ee14aa 100644 --- a/crates/kratad/src/lib.rs +++ b/crates/kratad/src/lib.rs @@ -4,7 +4,7 @@ use anyhow::Result; use control::RuntimeControlService; use db::GuestStore; use event::{DaemonEventContext, DaemonEventGenerator}; -use idm::DaemonIdm; +use idm::{DaemonIdm, DaemonIdmHandle}; use krata::{dial::ControlDialAddress, v1::control::control_service_server::ControlServiceServer}; use kratart::Runtime; use log::info; @@ -32,7 +32,7 @@ pub struct Daemon { guest_reconciler_task: JoinHandle<()>, guest_reconciler_notify: Sender, generator_task: JoinHandle<()>, - idm_task: JoinHandle<()>, + _idm: DaemonIdmHandle, } const GUEST_RECONCILER_QUEUE_LEN: usize = 1000; @@ -41,22 +41,18 @@ impl Daemon { pub async fn new(store: String, runtime: Runtime) -> Result { let guests_db_path = format!("{}/guests.db", store); let guests = GuestStore::open(&PathBuf::from(guests_db_path))?; - let runtime_for_events = runtime.dupe().await?; let (guest_reconciler_notify, guest_reconciler_receiver) = channel::(GUEST_RECONCILER_QUEUE_LEN); - let (events, generator) = DaemonEventGenerator::new( - guests.clone(), - guest_reconciler_notify.clone(), - runtime_for_events, - ) - .await?; + let idm = DaemonIdm::new().await?; + let idm = idm.launch().await?; + let (events, generator) = + DaemonEventGenerator::new(guests.clone(), guest_reconciler_notify.clone(), idm.clone()) + .await?; let runtime_for_reconciler = runtime.dupe().await?; let guest_reconciler = GuestReconciler::new(guests.clone(), events.clone(), runtime_for_reconciler)?; let guest_reconciler_task = guest_reconciler.launch(guest_reconciler_receiver).await?; - let idm = DaemonIdm::new().await?; - let idm_task = idm.launch().await?; let generator_task = generator.launch().await?; Ok(Self { store, @@ -66,7 +62,7 @@ impl Daemon { guest_reconciler_task, guest_reconciler_notify, generator_task, - idm_task, + _idm: idm, }) } @@ -130,6 +126,5 @@ impl Drop for Daemon { fn drop(&mut self) { self.guest_reconciler_task.abort(); self.generator_task.abort(); - self.idm_task.abort(); } } diff --git a/crates/krataguest/src/background.rs b/crates/krataguest/src/background.rs index 5b4ff27..d114a2e 100644 --- a/crates/krataguest/src/background.rs +++ b/crates/krataguest/src/background.rs @@ -3,17 +3,24 @@ use crate::{ death, }; use anyhow::Result; +use krata::idm::{ + client::IdmClient, + protocol::{idm_packet::Message, IdmExitMessage, IdmPacket}, +}; +use log::error; use nix::unistd::Pid; use tokio::select; pub struct GuestBackground { + idm: IdmClient, child: Pid, wait: ChildWait, } impl GuestBackground { - pub async fn new(child: Pid) -> Result { + pub async fn new(idm: IdmClient, child: Pid) -> Result { Ok(GuestBackground { + idm, child, wait: ChildWait::new()?, }) @@ -22,6 +29,17 @@ impl GuestBackground { pub async fn run(&mut self) -> Result<()> { loop { select! { + x = self.idm.receiver.recv() => match x { + Some(_packet) => { + + }, + + None => { + error!("idm packet channel closed"); + break; + } + }, + event = self.wait.recv() => match event { Some(event) => self.child_event(event).await?, None => { @@ -35,6 +53,12 @@ impl GuestBackground { async fn child_event(&mut self, event: ChildEvent) -> Result<()> { if event.pid == self.child { + self.idm + .sender + .send(IdmPacket { + message: Some(Message::Exit(IdmExitMessage { code: event.status })), + }) + .await?; death(event.status).await?; } Ok(()) diff --git a/crates/krataguest/src/init.rs b/crates/krataguest/src/init.rs index 99404a6..90f12c8 100644 --- a/crates/krataguest/src/init.rs +++ b/crates/krataguest/src/init.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, Result}; use futures::stream::TryStreamExt; use ipnetwork::IpNetwork; use krata::ethtool::EthtoolHandle; +use krata::idm::client::IdmClient; use krata::launchcfg::{LaunchInfo, LaunchNetwork}; use libc::{setsid, TIOCSCTTY}; use log::{trace, warn}; @@ -12,6 +13,7 @@ use path_absolutize::Absolutize; use std::collections::HashMap; use std::ffi::CString; use std::fs::{File, OpenOptions, Permissions}; +use std::io; use std::net::{Ipv4Addr, Ipv6Addr}; use std::os::fd::AsRawFd; use std::os::linux::fs::MetadataExt; @@ -19,8 +21,8 @@ use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{chroot, symlink, PermissionsExt}; use std::path::{Path, PathBuf}; use std::str::FromStr; -use std::{fs, io}; use sys_mount::{FilesystemType, Mount, MountFlags}; +use tokio::fs; use walkdir::WalkDir; use crate::background::GuestBackground; @@ -64,7 +66,7 @@ impl GuestInit { } pub async fn init(&mut self) -> Result<()> { - self.early_init()?; + self.early_init().await?; trace!("opening console descriptor"); match OpenOptions::new() @@ -76,21 +78,28 @@ impl GuestInit { Err(error) => warn!("failed to open console: {}", error), }; - self.mount_squashfs_images()?; - let config = self.parse_image_config()?; - let launch = self.parse_launch_config()?; - self.mount_new_root()?; - self.nuke_initrd()?; - self.bind_new_root()?; + let idm = IdmClient::open("/dev/hvc1") + .await + .map_err(|x| anyhow!("failed to open idm client: {}", x))?; + self.mount_squashfs_images().await?; + + let config = self.parse_image_config().await?; + let launch = self.parse_launch_config().await?; + + self.mount_new_root().await?; + self.nuke_initrd().await?; + self.bind_new_root().await?; if let Some(network) = &launch.network { + trace!("initializing network"); if let Err(error) = self.network_setup(network).await { warn!("failed to initialize network: {}", error); } } if let Some(cfg) = config.config() { - self.run(cfg, &launch).await?; + trace!("running guest task"); + self.run(cfg, &launch, idm).await?; } else { return Err(anyhow!( "unable to determine what to execute, image config doesn't tell us" @@ -99,37 +108,38 @@ impl GuestInit { Ok(()) } - fn early_init(&mut self) -> Result<()> { + async fn early_init(&mut self) -> Result<()> { trace!("early init"); - self.create_dir("/dev", Some(0o0755))?; - self.create_dir("/proc", None)?; - self.create_dir("/sys", None)?; - self.create_dir("/root", Some(0o0700))?; - self.create_dir("/tmp", None)?; - self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755")?; - self.mount_kernel_fs("proc", "/proc", "")?; - self.mount_kernel_fs("sysfs", "/sys", "")?; + self.create_dir("/dev", Some(0o0755)).await?; + self.create_dir("/proc", None).await?; + self.create_dir("/sys", None).await?; + self.create_dir("/root", Some(0o0700)).await?; + self.create_dir("/tmp", None).await?; + self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755") + .await?; + self.mount_kernel_fs("proc", "/proc", "").await?; + self.mount_kernel_fs("sysfs", "/sys", "").await?; symlink("/proc/self/fd", "/dev/fd")?; Ok(()) } - fn create_dir(&mut self, path: &str, mode: Option) -> Result<()> { + async fn create_dir(&mut self, path: &str, mode: Option) -> Result<()> { let path = Path::new(path); if !path.is_dir() { trace!("creating directory {:?}", path); - fs::create_dir(path)?; + fs::create_dir(path).await?; } if let Some(mode) = mode { let permissions = Permissions::from_mode(mode); trace!("setting directory {:?} permissions to {:?}", path, mode); - fs::set_permissions(path, permissions)?; + fs::set_permissions(path, permissions).await?; } Ok(()) } - fn mount_kernel_fs(&mut self, fstype: &str, path: &str, data: &str) -> Result<()> { - let metadata = fs::metadata(path)?; - if metadata.st_dev() == fs::metadata("/")?.st_dev() { + async fn mount_kernel_fs(&mut self, fstype: &str, path: &str, data: &str) -> Result<()> { + let metadata = fs::metadata(path).await?; + if metadata.st_dev() == fs::metadata("/").await?.st_dev() { trace!("mounting kernel fs {} to {}", fstype, path); Mount::builder() .fstype(FilesystemType::Manual(fstype)) @@ -148,19 +158,21 @@ impl GuestInit { Ok(()) } - fn mount_squashfs_images(&mut self) -> Result<()> { + async fn mount_squashfs_images(&mut self) -> Result<()> { trace!("mounting squashfs images"); let image_mount_path = Path::new(IMAGE_MOUNT_PATH); let config_mount_path = Path::new(CONFIG_MOUNT_PATH); - self.mount_squashfs(Path::new(IMAGE_BLOCK_DEVICE_PATH), image_mount_path)?; - self.mount_squashfs(Path::new(CONFIG_BLOCK_DEVICE_PATH), config_mount_path)?; + self.mount_squashfs(Path::new(IMAGE_BLOCK_DEVICE_PATH), image_mount_path) + .await?; + self.mount_squashfs(Path::new(CONFIG_BLOCK_DEVICE_PATH), config_mount_path) + .await?; Ok(()) } - fn mount_squashfs(&mut self, from: &Path, to: &Path) -> Result<()> { + async fn mount_squashfs(&mut self, from: &Path, to: &Path) -> Result<()> { trace!("mounting squashfs image {:?} to {:?}", from, to); if !to.is_dir() { - fs::create_dir(to)?; + fs::create_dir(to).await?; } Mount::builder() .fstype(FilesystemType::Manual("squashfs")) @@ -169,10 +181,10 @@ impl GuestInit { Ok(()) } - fn mount_move_subtree(&mut self, from: &Path, to: &Path) -> Result<()> { + async fn mount_move_subtree(&mut self, from: &Path, to: &Path) -> Result<()> { trace!("moving subtree {:?} to {:?}", from, to); if !to.is_dir() { - fs::create_dir(to)?; + fs::create_dir(to).await?; } Mount::builder() .fstype(FilesystemType::Manual("none")) @@ -181,28 +193,28 @@ impl GuestInit { Ok(()) } - fn mount_new_root(&mut self) -> Result<()> { + async fn mount_new_root(&mut self) -> Result<()> { trace!("mounting new root"); - self.mount_overlay_tmpfs()?; - self.bind_image_to_overlay_tmpfs()?; - self.mount_overlay_to_new_root()?; + self.mount_overlay_tmpfs().await?; + self.bind_image_to_overlay_tmpfs().await?; + self.mount_overlay_to_new_root().await?; std::env::set_current_dir(NEW_ROOT_PATH)?; trace!("mounted new root"); Ok(()) } - fn mount_overlay_tmpfs(&mut self) -> Result<()> { - fs::create_dir(OVERLAY_MOUNT_PATH)?; + async fn mount_overlay_tmpfs(&mut self) -> Result<()> { + fs::create_dir(OVERLAY_MOUNT_PATH).await?; Mount::builder() .fstype(FilesystemType::Manual("tmpfs")) .mount("tmpfs", OVERLAY_MOUNT_PATH)?; - fs::create_dir(OVERLAY_UPPER_PATH)?; - fs::create_dir(OVERLAY_WORK_PATH)?; + fs::create_dir(OVERLAY_UPPER_PATH).await?; + fs::create_dir(OVERLAY_WORK_PATH).await?; Ok(()) } - fn bind_image_to_overlay_tmpfs(&mut self) -> Result<()> { - fs::create_dir(OVERLAY_IMAGE_BIND_PATH)?; + async fn bind_image_to_overlay_tmpfs(&mut self) -> Result<()> { + fs::create_dir(OVERLAY_IMAGE_BIND_PATH).await?; Mount::builder() .fstype(FilesystemType::Manual("none")) .flags(MountFlags::BIND | MountFlags::RDONLY) @@ -210,8 +222,8 @@ impl GuestInit { Ok(()) } - fn mount_overlay_to_new_root(&mut self) -> Result<()> { - fs::create_dir(NEW_ROOT_PATH)?; + async fn mount_overlay_to_new_root(&mut self) -> Result<()> { + fs::create_dir(NEW_ROOT_PATH).await?; Mount::builder() .fstype(FilesystemType::Manual("overlay")) .flags(MountFlags::NOATIME) @@ -223,22 +235,23 @@ impl GuestInit { Ok(()) } - fn parse_image_config(&mut self) -> Result { - trace!("parsing image config"); + async fn parse_image_config(&mut self) -> Result { let image_config_path = Path::new(IMAGE_CONFIG_JSON_PATH); - let config = ImageConfiguration::from_file(image_config_path)?; + let content = fs::read_to_string(image_config_path).await?; + let config = serde_json::from_str(&content)?; Ok(config) } - fn parse_launch_config(&mut self) -> Result { + async fn parse_launch_config(&mut self) -> Result { trace!("parsing launch config"); let launch_config = Path::new(LAUNCH_CONFIG_JSON_PATH); - Ok(serde_json::from_str(&fs::read_to_string(launch_config)?)?) + let content = fs::read_to_string(launch_config).await?; + Ok(serde_json::from_str(&content)?) } - fn nuke_initrd(&mut self) -> Result<()> { + async fn nuke_initrd(&mut self) -> Result<()> { trace!("nuking initrd"); - let initrd_dev = fs::metadata("/")?.st_dev(); + let initrd_dev = fs::metadata("/").await?.st_dev(); for item in WalkDir::new("/") .same_file_system(true) .follow_links(false) @@ -259,10 +272,10 @@ impl GuestInit { } if metadata.is_symlink() || metadata.is_file() { - let _ = fs::remove_file(item.path()); + let _ = fs::remove_file(item.path()).await; trace!("deleting file {:?}", item.path()); } else if metadata.is_dir() { - let _ = fs::remove_dir(item.path()); + let _ = fs::remove_dir(item.path()).await; trace!("deleting directory {:?}", item.path()); } } @@ -270,10 +283,13 @@ impl GuestInit { Ok(()) } - fn bind_new_root(&mut self) -> Result<()> { - self.mount_move_subtree(Path::new(SYS_PATH), Path::new(NEW_ROOT_SYS_PATH))?; - self.mount_move_subtree(Path::new(PROC_PATH), Path::new(NEW_ROOT_PROC_PATH))?; - self.mount_move_subtree(Path::new(DEV_PATH), Path::new(NEW_ROOT_DEV_PATH))?; + async fn bind_new_root(&mut self) -> Result<()> { + self.mount_move_subtree(Path::new(SYS_PATH), Path::new(NEW_ROOT_SYS_PATH)) + .await?; + self.mount_move_subtree(Path::new(PROC_PATH), Path::new(NEW_ROOT_PROC_PATH)) + .await?; + self.mount_move_subtree(Path::new(DEV_PATH), Path::new(NEW_ROOT_DEV_PATH)) + .await?; trace!("binding new root"); Mount::builder() .fstype(FilesystemType::Manual("none")) @@ -291,7 +307,7 @@ impl GuestInit { let etc = PathBuf::from_str("/etc")?; if !etc.exists() { - fs::create_dir(etc)?; + fs::create_dir(etc).await?; } let resolv = PathBuf::from_str("/etc/resolv.conf")?; let mut lines = vec!["# krata resolver configuration".to_string()]; @@ -301,7 +317,7 @@ impl GuestInit { let mut conf = lines.join("\n"); conf.push('\n'); - fs::write(resolv, conf)?; + fs::write(resolv, conf).await?; self.network_configure_ethtool(network).await?; self.network_configure_link(network).await?; Ok(()) @@ -383,7 +399,7 @@ impl GuestInit { Ok(()) } - async fn run(&mut self, config: &Config, launch: &LaunchInfo) -> Result<()> { + async fn run(&mut self, config: &Config, launch: &LaunchInfo, idm: IdmClient) -> Result<()> { let mut cmd = match config.cmd() { None => vec![], Some(value) => value.clone(), @@ -423,7 +439,7 @@ impl GuestInit { cmd.insert(0, file_name.to_string()); let env = GuestInit::env_list(env); - trace!("running container command: {}", cmd.join(" ")); + trace!("running guest command: {}", cmd.join(" ")); let path = CString::new(path.as_os_str().as_bytes())?; let cmd = GuestInit::strings_as_cstrings(cmd)?; @@ -438,7 +454,7 @@ impl GuestInit { working_dir = "/".to_string(); } - self.fork_and_exec(working_dir, path, cmd, env).await?; + self.fork_and_exec(idm, working_dir, path, cmd, env).await?; Ok(()) } @@ -489,13 +505,14 @@ impl GuestInit { async fn fork_and_exec( &mut self, + idm: IdmClient, working_dir: String, path: CString, cmd: Vec, env: Vec, ) -> Result<()> { match unsafe { fork()? } { - ForkResult::Parent { child } => self.background(child).await, + ForkResult::Parent { child } => self.background(idm, child).await, ForkResult::Child => self.foreground(working_dir, path, cmd, env).await, } } @@ -521,8 +538,8 @@ impl GuestInit { Ok(()) } - async fn background(&mut self, executed: Pid) -> Result<()> { - let mut background = GuestBackground::new(executed).await?; + async fn background(&mut self, idm: IdmClient, executed: Pid) -> Result<()> { + let mut background = GuestBackground::new(idm, executed).await?; background.run().await?; Ok(()) }