feat: implement kernel / initrd oci image support (#103)

* feat: implement kernel / initrd oci image support

* fix: implement image urls more faithfully

crates/daemon/Cargo.toml

@@ -27,6 +27,7 @@ scopeguard = { workspace = true }
 signal-hook = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
+krata-tokio-tar = { workspace = true }
 tonic = { workspace = true, features = ["tls"] }
 uuid = { workspace = true }
 

crates/daemon/src/control.rs

@@ -378,7 +378,7 @@ impl ControlService for DaemonControlService {
                             format: match packed.format {
                                 OciPackedFormat::Squashfs => OciImageFormat::Squashfs.into(),
                                 OciPackedFormat::Erofs => OciImageFormat::Erofs.into(),
-                                _ => OciImageFormat::Unknown.into(),
+                                OciPackedFormat::Tar => OciImageFormat::Tar.into(),
                             },
                         };
                         yield reply;

crates/daemon/src/lib.rs

@@ -1,6 +1,6 @@
 use std::{net::SocketAddr, path::PathBuf, str::FromStr};
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use console::{DaemonConsole, DaemonConsoleHandle};
 use control::DaemonControlService;
 use db::GuestStore;
@@ -74,9 +74,11 @@ impl Daemon {
             generated
         };
 
-        let packer = OciPackerService::new(None, &image_cache_dir, OciPlatform::current()).await?;
+        let initrd_path = detect_guest_file(&store, "initrd")?;
+        let kernel_path = detect_guest_file(&store, "kernel")?;
 
-        let runtime = Runtime::new(store.clone()).await?;
+        let packer = OciPackerService::new(None, &image_cache_dir, OciPlatform::current()).await?;
+        let runtime = Runtime::new().await?;
         let glt = GuestLookupTable::new(0, host_uuid);
         let guests_db_path = format!("{}/guests.db", store);
         let guests = GuestStore::open(&PathBuf::from(guests_db_path))?;
@@ -97,6 +99,8 @@ impl Daemon {
             runtime_for_reconciler,
             packer.clone(),
             guest_reconciler_notify.clone(),
+            kernel_path,
+            initrd_path,
         )?;
 
         let guest_reconciler_task = guest_reconciler.launch(guest_reconciler_receiver).await?;
@@ -181,3 +185,16 @@ impl Drop for Daemon {
         self.generator_task.abort();
     }
 }
+
+fn detect_guest_file(store: &str, name: &str) -> Result<PathBuf> {
+    let mut path = PathBuf::from(format!("{}/guest/{}", store, name));
+    if path.is_file() {
+        return Ok(path);
+    }
+
+    path = PathBuf::from(format!("/usr/share/krata/guest/{}", name));
+    if path.is_file() {
+        return Ok(path);
+    }
+    Err(anyhow!("unable to find required guest file: {}", name))
+}

crates/daemon/src/reconcile/guest/mod.rs

@@ -1,20 +1,17 @@
 use std::{
     collections::{hash_map::Entry, HashMap},
+    path::PathBuf,
     sync::Arc,
     time::Duration,
 };
 
-use anyhow::{anyhow, Result};
-use krata::launchcfg::LaunchPackedFormat;
+use anyhow::Result;
 use krata::v1::{
-    common::{
-        guest_image_spec::Image, Guest, GuestErrorInfo, GuestExitInfo, GuestNetworkState,
-        GuestState, GuestStatus, OciImageFormat,
-    },
+    common::{Guest, GuestErrorInfo, GuestExitInfo, GuestNetworkState, GuestState, GuestStatus},
     control::GuestChangedEvent,
 };
-use krataoci::packer::{service::OciPackerService, OciPackedFormat};
-use kratart::{launch::GuestLaunchRequest, GuestInfo, Runtime};
+use krataoci::packer::service::OciPackerService;
+use kratart::{GuestInfo, Runtime};
 use log::{error, info, trace, warn};
 use tokio::{
     select,
@@ -33,6 +30,10 @@ use crate::{
     glt::GuestLookupTable,
 };
 
+use self::start::GuestStarter;
+
+mod start;
+
 const PARALLEL_LIMIT: u32 = 5;
 
 #[derive(Debug)]
@@ -59,12 +60,15 @@ pub struct GuestReconciler {
     events: DaemonEventContext,
     runtime: Runtime,
     packer: OciPackerService,
+    kernel_path: PathBuf,
+    initrd_path: PathBuf,
     tasks: Arc<Mutex<HashMap<Uuid, GuestReconcilerEntry>>>,
     guest_reconciler_notify: Sender<Uuid>,
     reconcile_lock: Arc<RwLock<()>>,
 }
 
 impl GuestReconciler {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         glt: GuestLookupTable,
         guests: GuestStore,
@@ -72,6 +76,8 @@ impl GuestReconciler {
         runtime: Runtime,
         packer: OciPackerService,
         guest_reconciler_notify: Sender<Uuid>,
+        kernel_path: PathBuf,
+        initrd_path: PathBuf,
     ) -> Result<Self> {
         Ok(Self {
             glt,
@@ -79,6 +85,8 @@ impl GuestReconciler {
             events,
             runtime,
             packer,
+            kernel_path,
+            initrd_path,
             tasks: Arc::new(Mutex::new(HashMap::new())),
             guest_reconciler_notify,
             reconcile_lock: Arc::new(RwLock::with_max_readers((), PARALLEL_LIMIT)),
@@ -246,76 +254,14 @@ impl GuestReconciler {
     }
 
     async fn start(&self, uuid: Uuid, guest: &mut Guest) -> Result<GuestReconcilerResult> {
-        let Some(ref spec) = guest.spec else {
-            return Err(anyhow!("guest spec not specified"));
+        let starter = GuestStarter {
+            kernel_path: &self.kernel_path,
+            initrd_path: &self.initrd_path,
+            packer: &self.packer,
+            glt: &self.glt,
+            runtime: &self.runtime,
         };
-
-        let Some(ref image) = spec.image else {
-            return Err(anyhow!("image spec not provided"));
-        };
-        let oci = match image.image {
-            Some(Image::Oci(ref oci)) => oci,
-            None => {
-                return Err(anyhow!("oci spec not specified"));
-            }
-        };
-        let task = spec.task.as_ref().cloned().unwrap_or_default();
-
-        let image = self
-            .packer
-            .recall(
-                &oci.digest,
-                match oci.format() {
-                    OciImageFormat::Unknown => OciPackedFormat::Squashfs,
-                    OciImageFormat::Squashfs => OciPackedFormat::Squashfs,
-                    OciImageFormat::Erofs => OciPackedFormat::Erofs,
-                    OciImageFormat::Tar => {
-                        return Err(anyhow!("tar image format is not supported for guests"));
-                    }
-                },
-            )
-            .await?;
-
-        let Some(image) = image else {
-            return Err(anyhow!(
-                "image {} in the requested format did not exist",
-                oci.digest
-            ));
-        };
-
-        let info = self
-            .runtime
-            .launch(GuestLaunchRequest {
-                format: LaunchPackedFormat::Squashfs,
-                uuid: Some(uuid),
-                name: if spec.name.is_empty() {
-                    None
-                } else {
-                    Some(spec.name.clone())
-                },
-                image,
-                vcpus: spec.vcpus,
-                mem: spec.mem,
-                env: task
-                    .environment
-                    .iter()
-                    .map(|x| (x.key.clone(), x.value.clone()))
-                    .collect::<HashMap<_, _>>(),
-                run: empty_vec_optional(task.command.clone()),
-                debug: false,
-            })
-            .await?;
-        self.glt.associate(uuid, info.domid).await;
-        info!("started guest {}", uuid);
-        guest.state = Some(GuestState {
-            status: GuestStatus::Started.into(),
-            network: Some(guestinfo_to_networkstate(&info)),
-            exit_info: None,
-            error_info: None,
-            host: self.glt.host_uuid().to_string(),
-            domid: info.domid,
-        });
-        Ok(GuestReconcilerResult::Changed { rerun: false })
+        starter.start(uuid, guest).await
     }
 
     async fn exited(&self, guest: &mut Guest) -> Result<GuestReconcilerResult> {
@@ -390,15 +336,7 @@ impl GuestReconciler {
     }
 }
 
-fn empty_vec_optional<T>(value: Vec<T>) -> Option<Vec<T>> {
-    if value.is_empty() {
-        None
-    } else {
-        Some(value)
-    }
-}
-
-fn guestinfo_to_networkstate(info: &GuestInfo) -> GuestNetworkState {
+pub fn guestinfo_to_networkstate(info: &GuestInfo) -> GuestNetworkState {
     GuestNetworkState {
         guest_ipv4: info.guest_ipv4.map(|x| x.to_string()).unwrap_or_default(),
         guest_ipv6: info.guest_ipv6.map(|x| x.to_string()).unwrap_or_default(),

crates/daemon/src/reconcile/guest/start.rs

@@ -0,0 +1,182 @@
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use anyhow::{anyhow, Result};
+use futures::StreamExt;
+use krata::launchcfg::LaunchPackedFormat;
+use krata::v1::common::GuestOciImageSpec;
+use krata::v1::common::{guest_image_spec::Image, Guest, GuestState, GuestStatus, OciImageFormat};
+use krataoci::packer::{service::OciPackerService, OciPackedFormat};
+use kratart::{launch::GuestLaunchRequest, Runtime};
+use log::info;
+
+use tokio::fs::{self, File};
+use tokio::io::AsyncReadExt;
+use tokio_tar::Archive;
+use uuid::Uuid;
+
+use crate::{
+    glt::GuestLookupTable,
+    reconcile::guest::{guestinfo_to_networkstate, GuestReconcilerResult},
+};
+
+// if a kernel is >= 100MB, that's kinda scary.
+const OCI_SPEC_TAR_FILE_MAX_SIZE: usize = 100 * 1024 * 1024;
+
+pub struct GuestStarter<'a> {
+    pub kernel_path: &'a Path,
+    pub initrd_path: &'a Path,
+    pub packer: &'a OciPackerService,
+    pub glt: &'a GuestLookupTable,
+    pub runtime: &'a Runtime,
+}
+
+impl GuestStarter<'_> {
+    pub async fn oci_spec_tar_read_file(
+        &self,
+        file: &Path,
+        oci: &GuestOciImageSpec,
+    ) -> Result<Vec<u8>> {
+        if oci.format() != OciImageFormat::Tar {
+            return Err(anyhow!(
+                "oci image spec for {} is required to be in tar format",
+                oci.digest
+            ));
+        }
+
+        let image = self
+            .packer
+            .recall(&oci.digest, OciPackedFormat::Tar)
+            .await?;
+
+        let Some(image) = image else {
+            return Err(anyhow!("image {} was not found in tar format", oci.digest));
+        };
+
+        let mut archive = Archive::new(File::open(&image.path).await?);
+        let mut entries = archive.entries()?;
+        while let Some(entry) = entries.next().await {
+            let mut entry = entry?;
+            let path = entry.path()?;
+            if entry.header().size()? as usize > OCI_SPEC_TAR_FILE_MAX_SIZE {
+                return Err(anyhow!(
+                    "file {} in image {} is larger than the size limit",
+                    file.to_string_lossy(),
+                    oci.digest
+                ));
+            }
+            if path == file {
+                let mut buffer = Vec::new();
+                entry.read_to_end(&mut buffer).await?;
+                return Ok(buffer);
+            }
+        }
+        Err(anyhow!(
+            "unable to find file {} in image {}",
+            file.to_string_lossy(),
+            oci.digest
+        ))
+    }
+
+    pub async fn start(&self, uuid: Uuid, guest: &mut Guest) -> Result<GuestReconcilerResult> {
+        let Some(ref spec) = guest.spec else {
+            return Err(anyhow!("guest spec not specified"));
+        };
+
+        let Some(ref image) = spec.image else {
+            return Err(anyhow!("image spec not provided"));
+        };
+        let oci = match image.image {
+            Some(Image::Oci(ref oci)) => oci,
+            None => {
+                return Err(anyhow!("oci spec not specified"));
+            }
+        };
+        let task = spec.task.as_ref().cloned().unwrap_or_default();
+
+        let image = self
+            .packer
+            .recall(
+                &oci.digest,
+                match oci.format() {
+                    OciImageFormat::Unknown => OciPackedFormat::Squashfs,
+                    OciImageFormat::Squashfs => OciPackedFormat::Squashfs,
+                    OciImageFormat::Erofs => OciPackedFormat::Erofs,
+                    OciImageFormat::Tar => {
+                        return Err(anyhow!("tar image format is not supported for guests"));
+                    }
+                },
+            )
+            .await?;
+
+        let Some(image) = image else {
+            return Err(anyhow!(
+                "image {} in the requested format did not exist",
+                oci.digest
+            ));
+        };
+
+        let kernel = if let Some(ref spec) = spec.kernel {
+            let Some(Image::Oci(ref oci)) = spec.image else {
+                return Err(anyhow!("kernel image spec must be an oci image"));
+            };
+            self.oci_spec_tar_read_file(&PathBuf::from("kernel/image"), oci)
+                .await?
+        } else {
+            fs::read(&self.kernel_path).await?
+        };
+        let initrd = if let Some(ref spec) = spec.initrd {
+            let Some(Image::Oci(ref oci)) = spec.image else {
+                return Err(anyhow!("initrd image spec must be an oci image"));
+            };
+            self.oci_spec_tar_read_file(&PathBuf::from("krata/initrd"), oci)
+                .await?
+        } else {
+            fs::read(&self.initrd_path).await?
+        };
+
+        let info = self
+            .runtime
+            .launch(GuestLaunchRequest {
+                format: LaunchPackedFormat::Squashfs,
+                uuid: Some(uuid),
+                name: if spec.name.is_empty() {
+                    None
+                } else {
+                    Some(spec.name.clone())
+                },
+                image,
+                kernel,
+                initrd,
+                vcpus: spec.vcpus,
+                mem: spec.mem,
+                env: task
+                    .environment
+                    .iter()
+                    .map(|x| (x.key.clone(), x.value.clone()))
+                    .collect::<HashMap<_, _>>(),
+                run: empty_vec_optional(task.command.clone()),
+                debug: false,
+            })
+            .await?;
+        self.glt.associate(uuid, info.domid).await;
+        info!("started guest {}", uuid);
+        guest.state = Some(GuestState {
+            status: GuestStatus::Started.into(),
+            network: Some(guestinfo_to_networkstate(&info)),
+            exit_info: None,
+            error_info: None,
+            host: self.glt.host_uuid().to_string(),
+            domid: info.domid,
+        });
+        Ok(GuestReconcilerResult::Changed { rerun: false })
+    }
+}
+
+fn empty_vec_optional<T>(value: Vec<T>) -> Option<Vec<T>> {
+    if value.is_empty() {
+        None
+    } else {
+        Some(value)
+    }
+}