feat: addons mounting and kernel modules support

Cargo.lock

@@ -1481,6 +1481,7 @@ dependencies = [
  "nix 0.28.0",
  "oci-spec",
  "path-absolutize",
+ "platform-info",
  "rtnetlink",
  "serde",
  "serde_json",
@@ -1560,6 +1561,7 @@ dependencies = [
  "serde_json",
  "tokio",
  "uuid",
+ "walkdir",
 ]
 
 [[package]]
@@ -2070,6 +2072,16 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
+[[package]]
+name = "platform-info"
+version = "2.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5ff316b9c4642feda973c18f0decd6c8b0919d4722566f6e4337cce0dd88217"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "portable-atomic"
 version = "1.6.0"

Cargo.toml

@@ -57,6 +57,7 @@ oci-spec = "0.6.4"
 once_cell = "1.19.0"
 path-absolutize = "3.1.1"
 path-clean = "1.0.1"
+platform-info = "2.0.3"
 prost = "0.12.4"
 prost-build = "0.12.4"
 prost-reflect-build = "0.13.0"

crates/daemon/src/lib.rs

@@ -88,8 +88,9 @@ impl Daemon {
             generated
         };
 
-        let initrd_path = detect_guest_file(&store, "initrd")?;
+        let initrd_path = detect_guest_path(&store, "initrd")?;
-        let kernel_path = detect_guest_file(&store, "kernel")?;
+        let kernel_path = detect_guest_path(&store, "kernel")?;
+        let addons_path = detect_guest_path(&store, "addons.squashfs")?;
 
         let packer = OciPackerService::new(None, &image_cache_dir, OciPlatform::current()).await?;
         let runtime = Runtime::new().await?;
@@ -116,6 +117,7 @@ impl Daemon {
             guest_reconciler_notify.clone(),
             kernel_path,
             initrd_path,
+            addons_path,
         )?;
 
         let guest_reconciler_task = guest_reconciler.launch(guest_reconciler_receiver).await?;
@@ -204,7 +206,7 @@ impl Drop for Daemon {
     }
 }
 
-fn detect_guest_file(store: &str, name: &str) -> Result<PathBuf> {
+fn detect_guest_path(store: &str, name: &str) -> Result<PathBuf> {
     let mut path = PathBuf::from(format!("{}/guest/{}", store, name));
     if path.is_file() {
         return Ok(path);

crates/daemon/src/reconcile/guest/mod.rs

@@ -64,6 +64,7 @@ pub struct GuestReconciler {
     packer: OciPackerService,
     kernel_path: PathBuf,
     initrd_path: PathBuf,
+    addons_path: PathBuf,
     tasks: Arc<Mutex<HashMap<Uuid, GuestReconcilerEntry>>>,
     guest_reconciler_notify: Sender<Uuid>,
     reconcile_lock: Arc<RwLock<()>>,
@@ -81,6 +82,7 @@ impl GuestReconciler {
         guest_reconciler_notify: Sender<Uuid>,
         kernel_path: PathBuf,
         initrd_path: PathBuf,
+        modules_path: PathBuf,
     ) -> Result<Self> {
         Ok(Self {
             devices,
@@ -91,6 +93,7 @@ impl GuestReconciler {
             packer,
             kernel_path,
             initrd_path,
+            addons_path: modules_path,
             tasks: Arc::new(Mutex::new(HashMap::new())),
             guest_reconciler_notify,
             reconcile_lock: Arc::new(RwLock::with_max_readers((), PARALLEL_LIMIT)),
@@ -278,6 +281,7 @@ impl GuestReconciler {
             devices: &self.devices,
             kernel_path: &self.kernel_path,
             initrd_path: &self.initrd_path,
+            addons_path: &self.addons_path,
             packer: &self.packer,
             glt: &self.glt,
             runtime: &self.runtime,

crates/daemon/src/reconcile/guest/start.rs

@@ -32,6 +32,7 @@ pub struct GuestStarter<'a> {
     pub devices: &'a DaemonDeviceManager,
     pub kernel_path: &'a Path,
     pub initrd_path: &'a Path,
+    pub addons_path: &'a Path,
     pub packer: &'a OciPackerService,
     pub glt: &'a GuestLookupTable,
     pub runtime: &'a Runtime,
@@ -206,6 +207,7 @@ impl GuestStarter<'_> {
                     .collect::<HashMap<_, _>>(),
                 run: empty_vec_optional(task.command.clone()),
                 debug: false,
+                addons_image: Some(self.addons_path.to_path_buf()),
             })
             .await?;
         self.glt.associate(uuid, info.domid).await;

crates/guest/Cargo.toml

@@ -21,6 +21,7 @@ log = { workspace = true }
 nix = { workspace = true, features = ["ioctl", "process", "fs"] }
 oci-spec = { workspace = true }
 path-absolutize = { workspace = true }
+platform-info = { workspace = true }
 rtnetlink = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }

crates/guest/src/init.rs

@@ -12,6 +12,7 @@ use nix::ioctl_write_int_bad;
 use nix::unistd::{dup2, execve, fork, ForkResult, Pid};
 use oci_spec::image::{Config, ImageConfiguration};
 use path_absolutize::Absolutize;
+use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI};
 use std::collections::HashMap;
 use std::ffi::CString;
 use std::fs::{File, OpenOptions, Permissions};
@@ -50,6 +51,10 @@ const NEW_ROOT_DEV_PATH: &str = "/newroot/dev";
 const IMAGE_CONFIG_JSON_PATH: &str = "/config/image/config.json";
 const LAUNCH_CONFIG_JSON_PATH: &str = "/config/launch.json";
 
+const ADDONS_DEVICE_PATH: &str = "/dev/xvdc";
+const ADDONS_MOUNT_PATH: &str = "/addons";
+const ADDONS_MODULES_PATH: &str = "/addons/modules";
+
 ioctl_write_int_bad!(set_controlling_terminal, TIOCSCTTY);
 
 pub struct GuestInit {}
@@ -88,7 +93,10 @@ impl GuestInit {
 
         self.mount_root_image(launch.root.format.clone()).await?;
 
+        self.mount_addons().await?;
+
         self.mount_new_root().await?;
+        self.mount_kernel_modules().await?;
         self.bind_new_root().await?;
 
         if let Some(hostname) = launch.hostname.clone() {
@@ -137,16 +145,60 @@ impl GuestInit {
         self.create_dir("/root", Some(0o0700)).await?;
         self.create_dir("/tmp", None).await?;
         self.create_dir("/run", Some(0o0755)).await?;
-        self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755", None)
+        self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755", None, None)
+            .await?;
+        self.mount_kernel_fs("proc", "/proc", "", None, None)
+            .await?;
+        self.mount_kernel_fs("sysfs", "/sys", "", None, None)
+            .await?;
+        self.create_dir("/dev/pts", Some(0o0755)).await?;
+        self.mount_kernel_fs("devpts", "/dev/pts", "", None, Some("/dev/ptmx"))
             .await?;
-        self.mount_kernel_fs("proc", "/proc", "", None).await?;
-        self.mount_kernel_fs("sysfs", "/sys", "", None).await?;
         fs::symlink("/proc/self/fd", "/dev/fd").await?;
         fs::symlink("/proc/self/fd/0", "/dev/stdin").await?;
         fs::symlink("/proc/self/fd/1", "/dev/stdout").await?;
         fs::symlink("/proc/self/fd/2", "/dev/stderr").await?;
-        self.mount_kernel_fs("cgroup2", "/sys/fs/cgroup", "", Some(MountFlags::RELATIME))
+        self.mount_kernel_fs(
-            .await?;
+            "cgroup2",
+            "/sys/fs/cgroup",
+            "",
+            Some(MountFlags::RELATIME),
+            None,
+        )
+        .await?;
+        Ok(())
+    }
+
+    async fn mount_addons(&mut self) -> Result<()> {
+        if !fs::try_exists(ADDONS_DEVICE_PATH).await? {
+            return Ok(());
+        }
+
+        self.mount_image(
+            &PathBuf::from(ADDONS_DEVICE_PATH),
+            &PathBuf::from(ADDONS_MOUNT_PATH),
+            LaunchPackedFormat::Squashfs,
+        )
+        .await?;
+        Ok(())
+    }
+
+    async fn mount_kernel_modules(&mut self) -> Result<()> {
+        if !fs::try_exists(ADDONS_MODULES_PATH).await? {
+            return Ok(());
+        }
+
+        let Some(platform_info) = PlatformInfo::new().ok() else {
+            return Ok(());
+        };
+
+        let kernel_release = platform_info.release().to_string_lossy().to_string();
+        let modules_path = format!("/newroot/lib/modules/{}", kernel_release);
+        fs::create_dir_all(&modules_path).await?;
+        Mount::builder()
+            .fstype(FilesystemType::Manual("none"))
+            .flags(MountFlags::BIND | MountFlags::RDONLY)
+            .mount(ADDONS_MODULES_PATH, modules_path)?;
         Ok(())
     }
 
@@ -170,13 +222,14 @@ impl GuestInit {
         path: &str,
         data: &str,
         flags: Option<MountFlags>,
+        source: Option<&str>,
     ) -> Result<()> {
         trace!("mounting kernel fs {} to {}", fstype, path);
         Mount::builder()
             .fstype(FilesystemType::Manual(fstype))
             .flags(MountFlags::NOEXEC | MountFlags::NOSUID | flags.unwrap_or(MountFlags::empty()))
             .data(data)
-            .mount(fstype, path)?;
+            .mount(source.unwrap_or(fstype), path)?;
         Ok(())
     }
 

crates/runtime/Cargo.toml

@@ -24,6 +24,7 @@ krata-xenclient = { path = "../xen/xenclient", version = "^0.0.10" }
 krata-xenevtchn = { path = "../xen/xenevtchn", version = "^0.0.10" }
 krata-xengnt = { path = "../xen/xengnt", version = "^0.0.10" }
 krata-xenstore = { path = "../xen/xenstore", version = "^0.0.10" }
+walkdir = { workspace = true }
 
 [lib]
 name = "kratart"

crates/runtime/src/cfgblk.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
-use backhand::{FilesystemWriter, NodeHeader};
+use backhand::compression::Compressor;
+use backhand::{FilesystemCompressor, FilesystemWriter, NodeHeader};
 use krata::launchcfg::LaunchInfo;
 use krataoci::packer::OciPackedImage;
 use log::trace;
@@ -8,14 +9,14 @@ use std::fs::File;
 use std::path::PathBuf;
 use uuid::Uuid;
 
-pub struct ConfigBlock<'a> {
+pub struct ConfigBlock {
-    pub image: &'a OciPackedImage,
+    pub image: OciPackedImage,
     pub file: PathBuf,
     pub dir: PathBuf,
 }
 
-impl ConfigBlock<'_> {
+impl ConfigBlock {
-    pub fn new<'a>(uuid: &Uuid, image: &'a OciPackedImage) -> Result<ConfigBlock<'a>> {
+    pub fn new(uuid: &Uuid, image: OciPackedImage) -> Result<ConfigBlock> {
         let mut dir = std::env::temp_dir().clone();
         dir.push(format!("krata-cfg-{}", uuid));
         fs::create_dir_all(&dir)?;
@@ -29,6 +30,7 @@ impl ConfigBlock<'_> {
         let config = self.image.config.raw();
         let launch = serde_json::to_string(launch_config)?;
         let mut writer = FilesystemWriter::default();
+        writer.set_compressor(FilesystemCompressor::new(Compressor::Gzip, None)?);
         writer.push_dir(
             "/image",
             NodeHeader {

crates/runtime/src/launch.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 use std::net::{IpAddr, Ipv6Addr};
+use std::path::PathBuf;
 use std::sync::Arc;
 use std::{fs, net::Ipv4Addr, str::FromStr};
 
@@ -38,6 +39,7 @@ pub struct GuestLaunchRequest {
     pub pcis: Vec<PciDevice>,
     pub debug: bool,
     pub image: OciPackedImage,
+    pub addons_image: Option<PathBuf>,
 }
 
 pub struct GuestLauncher {
@@ -105,8 +107,10 @@ impl GuestLauncher {
             run: request.run,
         };
 
-        let cfgblk = ConfigBlock::new(&uuid, &request.image)?;
+        let cfgblk = ConfigBlock::new(&uuid, request.image.clone())?;
-        cfgblk.build(&launch_config)?;
+        let cfgblk_file = cfgblk.file.clone();
+        let cfgblk_dir = cfgblk.dir.clone();
+        tokio::task::spawn_blocking(move || cfgblk.build(&launch_config)).await??;
 
         let image_squashfs_path = request
             .image
@@ -114,18 +118,33 @@ impl GuestLauncher {
             .to_str()
             .ok_or_else(|| anyhow!("failed to convert image path to string"))?;
 
-        let cfgblk_dir_path = cfgblk
+        let cfgblk_dir_path = cfgblk_dir
-            .dir
             .to_str()
             .ok_or_else(|| anyhow!("failed to convert cfgblk directory path to string"))?;
-        let cfgblk_squashfs_path = cfgblk
+        let cfgblk_squashfs_path = cfgblk_file
-            .file
             .to_str()
             .ok_or_else(|| anyhow!("failed to convert cfgblk squashfs path to string"))?;
+        let addons_squashfs_path = request
+            .addons_image
+            .map(|x| x.to_str().map(|x| x.to_string()))
+            .map(|x| {
+                Some(x.ok_or_else(|| anyhow!("failed to convert addons squashfs path to string")))
+            })
+            .unwrap_or(None);
+
+        let addons_squashfs_path = if let Some(path) = addons_squashfs_path {
+            Some(path?)
+        } else {
+            None
+        };
 
         let image_squashfs_loop = context.autoloop.loopify(image_squashfs_path)?;
         let cfgblk_squashfs_loop = context.autoloop.loopify(cfgblk_squashfs_path)?;
-
+        let addons_squashfs_loop = if let Some(ref addons_squashfs_path) = addons_squashfs_path {
+            Some(context.autoloop.loopify(addons_squashfs_path)?)
+        } else {
+            None
+        };
         let cmdline_options = [
             if request.debug { "debug" } else { "quiet" },
             "elevator=noop",
@@ -135,19 +154,48 @@ impl GuestLauncher {
         let guest_mac_string = container_mac.to_string().replace('-', ":");
         let gateway_mac_string = gateway_mac.to_string().replace('-', ":");
 
+        let mut disks = vec![
+            DomainDisk {
+                vdev: "xvda".to_string(),
+                block: image_squashfs_loop.clone(),
+                writable: false,
+            },
+            DomainDisk {
+                vdev: "xvdb".to_string(),
+                block: cfgblk_squashfs_loop.clone(),
+                writable: false,
+            },
+        ];
+
+        if let Some(ref addons) = addons_squashfs_loop {
+            disks.push(DomainDisk {
+                vdev: "xvdc".to_string(),
+                block: addons.clone(),
+                writable: false,
+            });
+        }
+
+        let mut loops = vec![
+            format!("{}:{}:none", image_squashfs_loop.path, image_squashfs_path),
+            format!(
+                "{}:{}:{}",
+                cfgblk_squashfs_loop.path, cfgblk_squashfs_path, cfgblk_dir_path
+            ),
+        ];
+
+        if let Some(ref addons) = addons_squashfs_loop {
+            loops.push(format!(
+                "{}:{}:none",
+                addons.path,
+                addons_squashfs_path
+                    .clone()
+                    .ok_or_else(|| anyhow!("addons squashfs path missing"))?
+            ));
+        }
+
         let mut extra_keys = vec![
             ("krata/uuid".to_string(), uuid.to_string()),
-            (
+            ("krata/loops".to_string(), loops.join(",")),
-                "krata/loops".to_string(),
-                format!(
-                    "{}:{}:none,{}:{}:{}",
-                    &image_squashfs_loop.path,
-                    image_squashfs_path,
-                    &cfgblk_squashfs_loop.path,
-                    cfgblk_squashfs_path,
-                    cfgblk_dir_path,
-                ),
-            ),
             (
                 "krata/network/guest/ipv4".to_string(),
                 format!("{}/{}", guest_ipv4, ipv4_network_mask),
@@ -187,18 +235,7 @@ impl GuestLauncher {
             initrd: request.initrd,
             cmdline,
             use_console_backend: Some("krata-console".to_string()),
-            disks: vec![
+            disks,
-                DomainDisk {
-                    vdev: "xvda".to_string(),
-                    block: image_squashfs_loop.clone(),
-                    writable: false,
-                },
-                DomainDisk {
-                    vdev: "xvdb".to_string(),
-                    block: cfgblk_squashfs_loop.clone(),
-                    writable: false,
-                },
-            ],
             channels: vec![DomainChannel {
                 typ: "krata-channel".to_string(),
                 initialized: false,
@@ -245,7 +282,7 @@ impl GuestLauncher {
             Err(error) => {
                 let _ = context.autoloop.unloop(&image_squashfs_loop.path).await;
                 let _ = context.autoloop.unloop(&cfgblk_squashfs_loop.path).await;
-                let _ = fs::remove_dir(&cfgblk.dir);
+                let _ = fs::remove_dir(&cfgblk_dir);
                 Err(error.into())
             }
         }

hack/dist/bundle.sh

@@ -33,6 +33,7 @@ cd "${BUNDLE_DIR}"
 
 cp "${KRATA_DIR}/target/initrd/initrd-${TARGET_ARCH}" initrd
 cp "${KRATA_DIR}/target/kernel/kernel-${TARGET_ARCH}" kernel
+cp "${KRATA_DIR}/target/kernel/addons-${TARGET_ARCH}.squashfs" addons.squashfs
 cp "${KRATA_DIR}/resources/systemd/kratad.service" kratad.service
 cp "${KRATA_DIR}/resources/systemd/kratanet.service" kratanet.service
 cp "${KRATA_DIR}/resources/bundle/install.sh" install.sh

hack/dist/systar.sh

@@ -40,6 +40,7 @@ fi
 
 mkdir -p usr/share/krata/guest
 mv ../krata/kernel ../krata/initrd usr/share/krata/guest
+mv ../krata/addons.squashfs usr/share/krata/guest/addons.squashfs
 
 tar czf "${SYSTAR}" --owner 0 --group 0 .
 

hack/kernel/build.sh

@@ -3,7 +3,7 @@ set -e
 
 REAL_SCRIPT="$(realpath "${0}")"
 cd "$(dirname "${REAL_SCRIPT}")/../.."
-KRATA_DIR="${PWD}"
+KRATA_DIR="$(realpath "${PWD}")"
 KERNEL_DIR="${KRATA_DIR}/kernel"
 
 cd "${KRATA_DIR}"
@@ -62,7 +62,24 @@ then
   IMAGE_TARGET="Image.gz"
 fi
 
-make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" -j"${KRATA_KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" "${IMAGE_TARGET}"
+make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" -j"${KRATA_KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" "${IMAGE_TARGET}" modules
+
+MODULES_INSTALL_PATH="${OUTPUT_DIR}/modules-install-${TARGET_ARCH_STANDARD}"
+MODULES_OUTPUT_PATH="${OUTPUT_DIR}/addons-${TARGET_ARCH_STANDARD}/modules"
+ADDONS_SQUASHFS_PATH="${OUTPUT_DIR}/addons-${TARGET_ARCH_STANDARD}.squashfs"
+
+rm -rf "${MODULES_INSTALL_PATH}"
+rm -rf "${MODULES_OUTPUT_PATH}"
+rm -rf "${MODULES_SQUASHFS_PATH}"
+
+make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" -j"${KRATA_KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" INSTALL_MOD_PATH="${MODULES_INSTALL_PATH}" modules_install
+KERNEL_MODULES_VER="$(ls "${MODULES_INSTALL_PATH}/lib/modules")"
+
+mkdir -p "${MODULES_OUTPUT_PATH}"
+mv "${MODULES_INSTALL_PATH}/lib/modules/${KERNEL_MODULES_VER}" "${MODULES_OUTPUT_PATH}"
+rm -rf "${MODULES_INSTALL_PATH}"
+unlink "${MODULES_OUTPUT_PATH}/build"
+mksquashfs "${MODULES_OUTPUT_PATH}" "${ADDONS_SQUASHFS_PATH}"
 
 if [ "${TARGET_ARCH_STANDARD}" = "x86_64" ]
 then