feat: pci passthrough (#114)

* feat: pci passthrough

* feat: guest device management

* feat: addons mounting and kernel modules support

* feat: more pci work

* fix: kernel build squashfs fixes

* fix: e820entry should be available on all platforms

crates/daemon/src/reconcile/guest/mod.rs

@@ -26,6 +26,7 @@ use uuid::Uuid;
 
 use crate::{
     db::GuestStore,
+    devices::DaemonDeviceManager,
     event::{DaemonEvent, DaemonEventContext},
     glt::GuestLookupTable,
 };
@@ -55,6 +56,7 @@ impl Drop for GuestReconcilerEntry {
 
 #[derive(Clone)]
 pub struct GuestReconciler {
+    devices: DaemonDeviceManager,
     glt: GuestLookupTable,
     guests: GuestStore,
     events: DaemonEventContext,
@@ -62,6 +64,7 @@ pub struct GuestReconciler {
     packer: OciPackerService,
     kernel_path: PathBuf,
     initrd_path: PathBuf,
+    addons_path: PathBuf,
     tasks: Arc<Mutex<HashMap<Uuid, GuestReconcilerEntry>>>,
     guest_reconciler_notify: Sender<Uuid>,
     reconcile_lock: Arc<RwLock<()>>,
@@ -70,6 +73,7 @@ pub struct GuestReconciler {
 impl GuestReconciler {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
+        devices: DaemonDeviceManager,
         glt: GuestLookupTable,
         guests: GuestStore,
         events: DaemonEventContext,
@@ -78,8 +82,10 @@ impl GuestReconciler {
         guest_reconciler_notify: Sender<Uuid>,
         kernel_path: PathBuf,
         initrd_path: PathBuf,
+        modules_path: PathBuf,
     ) -> Result<Self> {
         Ok(Self {
+            devices,
             glt,
             guests,
             events,
@@ -87,6 +93,7 @@ impl GuestReconciler {
             packer,
             kernel_path,
             initrd_path,
+            addons_path: modules_path,
             tasks: Arc::new(Mutex::new(HashMap::new())),
             guest_reconciler_notify,
             reconcile_lock: Arc::new(RwLock::with_max_readers((), PARALLEL_LIMIT)),
@@ -152,6 +159,8 @@ impl GuestReconciler {
             self.guests.remove(guest.uuid).await?;
         }
 
+        let mut device_claims = HashMap::new();
+
         for (uuid, mut stored_guest) in stored_guests {
             let previous_guest = stored_guest.clone();
             let runtime_guest = runtime_guests.iter().find(|x| x.uuid == uuid);
@@ -173,6 +182,17 @@ impl GuestReconciler {
                     } else {
                         state.status = GuestStatus::Started.into();
                     }
+
+                    for device in &stored_guest
+                        .spec
+                        .as_ref()
+                        .cloned()
+                        .unwrap_or_default()
+                        .devices
+                    {
+                        device_claims.insert(device.name.clone(), uuid);
+                    }
+
                     state.network = Some(guestinfo_to_networkstate(runtime));
                     stored_guest.state = Some(state);
                 }
@@ -185,6 +205,9 @@ impl GuestReconciler {
                 let _ = self.guest_reconciler_notify.try_send(uuid);
             }
         }
+
+        self.devices.update_claims(device_claims).await?;
+
         Ok(())
     }
 
@@ -255,8 +278,10 @@ impl GuestReconciler {
 
     async fn start(&self, uuid: Uuid, guest: &mut Guest) -> Result<GuestReconcilerResult> {
         let starter = GuestStarter {
+            devices: &self.devices,
             kernel_path: &self.kernel_path,
             initrd_path: &self.initrd_path,
+            addons_path: &self.addons_path,
             packer: &self.packer,
             glt: &self.glt,
             runtime: &self.runtime,
@@ -293,6 +318,7 @@ impl GuestReconciler {
             host: self.glt.host_uuid().to_string(),
             domid: domid.unwrap_or(u32::MAX),
         });
+        self.devices.release_all(uuid).await?;
         Ok(GuestReconcilerResult::Changed { rerun: false })
     }
 

crates/daemon/src/reconcile/guest/start.rs

@@ -1,5 +1,7 @@
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
+use std::str::FromStr;
+use std::sync::atomic::{AtomicBool, Ordering};
 
 use anyhow::{anyhow, Result};
 use futures::StreamExt;
@@ -7,6 +9,7 @@ use krata::launchcfg::LaunchPackedFormat;
 use krata::v1::common::GuestOciImageSpec;
 use krata::v1::common::{guest_image_spec::Image, Guest, GuestState, GuestStatus, OciImageFormat};
 use krataoci::packer::{service::OciPackerService, OciPackedFormat};
+use kratart::launch::{PciBdf, PciDevice, PciRdmReservePolicy};
 use kratart::{launch::GuestLaunchRequest, Runtime};
 use log::info;
 
@@ -15,6 +18,8 @@ use tokio::io::AsyncReadExt;
 use tokio_tar::Archive;
 use uuid::Uuid;
 
+use crate::config::DaemonPciDeviceRdmReservePolicy;
+use crate::devices::DaemonDeviceManager;
 use crate::{
     glt::GuestLookupTable,
     reconcile::guest::{guestinfo_to_networkstate, GuestReconcilerResult},
@@ -24,8 +29,10 @@ use crate::{
 const OCI_SPEC_TAR_FILE_MAX_SIZE: usize = 100 * 1024 * 1024;
 
 pub struct GuestStarter<'a> {
+    pub devices: &'a DaemonDeviceManager,
     pub kernel_path: &'a Path,
     pub initrd_path: &'a Path,
+    pub addons_path: &'a Path,
     pub packer: &'a OciPackerService,
     pub glt: &'a GuestLookupTable,
     pub runtime: &'a Runtime,
@@ -135,6 +142,48 @@ impl GuestStarter<'_> {
             fs::read(&self.initrd_path).await?
         };
 
+        let success = AtomicBool::new(false);
+
+        let _device_release_guard = scopeguard::guard(
+            (spec.devices.clone(), self.devices.clone()),
+            |(devices, manager)| {
+                if !success.load(Ordering::Acquire) {
+                    tokio::task::spawn(async move {
+                        for device in devices {
+                            let _ = manager.release(&device.name, uuid).await;
+                        }
+                    });
+                }
+            },
+        );
+
+        let mut pcis = Vec::new();
+        for device in &spec.devices {
+            let state = self.devices.claim(&device.name, uuid).await?;
+            if let Some(cfg) = state.pci {
+                for location in cfg.locations {
+                    let pci = PciDevice {
+                        bdf: PciBdf::from_str(&location)?.with_domain(0),
+                        permissive: cfg.permissive,
+                        msi_translate: cfg.msi_translate,
+                        power_management: cfg.power_management,
+                        rdm_reserve_policy: match cfg.rdm_reserve_policy {
+                            DaemonPciDeviceRdmReservePolicy::Strict => PciRdmReservePolicy::Strict,
+                            DaemonPciDeviceRdmReservePolicy::Relaxed => {
+                                PciRdmReservePolicy::Relaxed
+                            }
+                        },
+                    };
+                    pcis.push(pci);
+                }
+            } else {
+                return Err(anyhow!(
+                    "device '{}' isn't a known device type",
+                    device.name
+                ));
+            }
+        }
+
         let info = self
             .runtime
             .launch(GuestLaunchRequest {
@@ -150,6 +199,7 @@ impl GuestStarter<'_> {
                 initrd,
                 vcpus: spec.vcpus,
                 mem: spec.mem,
+                pcis,
                 env: task
                     .environment
                     .iter()
@@ -157,6 +207,7 @@ impl GuestStarter<'_> {
                     .collect::<HashMap<_, _>>(),
                 run: empty_vec_optional(task.command.clone()),
                 debug: false,
+                addons_image: Some(self.addons_path.to_path_buf()),
             })
             .await?;
         self.glt.associate(uuid, info.domid).await;
@@ -169,6 +220,7 @@ impl GuestStarter<'_> {
             host: self.glt.host_uuid().to_string(),
             domid: info.domid,
         });
+        success.store(true, Ordering::Release);
         Ok(GuestReconcilerResult::Changed { rerun: false })
     }
 }