chore(xen): move domain creation to xenplatform

crates/runtime/src/launch.rs

@@ -15,6 +15,7 @@ use krataoci::packer::OciPackedImage;
 use tokio::sync::Semaphore;
 use uuid::Uuid;
 use xenclient::{DomainChannel, DomainConfig, DomainDisk, DomainNetworkInterface};
+use xenplatform::domain::BaseDomainConfig;
 
 use crate::cfgblk::ConfigBlock;
 use crate::RuntimeContext;
@@ -220,13 +221,18 @@ impl GuestLauncher {
         }
 
         let config = DomainConfig {
+            base: BaseDomainConfig {
+                max_vcpus: request.vcpus,
+                mem_mb: request.mem,
+                kernel: request.kernel,
+                initrd: request.initrd,
+                cmdline,
+                uuid,
+                owner_domid: 0,
+                enable_iommu: true,
+            },
             backend_domid: 0,
             name: xen_name,
-            max_vcpus: request.vcpus,
-            mem_mb: request.mem,
-            kernel: request.kernel,
-            initrd: request.initrd,
-            cmdline,
             swap_console_backend: Some("krata-console".to_string()),
             disks,
             channels: vec![DomainChannel {

crates/xen/xencall/src/sys.rs

@@ -1,7 +1,6 @@
 /// Handwritten hypercall bindings.
 use nix::ioctl_readwrite_bad;
 use std::ffi::{c_char, c_int, c_uint, c_ulong};
-use uuid::Uuid;
 
 #[repr(C)]
 #[derive(Copy, Clone, Debug)]
@@ -269,7 +268,7 @@ impl Default for CreateDomain {
     fn default() -> Self {
         CreateDomain {
             ssidref: SECINITSID_DOMU,
-            handle: Uuid::new_v4().into_bytes(),
+            handle: [0; 16],
             flags: 0,
             iommu_opts: 0,
             max_vcpus: 1,

crates/xen/xenclient/examples/boot.rs

@@ -1,7 +1,9 @@
 use std::{env, process};
 use tokio::fs;
+use uuid::Uuid;
 use xenclient::error::Result;
 use xenclient::{DomainConfig, XenClient};
+use xenplatform::domain::BaseDomainConfig;
 
 #[cfg(target_arch = "x86_64")]
 type RuntimePlatform = xenplatform::x86pv::X86PvPlatform;
@@ -22,13 +24,18 @@ async fn main() -> Result<()> {
     let initrd_path = args.get(2).expect("argument not specified");
     let client = XenClient::new(0, RuntimePlatform::new()).await?;
     let config = DomainConfig {
+        base: BaseDomainConfig {
+            uuid: Uuid::new_v4(),
+            max_vcpus: 1,
+            mem_mb: 512,
+            enable_iommu: true,
+            kernel: fs::read(&kernel_image_path).await?,
+            initrd: fs::read(&initrd_path).await?,
+            cmdline: "earlyprintk=xen earlycon=xen console=hvc0 init=/init".to_string(),
+            owner_domid: 0,
+        },
         backend_domid: 0,
         name: "xenclient-test".to_string(),
-        max_vcpus: 1,
-        mem_mb: 512,
-        kernel: fs::read(&kernel_image_path).await?,
-        initrd: fs::read(&initrd_path).await?,
-        cmdline: "earlyprintk=xen earlycon=xen console=hvc0 init=/init".to_string(),
         swap_console_backend: None,
         disks: vec![],
         channels: vec![],

crates/xen/xenclient/src/lib.rs

@@ -2,19 +2,18 @@ pub mod error;
 
 use crate::error::{Error, Result};
 use indexmap::IndexMap;
-use log::{debug, trace, warn};
+use log::{debug, trace};
 use pci::{PciBdf, XenPciBackend};
 use tokio::time::timeout;
-use xenplatform::boot::{BootDomain, BootSetup, BootSetupPlatform};
-use xenplatform::elfloader::ElfImageLoader;
+use xenplatform::boot::BootSetupPlatform;
+use xenplatform::domain::{BaseDomainConfig, BaseDomainManager, CreatedDomain};
 use xenplatform::sys::XEN_PAGE_SHIFT;
 
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
-use uuid::Uuid;
-use xencall::sys::{CreateDomain, DOMCTL_DEV_RDM_RELAXED};
+use xencall::sys::DOMCTL_DEV_RDM_RELAXED;
 use xencall::XenCall;
 use xenstore::{
     XsPermission, XsdClient, XsdInterface, XsdTransaction, XS_PERM_NONE, XS_PERM_READ,
@@ -27,7 +26,7 @@ pub mod pci;
 pub struct XenClient<P: BootSetupPlatform> {
     pub store: XsdClient,
     call: XenCall,
-    platform: Arc<P>,
+    domain_manager: Arc<BaseDomainManager<P>>,
 }
 
 #[derive(Clone, Debug)]
@@ -98,13 +97,9 @@ pub struct DomainPciDevice {
 
 #[derive(Clone, Debug)]
 pub struct DomainConfig {
+    pub base: BaseDomainConfig,
     pub backend_domid: u32,
     pub name: String,
-    pub max_vcpus: u32,
-    pub mem_mb: u64,
-    pub kernel: Vec<u8>,
-    pub initrd: Vec<u8>,
-    pub cmdline: String,
     pub disks: Vec<DomainDisk>,
     pub swap_console_backend: Option<String>,
     pub channels: Vec<DomainChannel>,
@@ -122,49 +117,37 @@ pub struct CreatedChannel {
     pub evtchn: u32,
 }
 
-#[derive(Debug)]
-pub struct CreatedDomain {
-    pub domid: u32,
-    pub channels: Vec<CreatedChannel>,
-}
-
 #[allow(clippy::too_many_arguments)]
 impl<P: BootSetupPlatform> XenClient<P> {
     pub async fn new(current_domid: u32, platform: P) -> Result<XenClient<P>> {
         let store = XsdClient::open().await?;
-        let call = XenCall::open(current_domid)?;
+        let call: XenCall = XenCall::open(current_domid)?;
+        let domain_manager = BaseDomainManager::new(call.clone(), platform).await?;
         Ok(XenClient {
             store,
             call,
-            platform: Arc::new(platform),
+            domain_manager: Arc::new(domain_manager),
         })
     }
 
     pub async fn create(&self, config: &DomainConfig) -> Result<CreatedDomain> {
-        let mut domain = self.platform.create_domain(!config.pcis.is_empty());
-        domain.max_vcpus = config.max_vcpus;
-        let domid = self.call.create_domain(domain).await?;
-        match self.init(domid, &domain, config).await {
-            Ok(created) => Ok(created),
+        let created = self.domain_manager.create(config.base.clone()).await?;
+        match self.init(created.domid, &created, config).await {
+            Ok(_) => Ok(created),
             Err(err) => {
                 // ignore since destroying a domain is best
                 // effort when an error occurs
-                let _ = self.destroy(domid).await;
+                let _ = self.domain_manager.destroy(created.domid).await;
                 Err(err)
             }
         }
     }
 
-    async fn init(
-        &self,
-        domid: u32,
-        created: &CreateDomain,
-        config: &DomainConfig,
-    ) -> Result<CreatedDomain> {
-        trace!("XenClient init domid={} domain={:?}", domid, created,);
+    async fn init(&self, domid: u32, created: &CreatedDomain, config: &DomainConfig) -> Result<()> {
+        trace!("XenClient init domid={} domain={:?}", domid, created);
         let backend_dom_path = self.store.get_domain_path(0).await?;
         let dom_path = self.store.get_domain_path(domid).await?;
-        let uuid_string = Uuid::from_bytes(created.handle).to_string();
+        let uuid_string = config.base.uuid.to_string();
         let vm_path = format!("/vm/{}", uuid_string);
 
         let ro_perm = &[
@@ -243,11 +226,8 @@ impl<P: BootSetupPlatform> XenClient<P> {
             tx.mknod(format!("{}/error", dom_path).as_str(), rw_perm)
                 .await?;
 
-            tx.write_string(
-                format!("{}/uuid", vm_path).as_str(),
-                &Uuid::from_bytes(created.handle).to_string(),
-            )
-            .await?;
+            tx.write_string(format!("{}/uuid", vm_path).as_str(), &uuid_string)
+                .await?;
             tx.write_string(format!("{}/name", dom_path).as_str(), &config.name)
                 .await?;
             tx.write_string(format!("{}/name", vm_path).as_str(), &config.name)
@@ -266,44 +246,24 @@ impl<P: BootSetupPlatform> XenClient<P> {
             tx.commit().await?;
         }
 
-        self.call.set_max_vcpus(domid, config.max_vcpus).await?;
-        self.call
-            .set_max_mem(domid, (config.mem_mb * 1024) + 2048)
-            .await?;
-        let mut domain: BootDomain;
-        {
-            let loader = ElfImageLoader::load_file_kernel(&config.kernel)?;
-            let platform = (*self.platform).clone();
-            let mut boot = BootSetup::new(self.call.clone(), domid, platform, loader, None);
-            domain = boot
-                .initialize(
-                    &config.initrd,
-                    config.mem_mb,
-                    config.max_vcpus,
-                    &config.cmdline,
-                )
-                .await?;
-            boot.boot(&mut domain).await?;
-        }
-
         {
             let tx = self.store.transaction().await?;
             tx.write_string(format!("{}/image/os_type", vm_path).as_str(), "linux")
                 .await?;
             tx.write_string(
                 format!("{}/image/cmdline", vm_path).as_str(),
-                &config.cmdline,
+                &config.base.cmdline,
             )
             .await?;
 
             tx.write_string(
                 format!("{}/memory/static-max", dom_path).as_str(),
-                &(config.mem_mb * 1024).to_string(),
+                &(config.base.mem_mb * 1024).to_string(),
             )
             .await?;
             tx.write_string(
                 format!("{}/memory/target", dom_path).as_str(),
-                &(config.mem_mb * 1024).to_string(),
+                &(config.base.mem_mb * 1024).to_string(),
             )
             .await?;
             tx.write_string(format!("{}/memory/videoram", dom_path).as_str(), "0")
@@ -314,15 +274,15 @@ impl<P: BootSetupPlatform> XenClient<P> {
                 .await?;
             tx.write_string(
                 format!("{}/store/port", dom_path).as_str(),
-                &domain.store_evtchn.to_string(),
+                &created.store_evtchn.to_string(),
             )
             .await?;
             tx.write_string(
                 format!("{}/store/ring-ref", dom_path).as_str(),
-                &domain.store_mfn.to_string(),
+                &created.store_mfn.to_string(),
             )
             .await?;
-            for i in 0..config.max_vcpus {
+            for i in 0..config.base.max_vcpus {
                 let path = format!("{}/cpu/{}", dom_path, i);
                 tx.mkdir(&path).await?;
                 tx.set_perms(&path, ro_perm).await?;
@@ -334,7 +294,7 @@ impl<P: BootSetupPlatform> XenClient<P> {
         }
         if !self
             .store
-            .introduce_domain(domid, domain.store_mfn, domain.store_evtchn)
+            .introduce_domain(domid, created.store_mfn, created.store_evtchn)
             .await?
         {
             return Err(Error::IntroduceDomainFailed);
@@ -343,7 +303,7 @@ impl<P: BootSetupPlatform> XenClient<P> {
         let tx = self.store.transaction().await?;
         self.console_device_add(
             &tx,
-            &mut domain,
+            created,
             &DomainChannel {
                 typ: config
                     .swap_console_backend
@@ -360,24 +320,18 @@ impl<P: BootSetupPlatform> XenClient<P> {
         )
         .await?;
 
-        let mut channels: Vec<CreatedChannel> = Vec::new();
         for (index, channel) in config.channels.iter().enumerate() {
-            let (Some(ring_ref), Some(evtchn)) = self
-                .console_device_add(
-                    &tx,
-                    &mut domain,
-                    channel,
-                    &dom_path,
-                    &backend_dom_path,
-                    config.backend_domid,
-                    domid,
-                    index + 1,
-                )
-                .await?
-            else {
-                continue;
-            };
-            channels.push(CreatedChannel { ring_ref, evtchn });
+            self.console_device_add(
+                &tx,
+                created,
+                channel,
+                &dom_path,
+                &backend_dom_path,
+                config.backend_domid,
+                domid,
+                index + 1,
+            )
+            .await?;
         }
 
         for (index, disk) in config.disks.iter().enumerate() {
@@ -448,7 +402,7 @@ impl<P: BootSetupPlatform> XenClient<P> {
         tx.commit().await?;
 
         self.call.unpause_domain(domid).await?;
-        Ok(CreatedDomain { domid, channels })
+        Ok(())
     }
 
     async fn disk_device_add(
@@ -509,17 +463,16 @@ impl<P: BootSetupPlatform> XenClient<P> {
     async fn console_device_add(
         &self,
         tx: &XsdTransaction,
-        domain: &mut BootDomain,
+        domain: &CreatedDomain,
         channel: &DomainChannel,
         dom_path: &str,
         backend_dom_path: &str,
         backend_domid: u32,
         domid: u32,
         index: usize,
-    ) -> Result<(Option<u64>, Option<u32>)> {
-        let console = domain.consoles.get(index);
-        let port = console.map(|x| x.0);
-        let ring = console.map(|x| x.1);
+    ) -> Result<Option<(u32, u64)>> {
+        let port = domain.console_evtchn;
+        let ring = domain.console_mfn;
 
         let mut backend_entries = vec![
             ("frontend-id", domid.to_string()),
@@ -537,15 +490,13 @@ impl<P: BootSetupPlatform> XenClient<P> {
         frontend_entries.push(("type", channel.typ.clone()));
         backend_entries.push(("type", channel.typ.clone()));
 
-        if port.is_some() && ring.is_some() {
+        if index == 0 {
             if channel.typ != "xenconsoled" {
                 frontend_entries.push(("state", "1".to_string()));
             }
 
-            frontend_entries.extend_from_slice(&[
-                ("port", port.unwrap().to_string()),
-                ("ring-ref", ring.unwrap().to_string()),
-            ]);
+            frontend_entries
+                .extend_from_slice(&[("port", port.to_string()), ("ring-ref", ring.to_string())]);
         } else {
             frontend_entries.extend_from_slice(&[
                 ("state", "1".to_string()),
@@ -571,7 +522,7 @@ impl<P: BootSetupPlatform> XenClient<P> {
             backend_entries,
         )
         .await?;
-        Ok((ring, port))
+        Ok(Some((port, ring)))
     }
 
     async fn fs_9p_device_add(
@@ -863,10 +814,8 @@ impl<P: BootSetupPlatform> XenClient<P> {
     }
 
     pub async fn destroy(&self, domid: u32) -> Result<()> {
-        if let Err(err) = self.destroy_store(domid).await {
-            warn!("failed to destroy store for domain {}: {}", domid, err);
-        }
-        self.call.destroy_domain(domid).await?;
+        let _ = self.destroy_store(domid).await;
+        self.domain_manager.destroy(domid).await?;
         Ok(())
     }
 

crates/xen/xenplatform/src/boot.rs

@@ -43,7 +43,8 @@ pub struct BootDomain {
     pub store_evtchn: u32,
     pub store_mfn: u64,
     pub initrd_segment: DomainSegment,
-    pub consoles: Vec<(u32, u64)>,
+    pub console_evtchn: u32,
+    pub console_mfn: u64,
     pub cmdline: String,
 }
 
@@ -177,7 +178,8 @@ impl<I: BootImageLoader, P: BootSetupPlatform> BootSetup<I, P> {
             target_pages: total_pages,
             page_size: self.platform.page_size(),
             image_info,
-            consoles: Vec::new(),
+            console_evtchn: 0,
+            console_mfn: 0,
             max_vcpus,
             phys: PhysicalPages::new(self.call.clone(), self.domid, self.platform.page_shift()),
             initrd_segment: DomainSegment::default(),
@@ -261,7 +263,7 @@ impl<I: BootImageLoader, P: BootSetupPlatform> BootSetup<I, P> {
 
 #[async_trait::async_trait]
 pub trait BootSetupPlatform: Clone {
-    fn create_domain(&self, needs_passthrough: bool) -> CreateDomain;
+    fn create_domain(&self, enable_iommu: bool) -> CreateDomain;
     fn page_size(&self) -> u64;
     fn page_shift(&self) -> u64;
     fn needs_early_kernel(&self) -> bool;

crates/xen/xenplatform/src/domain.rs

@@ -0,0 +1,80 @@
+use std::sync::Arc;
+
+use crate::{
+    boot::{BootSetup, BootSetupPlatform},
+    elfloader::ElfImageLoader,
+};
+use uuid::Uuid;
+use xencall::XenCall;
+
+use crate::error::Result;
+
+pub struct BaseDomainManager<P: BootSetupPlatform> {
+    call: XenCall,
+    pub platform: Arc<P>,
+}
+
+impl<P: BootSetupPlatform> BaseDomainManager<P> {
+    pub async fn new(call: XenCall, platform: P) -> Result<BaseDomainManager<P>> {
+        Ok(BaseDomainManager {
+            call,
+            platform: Arc::new(platform),
+        })
+    }
+
+    pub async fn create(&self, config: BaseDomainConfig) -> Result<CreatedDomain> {
+        let mut domain = self.platform.create_domain(config.enable_iommu);
+        domain.handle = config.uuid.into_bytes();
+        domain.max_vcpus = config.max_vcpus;
+        let domid = self.call.create_domain(domain).await?;
+        self.call.set_max_vcpus(domid, config.max_vcpus).await?;
+        self.call
+            .set_max_mem(domid, (config.mem_mb * 1024) + 2048)
+            .await?;
+        let loader = ElfImageLoader::load_file_kernel(&config.kernel)?;
+        let platform = (*self.platform).clone();
+        let mut boot = BootSetup::new(self.call.clone(), domid, platform, loader, None);
+        let mut domain = boot
+            .initialize(
+                &config.initrd,
+                config.mem_mb,
+                config.max_vcpus,
+                &config.cmdline,
+            )
+            .await?;
+        boot.boot(&mut domain).await?;
+        Ok(CreatedDomain {
+            domid,
+            store_evtchn: domain.store_evtchn,
+            store_mfn: domain.store_mfn,
+            console_evtchn: domain.console_evtchn,
+            console_mfn: domain.console_mfn,
+        })
+    }
+
+    pub async fn destroy(&self, domid: u32) -> Result<()> {
+        self.call.destroy_domain(domid).await?;
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct BaseDomainConfig {
+    pub uuid: Uuid,
+    pub owner_domid: u32,
+    pub max_vcpus: u32,
+    pub mem_mb: u64,
+    pub kernel: Vec<u8>,
+    pub initrd: Vec<u8>,
+    pub cmdline: String,
+    pub enable_iommu: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct CreatedDomain {
+    pub domid: u32,
+    pub store_evtchn: u32,
+    pub store_mfn: u64,
+    pub console_evtchn: u32,
+    pub console_mfn: u64,
+}

crates/xen/xenplatform/src/lib.rs

@@ -6,6 +6,7 @@ pub mod sys;
 
 use crate::error::Error;
 
+pub mod domain;
 pub mod unsupported;
 #[cfg(target_arch = "x86_64")]
 pub mod x86pv;

crates/xen/xenplatform/src/x86pv.rs

@@ -434,9 +434,9 @@ impl X86PvPlatform {
 
 #[async_trait::async_trait]
 impl BootSetupPlatform for X86PvPlatform {
-    fn create_domain(&self, needs_passthrough: bool) -> CreateDomain {
+    fn create_domain(&self, enable_iommu: bool) -> CreateDomain {
         CreateDomain {
-            flags: if needs_passthrough {
+            flags: if enable_iommu {
                 XEN_DOMCTL_CDF_IOMMU
             } else {
                 0
@@ -718,9 +718,8 @@ impl BootSetupPlatform for X86PvPlatform {
         domain.store_mfn = domain.phys.p2m[self.xenstore_segment.as_ref().unwrap().pfn as usize];
         let evtchn = domain.call.evtchn_alloc_unbound(domain.domid, 0).await?;
         let page = domain.alloc_page()?;
-        domain
-            .consoles
-            .push((evtchn, domain.phys.p2m[page.pfn as usize]));
+        domain.console_evtchn = evtchn;
+        domain.console_mfn = domain.phys.p2m[page.pfn as usize];
         self.page_table_segment = self.alloc_page_tables(domain).await?;
         self.boot_stack_segment = Some(domain.alloc_page()?);
 
@@ -802,9 +801,8 @@ impl BootSetupPlatform for X86PvPlatform {
             (*info).flags = 0;
             (*info).store_evtchn = domain.store_evtchn;
             (*info).store_mfn = domain.phys.p2m[xenstore_segment.pfn as usize];
-            let console = domain.consoles.first().unwrap();
-            (*info).console.mfn = console.1;
-            (*info).console.evtchn = console.0;
+            (*info).console.mfn = domain.console_mfn;
+            (*info).console.evtchn = domain.console_evtchn;
             (*info).mod_start = domain.initrd_segment.vstart;
             (*info).mod_len = domain.initrd_segment.size;
             for (i, c) in domain.cmdline.chars().enumerate() {
@@ -893,7 +891,7 @@ impl BootSetupPlatform for X86PvPlatform {
             .as_ref()
             .ok_or(Error::MemorySetupFailed("xenstore_segment missing"))?;
 
-        let console_gfn = domain.consoles.first().map(|x| x.1).unwrap_or(0) as usize;
+        let console_gfn = domain.console_mfn as usize;
         let xenstore_gfn = domain.phys.p2m[xenstore_segment.pfn as usize];
         let addr = domain
             .call