diff --git a/crates/runtime/src/launch.rs b/crates/runtime/src/launch.rs index c26e352..6a3a0fa 100644 --- a/crates/runtime/src/launch.rs +++ b/crates/runtime/src/launch.rs @@ -15,6 +15,7 @@ use krataoci::packer::OciPackedImage; use tokio::sync::Semaphore; use uuid::Uuid; use xenclient::{DomainChannel, DomainConfig, DomainDisk, DomainNetworkInterface}; +use xenplatform::domain::BaseDomainConfig; use crate::cfgblk::ConfigBlock; use crate::RuntimeContext; @@ -220,13 +221,18 @@ impl GuestLauncher { } let config = DomainConfig { + base: BaseDomainConfig { + max_vcpus: request.vcpus, + mem_mb: request.mem, + kernel: request.kernel, + initrd: request.initrd, + cmdline, + uuid, + owner_domid: 0, + enable_iommu: true, + }, backend_domid: 0, name: xen_name, - max_vcpus: request.vcpus, - mem_mb: request.mem, - kernel: request.kernel, - initrd: request.initrd, - cmdline, swap_console_backend: Some("krata-console".to_string()), disks, channels: vec![DomainChannel { diff --git a/crates/xen/xencall/src/sys.rs b/crates/xen/xencall/src/sys.rs index fe0628b..9df2b5d 100644 --- a/crates/xen/xencall/src/sys.rs +++ b/crates/xen/xencall/src/sys.rs @@ -1,7 +1,6 @@ /// Handwritten hypercall bindings. use nix::ioctl_readwrite_bad; use std::ffi::{c_char, c_int, c_uint, c_ulong}; -use uuid::Uuid; #[repr(C)] #[derive(Copy, Clone, Debug)] @@ -269,7 +268,7 @@ impl Default for CreateDomain { fn default() -> Self { CreateDomain { ssidref: SECINITSID_DOMU, - handle: Uuid::new_v4().into_bytes(), + handle: [0; 16], flags: 0, iommu_opts: 0, max_vcpus: 1, diff --git a/crates/xen/xenclient/examples/boot.rs b/crates/xen/xenclient/examples/boot.rs index 3045177..a6ecfbd 100644 --- a/crates/xen/xenclient/examples/boot.rs +++ b/crates/xen/xenclient/examples/boot.rs @@ -1,7 +1,9 @@ use std::{env, process}; use tokio::fs; +use uuid::Uuid; use xenclient::error::Result; use xenclient::{DomainConfig, XenClient}; +use xenplatform::domain::BaseDomainConfig; #[cfg(target_arch = "x86_64")] type RuntimePlatform = xenplatform::x86pv::X86PvPlatform; @@ -22,13 +24,18 @@ async fn main() -> Result<()> { let initrd_path = args.get(2).expect("argument not specified"); let client = XenClient::new(0, RuntimePlatform::new()).await?; let config = DomainConfig { + base: BaseDomainConfig { + uuid: Uuid::new_v4(), + max_vcpus: 1, + mem_mb: 512, + enable_iommu: true, + kernel: fs::read(&kernel_image_path).await?, + initrd: fs::read(&initrd_path).await?, + cmdline: "earlyprintk=xen earlycon=xen console=hvc0 init=/init".to_string(), + owner_domid: 0, + }, backend_domid: 0, name: "xenclient-test".to_string(), - max_vcpus: 1, - mem_mb: 512, - kernel: fs::read(&kernel_image_path).await?, - initrd: fs::read(&initrd_path).await?, - cmdline: "earlyprintk=xen earlycon=xen console=hvc0 init=/init".to_string(), swap_console_backend: None, disks: vec![], channels: vec![], diff --git a/crates/xen/xenclient/src/lib.rs b/crates/xen/xenclient/src/lib.rs index 1021d56..0993bbe 100644 --- a/crates/xen/xenclient/src/lib.rs +++ b/crates/xen/xenclient/src/lib.rs @@ -2,19 +2,18 @@ pub mod error; use crate::error::{Error, Result}; use indexmap::IndexMap; -use log::{debug, trace, warn}; +use log::{debug, trace}; use pci::{PciBdf, XenPciBackend}; use tokio::time::timeout; -use xenplatform::boot::{BootDomain, BootSetup, BootSetupPlatform}; -use xenplatform::elfloader::ElfImageLoader; +use xenplatform::boot::BootSetupPlatform; +use xenplatform::domain::{BaseDomainConfig, BaseDomainManager, CreatedDomain}; use xenplatform::sys::XEN_PAGE_SHIFT; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use uuid::Uuid; -use xencall::sys::{CreateDomain, DOMCTL_DEV_RDM_RELAXED}; +use xencall::sys::DOMCTL_DEV_RDM_RELAXED; use xencall::XenCall; use xenstore::{ XsPermission, XsdClient, XsdInterface, XsdTransaction, XS_PERM_NONE, XS_PERM_READ, @@ -27,7 +26,7 @@ pub mod pci; pub struct XenClient { pub store: XsdClient, call: XenCall, - platform: Arc

, + domain_manager: Arc>, } #[derive(Clone, Debug)] @@ -98,13 +97,9 @@ pub struct DomainPciDevice { #[derive(Clone, Debug)] pub struct DomainConfig { + pub base: BaseDomainConfig, pub backend_domid: u32, pub name: String, - pub max_vcpus: u32, - pub mem_mb: u64, - pub kernel: Vec, - pub initrd: Vec, - pub cmdline: String, pub disks: Vec, pub swap_console_backend: Option, pub channels: Vec, @@ -122,49 +117,37 @@ pub struct CreatedChannel { pub evtchn: u32, } -#[derive(Debug)] -pub struct CreatedDomain { - pub domid: u32, - pub channels: Vec, -} - #[allow(clippy::too_many_arguments)] impl XenClient

{ pub async fn new(current_domid: u32, platform: P) -> Result> { let store = XsdClient::open().await?; - let call = XenCall::open(current_domid)?; + let call: XenCall = XenCall::open(current_domid)?; + let domain_manager = BaseDomainManager::new(call.clone(), platform).await?; Ok(XenClient { store, call, - platform: Arc::new(platform), + domain_manager: Arc::new(domain_manager), }) } pub async fn create(&self, config: &DomainConfig) -> Result { - let mut domain = self.platform.create_domain(!config.pcis.is_empty()); - domain.max_vcpus = config.max_vcpus; - let domid = self.call.create_domain(domain).await?; - match self.init(domid, &domain, config).await { - Ok(created) => Ok(created), + let created = self.domain_manager.create(config.base.clone()).await?; + match self.init(created.domid, &created, config).await { + Ok(_) => Ok(created), Err(err) => { // ignore since destroying a domain is best // effort when an error occurs - let _ = self.destroy(domid).await; + let _ = self.domain_manager.destroy(created.domid).await; Err(err) } } } - async fn init( - &self, - domid: u32, - created: &CreateDomain, - config: &DomainConfig, - ) -> Result { - trace!("XenClient init domid={} domain={:?}", domid, created,); + async fn init(&self, domid: u32, created: &CreatedDomain, config: &DomainConfig) -> Result<()> { + trace!("XenClient init domid={} domain={:?}", domid, created); let backend_dom_path = self.store.get_domain_path(0).await?; let dom_path = self.store.get_domain_path(domid).await?; - let uuid_string = Uuid::from_bytes(created.handle).to_string(); + let uuid_string = config.base.uuid.to_string(); let vm_path = format!("/vm/{}", uuid_string); let ro_perm = &[ @@ -243,11 +226,8 @@ impl XenClient

{ tx.mknod(format!("{}/error", dom_path).as_str(), rw_perm) .await?; - tx.write_string( - format!("{}/uuid", vm_path).as_str(), - &Uuid::from_bytes(created.handle).to_string(), - ) - .await?; + tx.write_string(format!("{}/uuid", vm_path).as_str(), &uuid_string) + .await?; tx.write_string(format!("{}/name", dom_path).as_str(), &config.name) .await?; tx.write_string(format!("{}/name", vm_path).as_str(), &config.name) @@ -266,44 +246,24 @@ impl XenClient

{ tx.commit().await?; } - self.call.set_max_vcpus(domid, config.max_vcpus).await?; - self.call - .set_max_mem(domid, (config.mem_mb * 1024) + 2048) - .await?; - let mut domain: BootDomain; - { - let loader = ElfImageLoader::load_file_kernel(&config.kernel)?; - let platform = (*self.platform).clone(); - let mut boot = BootSetup::new(self.call.clone(), domid, platform, loader, None); - domain = boot - .initialize( - &config.initrd, - config.mem_mb, - config.max_vcpus, - &config.cmdline, - ) - .await?; - boot.boot(&mut domain).await?; - } - { let tx = self.store.transaction().await?; tx.write_string(format!("{}/image/os_type", vm_path).as_str(), "linux") .await?; tx.write_string( format!("{}/image/cmdline", vm_path).as_str(), - &config.cmdline, + &config.base.cmdline, ) .await?; tx.write_string( format!("{}/memory/static-max", dom_path).as_str(), - &(config.mem_mb * 1024).to_string(), + &(config.base.mem_mb * 1024).to_string(), ) .await?; tx.write_string( format!("{}/memory/target", dom_path).as_str(), - &(config.mem_mb * 1024).to_string(), + &(config.base.mem_mb * 1024).to_string(), ) .await?; tx.write_string(format!("{}/memory/videoram", dom_path).as_str(), "0") @@ -314,15 +274,15 @@ impl XenClient

{ .await?; tx.write_string( format!("{}/store/port", dom_path).as_str(), - &domain.store_evtchn.to_string(), + &created.store_evtchn.to_string(), ) .await?; tx.write_string( format!("{}/store/ring-ref", dom_path).as_str(), - &domain.store_mfn.to_string(), + &created.store_mfn.to_string(), ) .await?; - for i in 0..config.max_vcpus { + for i in 0..config.base.max_vcpus { let path = format!("{}/cpu/{}", dom_path, i); tx.mkdir(&path).await?; tx.set_perms(&path, ro_perm).await?; @@ -334,7 +294,7 @@ impl XenClient

{ } if !self .store - .introduce_domain(domid, domain.store_mfn, domain.store_evtchn) + .introduce_domain(domid, created.store_mfn, created.store_evtchn) .await? { return Err(Error::IntroduceDomainFailed); @@ -343,7 +303,7 @@ impl XenClient

{ let tx = self.store.transaction().await?; self.console_device_add( &tx, - &mut domain, + created, &DomainChannel { typ: config .swap_console_backend @@ -360,24 +320,18 @@ impl XenClient

{ ) .await?; - let mut channels: Vec = Vec::new(); for (index, channel) in config.channels.iter().enumerate() { - let (Some(ring_ref), Some(evtchn)) = self - .console_device_add( - &tx, - &mut domain, - channel, - &dom_path, - &backend_dom_path, - config.backend_domid, - domid, - index + 1, - ) - .await? - else { - continue; - }; - channels.push(CreatedChannel { ring_ref, evtchn }); + self.console_device_add( + &tx, + created, + channel, + &dom_path, + &backend_dom_path, + config.backend_domid, + domid, + index + 1, + ) + .await?; } for (index, disk) in config.disks.iter().enumerate() { @@ -448,7 +402,7 @@ impl XenClient

{ tx.commit().await?; self.call.unpause_domain(domid).await?; - Ok(CreatedDomain { domid, channels }) + Ok(()) } async fn disk_device_add( @@ -509,17 +463,16 @@ impl XenClient

{ async fn console_device_add( &self, tx: &XsdTransaction, - domain: &mut BootDomain, + domain: &CreatedDomain, channel: &DomainChannel, dom_path: &str, backend_dom_path: &str, backend_domid: u32, domid: u32, index: usize, - ) -> Result<(Option, Option)> { - let console = domain.consoles.get(index); - let port = console.map(|x| x.0); - let ring = console.map(|x| x.1); + ) -> Result> { + let port = domain.console_evtchn; + let ring = domain.console_mfn; let mut backend_entries = vec![ ("frontend-id", domid.to_string()), @@ -537,15 +490,13 @@ impl XenClient

{ frontend_entries.push(("type", channel.typ.clone())); backend_entries.push(("type", channel.typ.clone())); - if port.is_some() && ring.is_some() { + if index == 0 { if channel.typ != "xenconsoled" { frontend_entries.push(("state", "1".to_string())); } - frontend_entries.extend_from_slice(&[ - ("port", port.unwrap().to_string()), - ("ring-ref", ring.unwrap().to_string()), - ]); + frontend_entries + .extend_from_slice(&[("port", port.to_string()), ("ring-ref", ring.to_string())]); } else { frontend_entries.extend_from_slice(&[ ("state", "1".to_string()), @@ -571,7 +522,7 @@ impl XenClient

{ backend_entries, ) .await?; - Ok((ring, port)) + Ok(Some((port, ring))) } async fn fs_9p_device_add( @@ -863,10 +814,8 @@ impl XenClient

{ } pub async fn destroy(&self, domid: u32) -> Result<()> { - if let Err(err) = self.destroy_store(domid).await { - warn!("failed to destroy store for domain {}: {}", domid, err); - } - self.call.destroy_domain(domid).await?; + let _ = self.destroy_store(domid).await; + self.domain_manager.destroy(domid).await?; Ok(()) } diff --git a/crates/xen/xenplatform/src/boot.rs b/crates/xen/xenplatform/src/boot.rs index e78d8e0..d7a5789 100644 --- a/crates/xen/xenplatform/src/boot.rs +++ b/crates/xen/xenplatform/src/boot.rs @@ -43,7 +43,8 @@ pub struct BootDomain { pub store_evtchn: u32, pub store_mfn: u64, pub initrd_segment: DomainSegment, - pub consoles: Vec<(u32, u64)>, + pub console_evtchn: u32, + pub console_mfn: u64, pub cmdline: String, } @@ -177,7 +178,8 @@ impl BootSetup { target_pages: total_pages, page_size: self.platform.page_size(), image_info, - consoles: Vec::new(), + console_evtchn: 0, + console_mfn: 0, max_vcpus, phys: PhysicalPages::new(self.call.clone(), self.domid, self.platform.page_shift()), initrd_segment: DomainSegment::default(), @@ -261,7 +263,7 @@ impl BootSetup { #[async_trait::async_trait] pub trait BootSetupPlatform: Clone { - fn create_domain(&self, needs_passthrough: bool) -> CreateDomain; + fn create_domain(&self, enable_iommu: bool) -> CreateDomain; fn page_size(&self) -> u64; fn page_shift(&self) -> u64; fn needs_early_kernel(&self) -> bool; diff --git a/crates/xen/xenplatform/src/domain.rs b/crates/xen/xenplatform/src/domain.rs new file mode 100644 index 0000000..d253842 --- /dev/null +++ b/crates/xen/xenplatform/src/domain.rs @@ -0,0 +1,80 @@ +use std::sync::Arc; + +use crate::{ + boot::{BootSetup, BootSetupPlatform}, + elfloader::ElfImageLoader, +}; +use uuid::Uuid; +use xencall::XenCall; + +use crate::error::Result; + +pub struct BaseDomainManager { + call: XenCall, + pub platform: Arc

, +} + +impl BaseDomainManager

{ + pub async fn new(call: XenCall, platform: P) -> Result> { + Ok(BaseDomainManager { + call, + platform: Arc::new(platform), + }) + } + + pub async fn create(&self, config: BaseDomainConfig) -> Result { + let mut domain = self.platform.create_domain(config.enable_iommu); + domain.handle = config.uuid.into_bytes(); + domain.max_vcpus = config.max_vcpus; + let domid = self.call.create_domain(domain).await?; + self.call.set_max_vcpus(domid, config.max_vcpus).await?; + self.call + .set_max_mem(domid, (config.mem_mb * 1024) + 2048) + .await?; + let loader = ElfImageLoader::load_file_kernel(&config.kernel)?; + let platform = (*self.platform).clone(); + let mut boot = BootSetup::new(self.call.clone(), domid, platform, loader, None); + let mut domain = boot + .initialize( + &config.initrd, + config.mem_mb, + config.max_vcpus, + &config.cmdline, + ) + .await?; + boot.boot(&mut domain).await?; + Ok(CreatedDomain { + domid, + store_evtchn: domain.store_evtchn, + store_mfn: domain.store_mfn, + console_evtchn: domain.console_evtchn, + console_mfn: domain.console_mfn, + }) + } + + pub async fn destroy(&self, domid: u32) -> Result<()> { + self.call.destroy_domain(domid).await?; + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub struct BaseDomainConfig { + pub uuid: Uuid, + pub owner_domid: u32, + pub max_vcpus: u32, + pub mem_mb: u64, + pub kernel: Vec, + pub initrd: Vec, + pub cmdline: String, + pub enable_iommu: bool, +} + +#[derive(Clone, Debug)] +pub struct CreatedDomain { + pub domid: u32, + pub store_evtchn: u32, + pub store_mfn: u64, + pub console_evtchn: u32, + pub console_mfn: u64, +} diff --git a/crates/xen/xenplatform/src/lib.rs b/crates/xen/xenplatform/src/lib.rs index 322c388..6c841fc 100644 --- a/crates/xen/xenplatform/src/lib.rs +++ b/crates/xen/xenplatform/src/lib.rs @@ -6,6 +6,7 @@ pub mod sys; use crate::error::Error; +pub mod domain; pub mod unsupported; #[cfg(target_arch = "x86_64")] pub mod x86pv; diff --git a/crates/xen/xenplatform/src/x86pv.rs b/crates/xen/xenplatform/src/x86pv.rs index cd4d920..1ed2d89 100644 --- a/crates/xen/xenplatform/src/x86pv.rs +++ b/crates/xen/xenplatform/src/x86pv.rs @@ -434,9 +434,9 @@ impl X86PvPlatform { #[async_trait::async_trait] impl BootSetupPlatform for X86PvPlatform { - fn create_domain(&self, needs_passthrough: bool) -> CreateDomain { + fn create_domain(&self, enable_iommu: bool) -> CreateDomain { CreateDomain { - flags: if needs_passthrough { + flags: if enable_iommu { XEN_DOMCTL_CDF_IOMMU } else { 0 @@ -718,9 +718,8 @@ impl BootSetupPlatform for X86PvPlatform { domain.store_mfn = domain.phys.p2m[self.xenstore_segment.as_ref().unwrap().pfn as usize]; let evtchn = domain.call.evtchn_alloc_unbound(domain.domid, 0).await?; let page = domain.alloc_page()?; - domain - .consoles - .push((evtchn, domain.phys.p2m[page.pfn as usize])); + domain.console_evtchn = evtchn; + domain.console_mfn = domain.phys.p2m[page.pfn as usize]; self.page_table_segment = self.alloc_page_tables(domain).await?; self.boot_stack_segment = Some(domain.alloc_page()?); @@ -802,9 +801,8 @@ impl BootSetupPlatform for X86PvPlatform { (*info).flags = 0; (*info).store_evtchn = domain.store_evtchn; (*info).store_mfn = domain.phys.p2m[xenstore_segment.pfn as usize]; - let console = domain.consoles.first().unwrap(); - (*info).console.mfn = console.1; - (*info).console.evtchn = console.0; + (*info).console.mfn = domain.console_mfn; + (*info).console.evtchn = domain.console_evtchn; (*info).mod_start = domain.initrd_segment.vstart; (*info).mod_len = domain.initrd_segment.size; for (i, c) in domain.cmdline.chars().enumerate() { @@ -893,7 +891,7 @@ impl BootSetupPlatform for X86PvPlatform { .as_ref() .ok_or(Error::MemorySetupFailed("xenstore_segment missing"))?; - let console_gfn = domain.consoles.first().map(|x| x.1).unwrap_or(0) as usize; + let console_gfn = domain.console_mfn as usize; let xenstore_gfn = domain.phys.p2m[xenstore_segment.pfn as usize]; let addr = domain .call