diff --git a/controller/bin/control.rs b/controller/bin/control.rs index 475c450..5eab2a9 100644 --- a/controller/bin/control.rs +++ b/controller/bin/control.rs @@ -1,11 +1,12 @@ use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use env_logger::Env; -use kratactrl::ctl::{ - console::ControllerConsole, - destroy::ControllerDestroy, - launch::{ControllerLaunch, ControllerLaunchRequest}, - ControllerContext, +use kratactrl::{ + ctl::{ + console::ControllerConsole, destroy::ControllerDestroy, launch::ControllerLaunch, + ControllerContext, + }, + launch::GuestLaunchRequest, }; use std::path::PathBuf; @@ -86,7 +87,7 @@ async fn main() -> Result<()> { let kernel = map_kernel_path(&store_path, kernel); let initrd = map_initrd_path(&store_path, initrd); let mut launch = ControllerLaunch::new(&mut context); - let request = ControllerLaunchRequest { + let request = GuestLaunchRequest { kernel_path: &kernel, initrd_path: &initrd, image: &image, @@ -96,11 +97,11 @@ async fn main() -> Result<()> { run: if run.is_empty() { None } else { Some(run) }, debug, }; - let (uuid, _domid) = launch.perform(request).await?; - println!("launched container: {}", uuid); + let info = launch.perform(request).await?; + println!("launched guest: {}", info.uuid); if attach { let mut console = ControllerConsole::new(&mut context); - console.perform(&uuid.to_string()).await?; + console.perform(&info.uuid.to_string()).await?; } } @@ -130,7 +131,7 @@ async fn main() -> Result<()> { } if table.num_records() == 1 { - println!("no containers have been launched"); + println!("no guests have been launched"); } else { println!("{}", table.to_string()); } diff --git a/controller/src/ctl/cfgblk.rs b/controller/src/ctl/cfgblk.rs index 651e852..9ffb7f3 100644 --- a/controller/src/ctl/cfgblk.rs +++ b/controller/src/ctl/cfgblk.rs @@ -29,7 +29,7 @@ impl ConfigBlock<'_> { } pub fn build(&self, launch_config: &LaunchInfo) -> Result<()> { - trace!("ConfigBlock build launch_config={:?}", launch_config); + trace!("build launch_config={:?}", launch_config); let manifest = self.image_info.config.to_string()?; let launch = serde_json::to_string(launch_config)?; let mut writer = FilesystemWriter::default(); @@ -63,9 +63,9 @@ impl ConfigBlock<'_> { }, )?; let mut file = File::create(&self.file)?; - trace!("ConfigBlock build write sqaushfs"); + trace!("build write sqaushfs"); writer.write(&mut file)?; - trace!("ConfigBlock build complete"); + trace!("build complete"); Ok(()) } } diff --git a/controller/src/ctl/console.rs b/controller/src/ctl/console.rs index d162ab3..4711d8d 100644 --- a/controller/src/ctl/console.rs +++ b/controller/src/ctl/console.rs @@ -24,7 +24,7 @@ impl ControllerConsole<'_> { .context .resolve(id) .await? - .ok_or_else(|| anyhow!("unable to resolve container: {}", id))?; + .ok_or_else(|| anyhow!("unable to resolve guest: {}", id))?; let domid = info.domid; let tty = self.context.xen.get_console_path(domid).await?; let console = XenConsole::new(&tty).await?; diff --git a/controller/src/ctl/destroy.rs b/controller/src/ctl/destroy.rs index acacb86..42cfbf3 100644 --- a/controller/src/ctl/destroy.rs +++ b/controller/src/ctl/destroy.rs @@ -20,7 +20,7 @@ impl ControllerDestroy<'_> { .context .resolve(id) .await? - .ok_or_else(|| anyhow!("unable to resolve container: {}", id))?; + .ok_or_else(|| anyhow!("unable to resolve guest: {}", id))?; let domid = info.domid; let mut store = XsdClient::open().await?; let dom_path = store.get_domain_path(domid).await?; diff --git a/controller/src/ctl/launch.rs b/controller/src/ctl/launch.rs index 4d25dd7..6ae9aec 100644 --- a/controller/src/ctl/launch.rs +++ b/controller/src/ctl/launch.rs @@ -1,29 +1,6 @@ -use std::{fs, net::Ipv4Addr, str::FromStr}; - -use advmac::MacAddr6; -use anyhow::{anyhow, Result}; -use ipnetwork::Ipv4Network; -use krata::{ - LaunchInfo, LaunchNetwork, LaunchNetworkIpv4, LaunchNetworkIpv6, LaunchNetworkResolver, -}; -use uuid::Uuid; -use xenclient::{DomainConfig, DomainDisk, DomainNetworkInterface}; -use xenstore::client::XsdInterface; - -use crate::image::{name::ImageName, ImageCompiler, ImageInfo}; - -use super::{cfgblk::ConfigBlock, ControllerContext}; - -pub struct ControllerLaunchRequest<'a> { - pub kernel_path: &'a str, - pub initrd_path: &'a str, - pub image: &'a str, - pub vcpus: u32, - pub mem: u64, - pub env: Option>, - pub run: Option>, - pub debug: bool, -} +use super::{ControllerContext, GuestInfo}; +use crate::launch::{GuestLaunchRequest, GuestLauncher}; +use anyhow::Result; pub struct ControllerLaunch<'a> { context: &'a mut ControllerContext, @@ -34,195 +11,11 @@ impl ControllerLaunch<'_> { ControllerLaunch { context } } - pub async fn perform(&mut self, request: ControllerLaunchRequest<'_>) -> Result<(Uuid, u32)> { - let uuid = Uuid::new_v4(); - let name = format!("krata-{uuid}"); - let image_info = self.compile(request.image).await?; - - let mut gateway_mac = MacAddr6::random(); - gateway_mac.set_local(true); - gateway_mac.set_multicast(false); - let mut container_mac = MacAddr6::random(); - container_mac.set_local(true); - container_mac.set_multicast(false); - - let guest_ipv4 = self.allocate_ipv4().await?; - let guest_ipv6 = container_mac.to_link_local_ipv6(); - let gateway_ipv4 = "192.168.42.1"; - let gateway_ipv6 = "fe80::1"; - let ipv4_network_mask: u32 = 24; - let ipv6_network_mask: u32 = 10; - - let launch_config = LaunchInfo { - network: Some(LaunchNetwork { - link: "eth0".to_string(), - ipv4: LaunchNetworkIpv4 { - address: format!("{}/{}", guest_ipv4, ipv4_network_mask), - gateway: gateway_ipv4.to_string(), - }, - ipv6: LaunchNetworkIpv6 { - address: format!("{}/{}", guest_ipv6, ipv6_network_mask), - gateway: gateway_ipv6.to_string(), - }, - resolver: LaunchNetworkResolver { - nameservers: vec![ - "1.1.1.1".to_string(), - "1.0.0.1".to_string(), - "2606:4700:4700::1111".to_string(), - "2606:4700:4700::1001".to_string(), - ], - }, - }), - env: request.env, - run: request.run, - }; - - let cfgblk = ConfigBlock::new(&uuid, &image_info)?; - cfgblk.build(&launch_config)?; - - let image_squashfs_path = image_info - .image_squashfs - .to_str() - .ok_or_else(|| anyhow!("failed to convert image squashfs path to string"))?; - - let cfgblk_dir_path = cfgblk - .dir - .to_str() - .ok_or_else(|| anyhow!("failed to convert cfgblk directory path to string"))?; - let cfgblk_squashfs_path = cfgblk - .file - .to_str() - .ok_or_else(|| anyhow!("failed to convert cfgblk squashfs path to string"))?; - - let image_squashfs_loop = self.context.autoloop.loopify(image_squashfs_path)?; - let cfgblk_squashfs_loop = self.context.autoloop.loopify(cfgblk_squashfs_path)?; - - let cmdline_options = [ - if request.debug { "debug" } else { "quiet" }, - "elevator=noop", - ]; - let cmdline = cmdline_options.join(" "); - - let container_mac_string = container_mac.to_string().replace('-', ":"); - let gateway_mac_string = gateway_mac.to_string().replace('-', ":"); - let config = DomainConfig { - backend_domid: 0, - name: &name, - max_vcpus: request.vcpus, - mem_mb: request.mem, - kernel_path: request.kernel_path, - initrd_path: request.initrd_path, - cmdline: &cmdline, - disks: vec![ - DomainDisk { - vdev: "xvda", - block: &image_squashfs_loop, - writable: false, - }, - DomainDisk { - vdev: "xvdb", - block: &cfgblk_squashfs_loop, - writable: false, - }, - ], - consoles: vec![], - vifs: vec![DomainNetworkInterface { - mac: &container_mac_string, - mtu: 1500, - bridge: None, - script: None, - }], - filesystems: vec![], - event_channels: vec![], - extra_keys: vec![ - ("krata/uuid".to_string(), uuid.to_string()), - ( - "krata/loops".to_string(), - format!( - "{}:{}:none,{}:{}:{}", - &image_squashfs_loop.path, - image_squashfs_path, - &cfgblk_squashfs_loop.path, - cfgblk_squashfs_path, - cfgblk_dir_path, - ), - ), - ("krata/image".to_string(), request.image.to_string()), - ( - "krata/network/guest/ipv4".to_string(), - format!("{}/{}", guest_ipv4, ipv4_network_mask), - ), - ( - "krata/network/guest/ipv6".to_string(), - format!("{}/{}", guest_ipv6, ipv6_network_mask), - ), - ( - "krata/network/guest/mac".to_string(), - container_mac_string.clone(), - ), - ( - "krata/network/gateway/ipv4".to_string(), - format!("{}/{}", gateway_ipv4, ipv4_network_mask), - ), - ( - "krata/network/gateway/ipv6".to_string(), - format!("{}/{}", gateway_ipv6, ipv6_network_mask), - ), - ( - "krata/network/gateway/mac".to_string(), - gateway_mac_string.clone(), - ), - ], - extra_rw_paths: vec!["krata/guest".to_string()], - }; - match self.context.xen.create(&config).await { - Ok(domid) => Ok((uuid, domid)), - Err(error) => { - let _ = self.context.autoloop.unloop(&image_squashfs_loop.path); - let _ = self.context.autoloop.unloop(&cfgblk_squashfs_loop.path); - let _ = fs::remove_dir(&cfgblk.dir); - Err(error.into()) - } - } - } - - async fn allocate_ipv4(&mut self) -> Result { - let network = Ipv4Network::new(Ipv4Addr::new(192, 168, 42, 0), 24)?; - let mut used: Vec = vec![ - Ipv4Addr::new(192, 168, 42, 0), - Ipv4Addr::new(192, 168, 42, 1), - Ipv4Addr::new(192, 168, 42, 255), - ]; - for domid_candidate in self.context.xen.store.list("/local/domain").await? { - let dom_path = format!("/local/domain/{}", domid_candidate); - let ip_path = format!("{}/krata/network/guest/ipv4", dom_path); - let existing_ip = self.context.xen.store.read_string(&ip_path).await?; - if let Some(existing_ip) = existing_ip { - let ipv4_network = Ipv4Network::from_str(&existing_ip)?; - used.push(ipv4_network.ip()); - } - } - - let mut found: Option = None; - for ip in network.iter() { - if !used.contains(&ip) { - found = Some(ip); - break; - } - } - - if found.is_none() { - return Err(anyhow!( - "unable to find ipv4 to allocate to container, ipv4 addresses are exhausted" - )); - } - - Ok(found.unwrap()) - } - - async fn compile(&self, image: &str) -> Result { - let image = ImageName::parse(image)?; - let compiler = ImageCompiler::new(&self.context.image_cache)?; - compiler.compile(&image).await + pub async fn perform<'c, 'r>( + &'c mut self, + request: GuestLaunchRequest<'r>, + ) -> Result { + let mut launcher = GuestLauncher::new()?; + launcher.launch(self.context, request).await } } diff --git a/controller/src/ctl/mod.rs b/controller/src/ctl/mod.rs index ddd12c8..c720870 100644 --- a/controller/src/ctl/mod.rs +++ b/controller/src/ctl/mod.rs @@ -27,7 +27,7 @@ pub struct ContainerLoopInfo { pub delete: Option, } -pub struct ContainerInfo { +pub struct GuestInfo { pub uuid: Uuid, pub domid: u32, pub image: String, @@ -53,8 +53,8 @@ impl ControllerContext { }) } - pub async fn list(&mut self) -> Result> { - let mut containers: Vec = Vec::new(); + pub async fn list(&mut self) -> Result> { + let mut containers: Vec = Vec::new(); for domid_candidate in self.xen.store.list("/local/domain").await? { let dom_path = format!("/local/domain/{}", domid_candidate); let uuid_string = match self @@ -93,7 +93,7 @@ impl ControllerContext { .await? .unwrap_or("unknown".to_string()); let loops = ControllerContext::parse_loop_set(&loops); - containers.push(ContainerInfo { + containers.push(GuestInfo { uuid, domid, image, @@ -105,7 +105,7 @@ impl ControllerContext { Ok(containers) } - pub async fn resolve(&mut self, id: &str) -> Result> { + pub async fn resolve(&mut self, id: &str) -> Result> { for container in self.list().await? { let uuid_string = container.uuid.to_string(); let domid_string = container.domid.to_string(); diff --git a/controller/src/launch/mod.rs b/controller/src/launch/mod.rs new file mode 100644 index 0000000..d8733c1 --- /dev/null +++ b/controller/src/launch/mod.rs @@ -0,0 +1,240 @@ +use std::{fs, net::Ipv4Addr, str::FromStr}; + +use advmac::MacAddr6; +use anyhow::{anyhow, Result}; +use ipnetwork::Ipv4Network; +use krata::{ + LaunchInfo, LaunchNetwork, LaunchNetworkIpv4, LaunchNetworkIpv6, LaunchNetworkResolver, +}; +use uuid::Uuid; +use xenclient::{DomainConfig, DomainDisk, DomainNetworkInterface}; +use xenstore::client::XsdInterface; + +use crate::{ + ctl::GuestInfo, + image::{cache::ImageCache, name::ImageName, ImageCompiler, ImageInfo}, +}; + +use crate::ctl::{cfgblk::ConfigBlock, ControllerContext}; + +pub struct GuestLaunchRequest<'a> { + pub kernel_path: &'a str, + pub initrd_path: &'a str, + pub image: &'a str, + pub vcpus: u32, + pub mem: u64, + pub env: Option>, + pub run: Option>, + pub debug: bool, +} + +pub struct GuestLauncher {} + +impl GuestLauncher { + pub fn new() -> Result { + Ok(Self {}) + } + + pub async fn launch<'c, 'r>( + &mut self, + context: &'c mut ControllerContext, + request: GuestLaunchRequest<'r>, + ) -> Result { + let uuid = Uuid::new_v4(); + let name = format!("krata-{uuid}"); + let image_info = self.compile(request.image, &context.image_cache).await?; + + let mut gateway_mac = MacAddr6::random(); + gateway_mac.set_local(true); + gateway_mac.set_multicast(false); + let mut container_mac = MacAddr6::random(); + container_mac.set_local(true); + container_mac.set_multicast(false); + + let guest_ipv4 = self.allocate_ipv4(context).await?; + let guest_ipv6 = container_mac.to_link_local_ipv6(); + let gateway_ipv4 = "192.168.42.1"; + let gateway_ipv6 = "fe80::1"; + let ipv4_network_mask: u32 = 24; + let ipv6_network_mask: u32 = 10; + + let launch_config = LaunchInfo { + network: Some(LaunchNetwork { + link: "eth0".to_string(), + ipv4: LaunchNetworkIpv4 { + address: format!("{}/{}", guest_ipv4, ipv4_network_mask), + gateway: gateway_ipv4.to_string(), + }, + ipv6: LaunchNetworkIpv6 { + address: format!("{}/{}", guest_ipv6, ipv6_network_mask), + gateway: gateway_ipv6.to_string(), + }, + resolver: LaunchNetworkResolver { + nameservers: vec![ + "1.1.1.1".to_string(), + "1.0.0.1".to_string(), + "2606:4700:4700::1111".to_string(), + "2606:4700:4700::1001".to_string(), + ], + }, + }), + env: request.env, + run: request.run, + }; + + let cfgblk = ConfigBlock::new(&uuid, &image_info)?; + cfgblk.build(&launch_config)?; + + let image_squashfs_path = image_info + .image_squashfs + .to_str() + .ok_or_else(|| anyhow!("failed to convert image squashfs path to string"))?; + + let cfgblk_dir_path = cfgblk + .dir + .to_str() + .ok_or_else(|| anyhow!("failed to convert cfgblk directory path to string"))?; + let cfgblk_squashfs_path = cfgblk + .file + .to_str() + .ok_or_else(|| anyhow!("failed to convert cfgblk squashfs path to string"))?; + + let image_squashfs_loop = context.autoloop.loopify(image_squashfs_path)?; + let cfgblk_squashfs_loop = context.autoloop.loopify(cfgblk_squashfs_path)?; + + let cmdline_options = [ + if request.debug { "debug" } else { "quiet" }, + "elevator=noop", + ]; + let cmdline = cmdline_options.join(" "); + + let container_mac_string = container_mac.to_string().replace('-', ":"); + let gateway_mac_string = gateway_mac.to_string().replace('-', ":"); + let config = DomainConfig { + backend_domid: 0, + name: &name, + max_vcpus: request.vcpus, + mem_mb: request.mem, + kernel_path: request.kernel_path, + initrd_path: request.initrd_path, + cmdline: &cmdline, + disks: vec![ + DomainDisk { + vdev: "xvda", + block: &image_squashfs_loop, + writable: false, + }, + DomainDisk { + vdev: "xvdb", + block: &cfgblk_squashfs_loop, + writable: false, + }, + ], + consoles: vec![], + vifs: vec![DomainNetworkInterface { + mac: &container_mac_string, + mtu: 1500, + bridge: None, + script: None, + }], + filesystems: vec![], + event_channels: vec![], + extra_keys: vec![ + ("krata/uuid".to_string(), uuid.to_string()), + ( + "krata/loops".to_string(), + format!( + "{}:{}:none,{}:{}:{}", + &image_squashfs_loop.path, + image_squashfs_path, + &cfgblk_squashfs_loop.path, + cfgblk_squashfs_path, + cfgblk_dir_path, + ), + ), + ("krata/image".to_string(), request.image.to_string()), + ( + "krata/network/guest/ipv4".to_string(), + format!("{}/{}", guest_ipv4, ipv4_network_mask), + ), + ( + "krata/network/guest/ipv6".to_string(), + format!("{}/{}", guest_ipv6, ipv6_network_mask), + ), + ( + "krata/network/guest/mac".to_string(), + container_mac_string.clone(), + ), + ( + "krata/network/gateway/ipv4".to_string(), + format!("{}/{}", gateway_ipv4, ipv4_network_mask), + ), + ( + "krata/network/gateway/ipv6".to_string(), + format!("{}/{}", gateway_ipv6, ipv6_network_mask), + ), + ( + "krata/network/gateway/mac".to_string(), + gateway_mac_string.clone(), + ), + ], + extra_rw_paths: vec!["krata/guest".to_string()], + }; + match context.xen.create(&config).await { + Ok(domid) => Ok(GuestInfo { + uuid, + domid, + image: request.image.to_string(), + loops: vec![], + ipv4: format!("{}/{}", guest_ipv4, ipv4_network_mask), + ipv6: format!("{}/{}", guest_ipv6, ipv6_network_mask), + }), + Err(error) => { + let _ = context.autoloop.unloop(&image_squashfs_loop.path); + let _ = context.autoloop.unloop(&cfgblk_squashfs_loop.path); + let _ = fs::remove_dir(&cfgblk.dir); + Err(error.into()) + } + } + } + + async fn compile(&self, image: &str, image_cache: &ImageCache) -> Result { + let image = ImageName::parse(image)?; + let compiler = ImageCompiler::new(image_cache)?; + compiler.compile(&image).await + } + + async fn allocate_ipv4(&mut self, context: &mut ControllerContext) -> Result { + let network = Ipv4Network::new(Ipv4Addr::new(192, 168, 42, 0), 24)?; + let mut used: Vec = vec![ + Ipv4Addr::new(192, 168, 42, 0), + Ipv4Addr::new(192, 168, 42, 1), + Ipv4Addr::new(192, 168, 42, 255), + ]; + for domid_candidate in context.xen.store.list("/local/domain").await? { + let dom_path = format!("/local/domain/{}", domid_candidate); + let ip_path = format!("{}/krata/network/guest/ipv4", dom_path); + let existing_ip = context.xen.store.read_string(&ip_path).await?; + if let Some(existing_ip) = existing_ip { + let ipv4_network = Ipv4Network::from_str(&existing_ip)?; + used.push(ipv4_network.ip()); + } + } + + let mut found: Option = None; + for ip in network.iter() { + if !used.contains(&ip) { + found = Some(ip); + break; + } + } + + if found.is_none() { + return Err(anyhow!( + "unable to find ipv4 to allocate to container, ipv4 addresses are exhausted" + )); + } + + Ok(found.unwrap()) + } +} diff --git a/controller/src/lib.rs b/controller/src/lib.rs index 82d321b..ceefb8f 100644 --- a/controller/src/lib.rs +++ b/controller/src/lib.rs @@ -2,3 +2,4 @@ pub mod autoloop; pub mod console; pub mod ctl; pub mod image; +pub mod launch; diff --git a/libs/xen/xenstore/src/bus.rs b/libs/xen/xenstore/src/bus.rs index a69d679..2f9ea51 100644 --- a/libs/xen/xenstore/src/bus.rs +++ b/libs/xen/xenstore/src/bus.rs @@ -96,9 +96,14 @@ impl XsdSocket { composed.extend_from_slice(buf); self.handle.xsd_write_all(&composed).await?; let mut result_buf = vec![0u8; size_of::()]; - self.handle - .xsd_read_exact(result_buf.as_mut_slice()) - .await?; + match self.handle.xsd_read_exact(result_buf.as_mut_slice()).await { + Ok(_) => {} + Err(error) => { + if result_buf.first().unwrap() == &0 { + return Err(error); + } + } + } let result_header = bytemuck::from_bytes::(&result_buf); let mut payload = vec![0u8; result_header.len as usize]; self.handle.xsd_read_exact(payload.as_mut_slice()).await?;