krata: work on parallel reconciliation

crates/runtime/src/autoloop.rs

@@ -1,17 +1,25 @@
+use std::{sync::Arc, time::Duration};
+
 use anyhow::{anyhow, Result};
+use log::debug;
 use loopdev::{LoopControl, LoopDevice};
+use tokio::time::sleep;
 use xenclient::BlockDeviceRef;
 
+#[derive(Clone)]
 pub struct AutoLoop {
-    control: LoopControl,
+    control: Arc<LoopControl>,
 }
 
 impl AutoLoop {
     pub fn new(control: LoopControl) -> AutoLoop {
-        AutoLoop { control }
+        AutoLoop {
+            control: Arc::new(control),
+        }
     }
 
     pub fn loopify(&self, file: &str) -> Result<BlockDeviceRef> {
+        debug!("creating loop for file {}", file);
         let device = self.control.next_free()?;
         device.with().read_only(true).attach(file)?;
         let path = device
@@ -25,9 +33,10 @@ impl AutoLoop {
         Ok(BlockDeviceRef { path, major, minor })
     }
 
-    pub fn unloop(&self, device: &str) -> Result<()> {
+    pub async fn unloop(&self, device: &str) -> Result<()> {
         let device = LoopDevice::open(device)?;
         device.detach()?;
+        sleep(Duration::from_millis(200)).await;
         Ok(())
     }
 }

crates/runtime/src/launch.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 use std::net::{IpAddr, Ipv6Addr};
+use std::sync::Arc;
 use std::{fs, net::Ipv4Addr, str::FromStr};
 
 use advmac::MacAddr6;
@@ -8,6 +9,7 @@ use ipnetwork::{IpNetwork, Ipv4Network};
 use krata::launchcfg::{
     LaunchInfo, LaunchNetwork, LaunchNetworkIpv4, LaunchNetworkIpv6, LaunchNetworkResolver,
 };
+use tokio::sync::Semaphore;
 use uuid::Uuid;
 use xenclient::{DomainChannel, DomainConfig, DomainDisk, DomainNetworkInterface};
 use xenstore::XsdInterface;
@@ -33,16 +35,18 @@ pub struct GuestLaunchRequest<'a> {
     pub debug: bool,
 }
 
-pub struct GuestLauncher {}
+pub struct GuestLauncher {
+    pub launch_semaphore: Arc<Semaphore>,
+}
 
 impl GuestLauncher {
-    pub fn new() -> Result<Self> {
-        Ok(Self {})
+    pub fn new(launch_semaphore: Arc<Semaphore>) -> Result<Self> {
+        Ok(Self { launch_semaphore })
     }
 
     pub async fn launch<'r>(
         &mut self,
-        context: &mut RuntimeContext,
+        context: &RuntimeContext,
         request: GuestLaunchRequest<'r>,
     ) -> Result<GuestInfo> {
         let uuid = request.uuid.unwrap_or_else(Uuid::new_v4);
@@ -56,6 +60,7 @@ impl GuestLauncher {
         container_mac.set_local(true);
         container_mac.set_multicast(false);
 
+        let _launch_permit = self.launch_semaphore.acquire().await?;
         let guest_ipv4 = self.allocate_ipv4(context).await?;
         let guest_ipv6 = container_mac.to_link_local_ipv6();
         let gateway_ipv4 = "10.75.70.1";
@@ -223,14 +228,14 @@ impl GuestLauncher {
                 )?),
                 gateway_ipv6: Some(IpNetwork::new(
                     IpAddr::V6(Ipv6Addr::from_str(gateway_ipv6)?),
-                    ipv4_network_mask as u8,
+                    ipv6_network_mask as u8,
                 )?),
                 gateway_mac: Some(gateway_mac_string.clone()),
                 state: GuestState { exit_code: None },
             }),
             Err(error) => {
-                let _ = context.autoloop.unloop(&image_squashfs_loop.path);
-                let _ = context.autoloop.unloop(&cfgblk_squashfs_loop.path);
+                let _ = context.autoloop.unloop(&image_squashfs_loop.path).await;
+                let _ = context.autoloop.unloop(&cfgblk_squashfs_loop.path).await;
                 let _ = fs::remove_dir(&cfgblk.dir);
                 Err(error.into())
             }
@@ -243,7 +248,7 @@ impl GuestLauncher {
         compiler.compile(&image).await
     }
 
-    async fn allocate_ipv4(&mut self, context: &mut RuntimeContext) -> Result<Ipv4Addr> {
+    async fn allocate_ipv4(&self, context: &RuntimeContext) -> Result<Ipv4Addr> {
         let network = Ipv4Network::new(Ipv4Addr::new(10, 75, 80, 0), 24)?;
         let mut used: Vec<Ipv4Addr> = vec![];
         for domid_candidate in context.xen.store.list("/local/domain").await? {
@@ -270,7 +275,7 @@ impl GuestLauncher {
 
         if found.is_none() {
             return Err(anyhow!(
-                "unable to find ipv4 to allocate to container, ipv4 addresses are exhausted"
+                "unable to find ipv4 to allocate to guest, ipv4 addresses are exhausted"
             ));
         }
 

crates/runtime/src/lib.rs

@@ -8,7 +8,7 @@ use std::{
 use anyhow::{anyhow, Result};
 use ipnetwork::IpNetwork;
 use loopdev::LoopControl;
-use tokio::sync::Mutex;
+use tokio::sync::Semaphore;
 use uuid::Uuid;
 use xenclient::XenClient;
 use xenstore::{XsdClient, XsdInterface};
@@ -51,6 +51,7 @@ pub struct GuestInfo {
     pub state: GuestState,
 }
 
+#[derive(Clone)]
 pub struct RuntimeContext {
     pub image_cache: ImageCache,
     pub autoloop: AutoLoop,
@@ -94,7 +95,7 @@ impl RuntimeContext {
         Err(anyhow!("unable to find required guest file: {}", name))
     }
 
-    pub async fn list(&mut self) -> Result<Vec<GuestInfo>> {
+    pub async fn list(&self) -> Result<Vec<GuestInfo>> {
         let mut guests: Vec<GuestInfo> = Vec::new();
         for domid_candidate in self.xen.store.list("/local/domain").await? {
             if domid_candidate == "0" {
@@ -218,7 +219,7 @@ impl RuntimeContext {
         Ok(guests)
     }
 
-    pub async fn resolve(&mut self, uuid: Uuid) -> Result<Option<GuestInfo>> {
+    pub async fn resolve(&self, uuid: Uuid) -> Result<Option<GuestInfo>> {
         for guest in self.list().await? {
             if guest.uuid == uuid {
                 return Ok(Some(guest));
@@ -254,7 +255,8 @@ impl RuntimeContext {
 #[derive(Clone)]
 pub struct Runtime {
     store: Arc<String>,
-    context: Arc<Mutex<RuntimeContext>>,
+    context: RuntimeContext,
+    launch_semaphore: Arc<Semaphore>,
 }
 
 impl Runtime {
@@ -262,24 +264,24 @@ impl Runtime {
         let context = RuntimeContext::new(store.clone()).await?;
         Ok(Self {
             store: Arc::new(store),
-            context: Arc::new(Mutex::new(context)),
+            context,
+            launch_semaphore: Arc::new(Semaphore::new(1)),
         })
     }
 
     pub async fn launch<'a>(&self, request: GuestLaunchRequest<'a>) -> Result<GuestInfo> {
-        let mut context = self.context.lock().await;
-        let mut launcher = GuestLauncher::new()?;
-        launcher.launch(&mut context, request).await
+        let mut launcher = GuestLauncher::new(self.launch_semaphore.clone())?;
+        launcher.launch(&self.context, request).await
     }
 
     pub async fn destroy(&self, uuid: Uuid) -> Result<Uuid> {
-        let mut context = self.context.lock().await;
-        let info = context
+        let info = self
+            .context
             .resolve(uuid)
             .await?
             .ok_or_else(|| anyhow!("unable to resolve guest: {}", uuid))?;
         let domid = info.domid;
-        let mut store = XsdClient::open().await?;
+        let store = XsdClient::open().await?;
         let dom_path = store.get_domain_path(domid).await?;
         let uuid = match store
             .read_string(format!("{}/krata/uuid", dom_path).as_str())
@@ -301,9 +303,9 @@ impl Runtime {
             .read_string(format!("{}/krata/loops", dom_path).as_str())
             .await?;
         let loops = RuntimeContext::parse_loop_set(&loops);
-        context.xen.destroy(domid).await?;
+        self.context.xen.destroy(domid).await?;
         for info in &loops {
-            context.autoloop.unloop(&info.device)?;
+            self.context.autoloop.unloop(&info.device).await?;
             match &info.delete {
                 None => {}
                 Some(delete) => {
@@ -320,19 +322,18 @@ impl Runtime {
     }
 
     pub async fn console(&self, uuid: Uuid) -> Result<XenConsole> {
-        let mut context = self.context.lock().await;
-        let info = context
+        let info = self
+            .context
             .resolve(uuid)
             .await?
             .ok_or_else(|| anyhow!("unable to resolve guest: {}", uuid))?;
         let domid = info.domid;
-        let tty = context.xen.get_console_path(domid).await?;
+        let tty = self.context.xen.get_console_path(domid).await?;
         XenConsole::new(&tty).await
     }
 
     pub async fn list(&self) -> Result<Vec<GuestInfo>> {
-        let mut context = self.context.lock().await;
-        context.list().await
+        self.context.list().await
     }
 
     pub async fn dupe(&self) -> Result<Runtime> {