network: auto-retry backend startup

network/src/autonet.rs

@@ -178,4 +178,8 @@ impl AutoNetworkCollector {
 
         Ok(AutoNetworkChangeset { added, removed })
     }
+
+    pub fn mark_unknown(&mut self, uuid: Uuid) -> Result<bool> {
+        Ok(self.known.remove(&uuid).is_some())
+    }
 }

network/src/backend.rs

@@ -60,15 +60,16 @@ impl NetworkStack<'_> {
                     trace!("failed to send guest packet to bridge: {}", error);
                 }
 
-                let slice = SlicedPacket::from_ethernet(packet)?;
+                if let Ok(slice) = SlicedPacket::from_ethernet(packet) {
-                let packet = RecvPacket::new(packet, &slice)?;
+                    let packet = RecvPacket::new(packet, &slice)?;
-                if let Err(error) = self.router.process(&packet).await {
+                    if let Err(error) = self.router.process(&packet).await {
-                    debug!("router failed to process packet: {}", error);
+                        debug!("router failed to process packet: {}", error);
-                }
+                    }
 
-                self.udev.rx = Some(packet.raw.into());
+                    self.udev.rx = Some(packet.raw.into());
-                self.interface
+                    self.interface
-                    .poll(Instant::now(), &mut self.udev, &mut self.sockets);
+                        .poll(Instant::now(), &mut self.udev, &mut self.sockets);
+                }
             }
 
             NetworkStackSelect::Send(Some(packet)) => self.kdev.write_all(&packet).await?,

network/src/lib.rs

@@ -2,7 +2,10 @@ use std::time::Duration;
 
 use anyhow::Result;
 use autonet::{AutoNetworkChangeset, AutoNetworkCollector, NetworkMetadata};
+use futures::{future::join_all, TryFutureExt};
+use log::warn;
 use tokio::time::sleep;
+use uuid::Uuid;
 use vbridge::VirtualBridge;
 
 use crate::backend::NetworkBackend;
@@ -34,25 +37,50 @@ impl NetworkService {
         let mut collector = AutoNetworkCollector::new()?;
         loop {
             let changeset = collector.read_changes()?;
-            self.process_network_changeset(changeset)?;
+            self.process_network_changeset(&mut collector, changeset)?;
             sleep(Duration::from_secs(2)).await;
         }
     }
 
-    fn process_network_changeset(&mut self, changeset: AutoNetworkChangeset) -> Result<()> {
+    fn process_network_changeset(
-        for metadata in &changeset.added {
+        &mut self,
-            futures::executor::block_on(async {
+        collector: &mut AutoNetworkCollector,
-                self.add_network_backend(metadata.clone()).await
+        changeset: AutoNetworkChangeset,
-            })?;
+    ) -> Result<()> {
+        let futures = changeset
+            .added
+            .iter()
+            .map(|metadata| {
+                self.add_network_backend(metadata.clone())
+                    .map_err(|x| (metadata.clone(), x))
+            })
+            .collect::<Vec<_>>();
+
+        let failed = futures::executor::block_on(async move {
+            let mut failed: Vec<Uuid> = Vec::new();
+            let results = join_all(futures).await;
+            for result in results {
+                if let Err((metadata, error)) = result {
+                    warn!(
+                        "failed to launch network backend for hypha guest {}: {}",
+                        metadata.uuid, error
+                    );
+                    failed.push(metadata.uuid);
+                }
+            }
+            failed
+        });
+
+        for uuid in failed {
+            collector.mark_unknown(uuid)?;
         }
 
         Ok(())
     }
 
-    async fn add_network_backend(&mut self, metadata: NetworkMetadata) -> Result<()> {
+    async fn add_network_backend(&self, metadata: NetworkMetadata) -> Result<()> {
         let mut network = NetworkBackend::new(metadata, self.bridge.clone())?;
         network.init().await?;
-        tokio::time::sleep(Duration::from_secs(1)).await;
         network.launch().await?;
         Ok(())
     }