fix(network): allocate host ip from allocation pool (#353)

crates/ctl/src/cli/host/identify.rs

@@ -17,6 +17,9 @@ impl HostStatusCommand {
         println!("Host UUID: {}", response.host_uuid);
         println!("Host Domain: {}", response.host_domid);
         println!("Krata Version: {}", response.krata_version);
+        println!("Host IPv4: {}", response.host_ipv4);
+        println!("Host IPv6: {}", response.host_ipv6);
+        println!("Host Ethernet Address: {}", response.host_mac);
         Ok(())
     }
 }

crates/daemon/src/config.rs

@@ -97,7 +97,7 @@ fn default_network_ipv4() -> DaemonIpv4NetworkConfig {
 }
 
 fn default_network_ipv4_subnet() -> String {
-    "10.75.80.0/24".to_string()
+    "10.75.0.0/16".to_string()
 }
 
 fn default_network_ipv6() -> DaemonIpv6NetworkConfig {

crates/daemon/src/console.rs

@@ -24,7 +24,7 @@ type BufferMap = Arc<Mutex<HashMap<u32, ConsoleBuffer>>>;
 
 #[derive(Clone)]
 pub struct DaemonConsoleHandle {
-    glt: ZoneLookupTable,
+    zlt: ZoneLookupTable,
     listeners: ListenerMap,
     buffers: BufferMap,
     sender: Sender<(u32, Vec<u8>)>,
@@ -57,7 +57,7 @@ impl DaemonConsoleHandle {
         uuid: Uuid,
         sender: Sender<Vec<u8>>,
     ) -> Result<DaemonConsoleAttachHandle> {
-        let Some(domid) = self.glt.lookup_domid_by_uuid(&uuid).await else {
+        let Some(domid) = self.zlt.lookup_domid_by_uuid(&uuid).await else {
             return Err(anyhow!("unable to find domain {}", uuid));
         };
         let buffers = self.buffers.lock().await;
@@ -84,7 +84,7 @@ impl Drop for DaemonConsoleHandle {
 }
 
 pub struct DaemonConsole {
-    glt: ZoneLookupTable,
+    zlt: ZoneLookupTable,
     listeners: ListenerMap,
     buffers: BufferMap,
     receiver: Receiver<(u32, Option<Vec<u8>>)>,
@@ -93,14 +93,14 @@ pub struct DaemonConsole {
 }
 
 impl DaemonConsole {
-    pub async fn new(glt: ZoneLookupTable) -> Result<DaemonConsole> {
+    pub async fn new(zlt: ZoneLookupTable) -> Result<DaemonConsole> {
         let (service, sender, receiver) =
             ChannelService::new("krata-console".to_string(), Some(0)).await?;
         let task = service.launch().await?;
         let listeners = Arc::new(Mutex::new(HashMap::new()));
         let buffers = Arc::new(Mutex::new(HashMap::new()));
         Ok(DaemonConsole {
-            glt,
+            zlt,
             listeners,
             buffers,
             receiver,
@@ -110,7 +110,7 @@ impl DaemonConsole {
     }
 
     pub async fn launch(mut self) -> Result<DaemonConsoleHandle> {
-        let glt = self.glt.clone();
+        let zlt = self.zlt.clone();
         let listeners = self.listeners.clone();
         let buffers = self.buffers.clone();
         let sender = self.sender.clone();
@@ -120,7 +120,7 @@ impl DaemonConsole {
             }
         });
         Ok(DaemonConsoleHandle {
-            glt,
+            zlt,
             listeners,
             buffers,
             sender,

crates/daemon/src/control.rs

@@ -1,4 +1,5 @@
 use crate::db::zone::ZoneStore;
+use crate::ip::assignment::IpAssignment;
 use crate::{
     command::DaemonCommand, console::DaemonConsoleHandle, devices::DaemonDeviceManager,
     event::DaemonEventContext, idm::DaemonIdmHandle, metrics::idm_metric_to_api,
@@ -68,12 +69,13 @@ impl From<ApiError> for Status {
 
 #[derive(Clone)]
 pub struct DaemonControlService {
-    glt: ZoneLookupTable,
+    zlt: ZoneLookupTable,
     devices: DaemonDeviceManager,
     events: DaemonEventContext,
     console: DaemonConsoleHandle,
     idm: DaemonIdmHandle,
     zones: ZoneStore,
+    ip: IpAssignment,
     zone_reconciler_notify: Sender<Uuid>,
     packer: OciPackerService,
     runtime: Runtime,
@@ -82,23 +84,25 @@ pub struct DaemonControlService {
 impl DaemonControlService {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
-        glt: ZoneLookupTable,
+        zlt: ZoneLookupTable,
         devices: DaemonDeviceManager,
         events: DaemonEventContext,
         console: DaemonConsoleHandle,
         idm: DaemonIdmHandle,
         zones: ZoneStore,
+        ip: IpAssignment,
         zone_reconciler_notify: Sender<Uuid>,
         packer: OciPackerService,
         runtime: Runtime,
     ) -> Self {
         Self {
-            glt,
+            zlt,
             devices,
             events,
             console,
             idm,
             zones,
+            ip,
             zone_reconciler_notify,
             packer,
             runtime,
@@ -138,10 +142,29 @@ impl ControlService for DaemonControlService {
         request: Request<HostStatusRequest>,
     ) -> Result<Response<HostStatusReply>, Status> {
         let _ = request.into_inner();
+        let host_reservation =
+            self.ip
+                .retrieve(self.zlt.host_uuid())
+                .await
+                .map_err(|x| ApiError {
+                    message: x.to_string(),
+                })?;
         Ok(Response::new(HostStatusReply {
-            host_domid: self.glt.host_domid(),
+            host_domid: self.zlt.host_domid(),
-            host_uuid: self.glt.host_uuid().to_string(),
+            host_uuid: self.zlt.host_uuid().to_string(),
             krata_version: DaemonCommand::version(),
+            host_ipv4: host_reservation
+                .as_ref()
+                .map(|x| format!("{}/{}", x.ipv4, x.ipv4_prefix))
+                .unwrap_or_default(),
+            host_ipv6: host_reservation
+                .as_ref()
+                .map(|x| format!("{}/{}", x.ipv6, x.ipv6_prefix))
+                .unwrap_or_default(),
+            host_mac: host_reservation
+                .as_ref()
+                .map(|x| x.mac.to_string().to_lowercase().replace('-', ":"))
+                .unwrap_or_default(),
         }))
     }
 
@@ -168,7 +191,7 @@ impl ControlService for DaemonControlService {
                         exit_status: None,
                         error_status: None,
                         resource_status: None,
-                        host: self.glt.host_uuid().to_string(),
+                        host: self.zlt.host_uuid().to_string(),
                         domid: u32::MAX,
                     }),
                     spec: Some(spec),
@@ -531,13 +554,13 @@ impl ControlService for DaemonControlService {
     ) -> Result<Response<Self::SnoopIdmStream>, Status> {
         let _ = request.into_inner();
         let mut messages = self.idm.snoop();
-        let glt = self.glt.clone();
+        let zlt = self.zlt.clone();
         let output = try_stream! {
             while let Ok(event) = messages.recv().await {
-                let Some(from_uuid) = glt.lookup_uuid_by_domid(event.from).await else {
+                let Some(from_uuid) = zlt.lookup_uuid_by_domid(event.from).await else {
                     continue;
                 };
-                let Some(to_uuid) = glt.lookup_uuid_by_domid(event.to).await else {
+                let Some(to_uuid) = zlt.lookup_uuid_by_domid(event.to).await else {
                     continue;
                 };
                 yield SnoopIdmReply { from: from_uuid.to_string(), to: to_uuid.to_string(), packet: Some(event.packet) };

crates/daemon/src/idm.rs

@@ -31,7 +31,7 @@ type ClientMap = Arc<Mutex<HashMap<u32, IdmInternalClient>>>;
 
 #[derive(Clone)]
 pub struct DaemonIdmHandle {
-    glt: ZoneLookupTable,
+    zlt: ZoneLookupTable,
     clients: ClientMap,
     feeds: BackendFeedMap,
     tx_sender: Sender<(u32, IdmTransportPacket)>,
@@ -45,7 +45,7 @@ impl DaemonIdmHandle {
     }
 
     pub async fn client(&self, uuid: Uuid) -> Result<IdmInternalClient> {
-        let Some(domid) = self.glt.lookup_domid_by_uuid(&uuid).await else {
+        let Some(domid) = self.zlt.lookup_domid_by_uuid(&uuid).await else {
             return Err(anyhow!("unable to find domain {}", uuid));
         };
         self.client_by_domid(domid).await
@@ -72,7 +72,7 @@ pub struct DaemonIdmSnoopPacket {
 }
 
 pub struct DaemonIdm {
-    glt: ZoneLookupTable,
+    zlt: ZoneLookupTable,
     clients: ClientMap,
     feeds: BackendFeedMap,
     tx_sender: Sender<(u32, IdmTransportPacket)>,
@@ -84,7 +84,7 @@ pub struct DaemonIdm {
 }
 
 impl DaemonIdm {
-    pub async fn new(glt: ZoneLookupTable) -> Result<DaemonIdm> {
+    pub async fn new(zlt: ZoneLookupTable) -> Result<DaemonIdm> {
         debug!("allocating channel service for idm");
         let (service, tx_raw_sender, rx_receiver) =
             ChannelService::new("krata-channel".to_string(), None).await?;
@@ -98,7 +98,7 @@ impl DaemonIdm {
         let feeds = Arc::new(Mutex::new(HashMap::new()));
 
         Ok(DaemonIdm {
-            glt,
+            zlt,
             rx_receiver,
             tx_receiver,
             tx_sender,
@@ -111,7 +111,7 @@ impl DaemonIdm {
     }
 
     pub async fn launch(mut self) -> Result<DaemonIdmHandle> {
-        let glt = self.glt.clone();
+        let zlt = self.zlt.clone();
         let clients = self.clients.clone();
         let feeds = self.feeds.clone();
         let tx_sender = self.tx_sender.clone();
@@ -124,7 +124,7 @@ impl DaemonIdm {
             }
         });
         Ok(DaemonIdmHandle {
-            glt,
+            zlt,
             clients,
             feeds,
             tx_sender,

crates/daemon/src/ip/assignment.rs

@@ -36,13 +36,13 @@ impl IpAssignment {
         store: IpReservationStore,
     ) -> Result<Self> {
         let mut state = IpAssignment::fetch_current_state(&store).await?;
-        let reservation = if let Some(reservation) = store.read(host_uuid).await? {
+        let gateway_reservation = if let Some(reservation) = store.read(Uuid::nil()).await? {
             reservation
         } else {
             IpAssignment::allocate(
                 &mut state,
                 &store,
-                host_uuid,
+                Uuid::nil(),
                 ipv4_network,
                 ipv6_network,
                 None,
@@ -51,12 +51,27 @@ impl IpAssignment {
             )
             .await?
         };
+
+        if store.read(host_uuid).await?.is_none() {
+            let _ = IpAssignment::allocate(
+                &mut state,
+                &store,
+                host_uuid,
+                ipv4_network,
+                ipv6_network,
+                Some(gateway_reservation.gateway_ipv4),
+                Some(gateway_reservation.gateway_ipv6),
+                Some(gateway_reservation.gateway_mac),
+            )
+            .await?;
+        }
+
         let assignment = IpAssignment {
             ipv4_network,
             ipv6_network,
-            gateway_ipv4: reservation.ipv4,
+            gateway_ipv4: gateway_reservation.ipv4,
-            gateway_ipv6: reservation.ipv6,
+            gateway_ipv6: gateway_reservation.ipv6,
-            gateway_mac: reservation.gateway_mac,
+            gateway_mac: gateway_reservation.mac,
             store,
             state: Arc::new(RwLock::new(state)),
         };
@@ -92,13 +107,17 @@ impl IpAssignment {
             .filter(|ip| {
                 let last = ip.octets()[3];
                 // filter for IPs ending in .1 to .250 because .250+ can have special meaning
-                last > 0 && last < 250
+                (1..250).contains(&last)
             })
             .find(|ip| !state.ipv4.contains_key(ip));
 
         let found_ipv6: Option<Ipv6Addr> = ipv6_network
             .iter()
             .filter(|ip| !ip.is_loopback() && !ip.is_multicast())
+            .filter(|ip| {
+                let last = ip.octets()[15];
+                last > 0
+            })
             .find(|ip| !state.ipv6.contains_key(ip));
 
         let Some(ipv4) = found_ipv4 else {
@@ -114,7 +133,7 @@ impl IpAssignment {
         };
 
         let mut mac = MacAddr6::random();
-        mac.set_local(false);
+        mac.set_local(true);
         mac.set_multicast(false);
 
         let reservation = IpReservation {

crates/daemon/src/lib.rs

@@ -45,9 +45,10 @@ pub mod zlt;
 pub struct Daemon {
     store: String,
     _config: Arc<DaemonConfig>,
-    glt: ZoneLookupTable,
+    zlt: ZoneLookupTable,
     devices: DaemonDeviceManager,
     zones: ZoneStore,
+    ip: IpAssignment,
     events: DaemonEventContext,
     zone_reconciler_task: JoinHandle<()>,
     zone_reconciler_notify: Sender<Uuid>,
@@ -127,7 +128,7 @@ impl Daemon {
         let ipv4_network = Ipv4Network::from_str(&config.network.ipv4.subnet)?;
         let ipv6_network = Ipv6Network::from_str(&config.network.ipv6.subnet)?;
         let ip_reservation_store = IpReservationStore::open(database)?;
-        let ip_assignment =
+        let ip =
             IpAssignment::new(host_uuid, ipv4_network, ipv6_network, ip_reservation_store).await?;
         debug!("initializing zone reconciler");
         let zone_reconciler = ZoneReconciler::new(
@@ -141,7 +142,7 @@ impl Daemon {
             kernel_path,
             initrd_path,
             addons_path,
-            ip_assignment,
+            ip.clone(),
             config.clone(),
         )?;
 
@@ -161,9 +162,10 @@ impl Daemon {
         Ok(Self {
             store,
             _config: config,
-            glt: zlt,
+            zlt,
             devices,
             zones,
+            ip,
             events,
             zone_reconciler_task,
             zone_reconciler_notify,
@@ -178,12 +180,13 @@ impl Daemon {
     pub async fn listen(&mut self, addr: ControlDialAddress) -> Result<()> {
         debug!("starting control service");
         let control_service = DaemonControlService::new(
-            self.glt.clone(),
+            self.zlt.clone(),
             self.devices.clone(),
             self.events.clone(),
             self.console.clone(),
             self.idm.clone(),
             self.zones.clone(),
+            self.ip.clone(),
             self.zone_reconciler_notify.clone(),
             self.packer.clone(),
             self.runtime.clone(),

crates/daemon/src/reconcile/zone/mod.rs

@@ -380,9 +380,9 @@ pub fn ip_reservation_to_network_status(ip: &IpReservation) -> ZoneNetworkStatus
     ZoneNetworkStatus {
         zone_ipv4: format!("{}/{}", ip.ipv4, ip.ipv4_prefix),
         zone_ipv6: format!("{}/{}", ip.ipv6, ip.ipv6_prefix),
-        zone_mac: ip.mac.to_string().replace('-', ":"),
+        zone_mac: ip.mac.to_string().to_lowercase().replace('-', ":"),
         gateway_ipv4: format!("{}/{}", ip.gateway_ipv4, ip.ipv4_prefix),
         gateway_ipv6: format!("{}/{}", ip.gateway_ipv6, ip.ipv6_prefix),
-        gateway_mac: ip.gateway_mac.to_string().replace('-', ":"),
+        gateway_mac: ip.gateway_mac.to_string().to_lowercase().replace('-', ":"),
     }
 }

crates/krata/proto/krata/v1/control.proto

@@ -45,6 +45,9 @@ message HostStatusReply {
     string host_uuid = 1;
     uint32 host_domid = 2;
     string krata_version = 3;
+    string host_ipv4 = 4;
+    string host_ipv6 = 5;
+    string host_mac = 6;
 }
 
 message CreateZoneRequest {

crates/network/src/backend.rs

@@ -127,7 +127,8 @@ impl NetworkBackend {
         let (tx_sender, tx_receiver) = channel::<BytesMut>(TX_CHANNEL_BUFFER_LEN);
         let mut udev = ChannelDevice::new(mtu, Medium::Ethernet, tx_sender.clone());
         let mac = self.metadata.gateway.mac;
-        let nat = Nat::new(mtu, proxy, mac, addresses.clone(), tx_sender.clone())?;
+        let local_cidrs = addresses.clone();
+        let nat = Nat::new(mtu, proxy, mac, local_cidrs, tx_sender.clone())?;
         let hardware_addr = HardwareAddress::Ethernet(mac);
         let config = Config::new(hardware_addr);
         let mut iface = Interface::new(config, &mut udev, Instant::now());

crates/network/src/hbridge.rs

@@ -1,21 +1,15 @@
-use std::{
+use std::{io::ErrorKind, net::IpAddr};
-    io::ErrorKind,
-    net::{IpAddr, Ipv4Addr},
-};
 
-use advmac::MacAddr6;
 use anyhow::{anyhow, Result};
 use bytes::BytesMut;
 use futures::TryStreamExt;
 use log::error;
-use smoltcp::wire::EthernetAddress;
+use smoltcp::wire::{EthernetAddress, Ipv4Cidr, Ipv6Cidr};
 use tokio::{select, task::JoinHandle};
 use tokio_tun::Tun;
 
 use crate::vbridge::{BridgeJoinHandle, VirtualBridge};
 
-const HOST_IPV4_ADDR: Ipv4Addr = Ipv4Addr::new(10, 75, 0, 1);
-
 #[derive(Debug)]
 enum HostBridgeProcessSelect {
     Send(Option<BytesMut>),
@@ -27,7 +21,14 @@ pub struct HostBridge {
 }
 
 impl HostBridge {
-    pub async fn new(mtu: usize, interface: String, bridge: &VirtualBridge) -> Result<HostBridge> {
+    pub async fn new(
+        mtu: usize,
+        interface: String,
+        bridge: &VirtualBridge,
+        ipv4: Ipv4Cidr,
+        ipv6: Ipv6Cidr,
+        mac: EthernetAddress,
+    ) -> Result<HostBridge> {
         let tun = Tun::builder()
             .name(&interface)
             .tap(true)
@@ -38,10 +39,6 @@ impl HostBridge {
         let (connection, handle, _) = rtnetlink::new_connection()?;
         tokio::spawn(connection);
 
-        let mut mac = MacAddr6::random();
-        mac.set_local(true);
-        mac.set_multicast(false);
-
         let mut links = handle.link().get().match_name(interface.clone()).execute();
         let link = links.try_next().await?;
         if link.is_none() {
@@ -54,25 +51,32 @@ impl HostBridge {
 
         handle
             .address()
-            .add(link.header.index, IpAddr::V4(HOST_IPV4_ADDR), 16)
+            .add(
+                link.header.index,
+                IpAddr::V4(ipv4.address().into()),
+                ipv4.prefix_len(),
+            )
             .execute()
             .await?;
 
         handle
             .address()
-            .add(link.header.index, IpAddr::V6(mac.to_link_local_ipv6()), 10)
+            .add(
+                link.header.index,
+                IpAddr::V6(ipv6.address().into()),
+                ipv6.prefix_len(),
+            )
             .execute()
             .await?;
 
         handle
             .link()
             .set(link.header.index)
-            .address(mac.to_array().to_vec())
+            .address(mac.0.to_vec())
             .up()
             .execute()
             .await?;
 
-        let mac = EthernetAddress(mac.to_array());
         let bridge_handle = bridge.join(mac).await?;
 
         let task = tokio::task::spawn(async move {

crates/network/src/lib.rs

@@ -1,17 +1,21 @@
-use std::{collections::HashMap, time::Duration};
+use std::{collections::HashMap, str::FromStr, time::Duration};
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use autonet::{AutoNetworkChangeset, AutoNetworkWatcher, NetworkMetadata};
 use futures::{future::join_all, TryFutureExt};
 use hbridge::HostBridge;
 use krata::{
     client::ControlClientProvider,
     dial::ControlDialAddress,
-    v1::{common::Zone, control::control_service_client::ControlServiceClient},
+    v1::{
+        common::Zone,
+        control::{control_service_client::ControlServiceClient, HostStatusRequest},
+    },
 };
 use log::warn;
+use smoltcp::wire::{EthernetAddress, Ipv4Cidr, Ipv6Cidr};
 use tokio::{task::JoinHandle, time::sleep};
-use tonic::transport::Channel;
+use tonic::{transport::Channel, Request};
 use uuid::Uuid;
 use vbridge::VirtualBridge;
 
@@ -41,10 +45,27 @@ pub struct NetworkService {
 
 impl NetworkService {
     pub async fn new(control_address: ControlDialAddress) -> Result<NetworkService> {
-        let control = ControlClientProvider::dial(control_address).await?;
+        let mut control = ControlClientProvider::dial(control_address).await?;
+        let host_status = control
+            .host_status(Request::new(HostStatusRequest {}))
+            .await?
+            .into_inner();
+        let host_ipv4 = Ipv4Cidr::from_str(&host_status.host_ipv4)
+            .map_err(|_| anyhow!("failed to parse host ipv4 cidr"))?;
+        let host_ipv6 = Ipv6Cidr::from_str(&host_status.host_ipv6)
+            .map_err(|_| anyhow!("failed to parse host ipv6 cidr"))?;
+        let host_mac = EthernetAddress::from_str(&host_status.host_mac)
+            .map_err(|_| anyhow!("failed to parse host mac address"))?;
         let bridge = VirtualBridge::new()?;
-        let hbridge =
+        let hbridge = HostBridge::new(
-            HostBridge::new(HOST_BRIDGE_MTU + EXTRA_MTU, "krata0".to_string(), &bridge).await?;
+            HOST_BRIDGE_MTU + EXTRA_MTU,
+            "krata0".to_string(),
+            &bridge,
+            host_ipv4,
+            host_ipv6,
+            host_mac,
+        )
+        .await?;
         Ok(NetworkService {
             control,
             zones: HashMap::new(),