fix(network): allocate host ip from allocation pool

This commit is contained in:
Alex Zenla 2024-08-22 15:24:18 -07:00
parent 1647a07226
commit f2c5060461
No known key found for this signature in database
GPG Key ID: 067B238899B51269
12 changed files with 140 additions and 63 deletions

View File

@ -17,6 +17,9 @@ impl HostStatusCommand {
println!("Host UUID: {}", response.host_uuid);
println!("Host Domain: {}", response.host_domid);
println!("Krata Version: {}", response.krata_version);
println!("Host IPv4: {}", response.host_ipv4);
println!("Host IPv6: {}", response.host_ipv6);
println!("Host Ethernet Address: {}", response.host_mac);
Ok(())
}
}

View File

@ -97,7 +97,7 @@ fn default_network_ipv4() -> DaemonIpv4NetworkConfig {
}
fn default_network_ipv4_subnet() -> String {
"10.75.80.0/24".to_string()
"10.75.0.0/16".to_string()
}
fn default_network_ipv6() -> DaemonIpv6NetworkConfig {

View File

@ -24,7 +24,7 @@ type BufferMap = Arc<Mutex<HashMap<u32, ConsoleBuffer>>>;
#[derive(Clone)]
pub struct DaemonConsoleHandle {
glt: ZoneLookupTable,
zlt: ZoneLookupTable,
listeners: ListenerMap,
buffers: BufferMap,
sender: Sender<(u32, Vec<u8>)>,
@ -57,7 +57,7 @@ impl DaemonConsoleHandle {
uuid: Uuid,
sender: Sender<Vec<u8>>,
) -> Result<DaemonConsoleAttachHandle> {
let Some(domid) = self.glt.lookup_domid_by_uuid(&uuid).await else {
let Some(domid) = self.zlt.lookup_domid_by_uuid(&uuid).await else {
return Err(anyhow!("unable to find domain {}", uuid));
};
let buffers = self.buffers.lock().await;
@ -84,7 +84,7 @@ impl Drop for DaemonConsoleHandle {
}
pub struct DaemonConsole {
glt: ZoneLookupTable,
zlt: ZoneLookupTable,
listeners: ListenerMap,
buffers: BufferMap,
receiver: Receiver<(u32, Option<Vec<u8>>)>,
@ -93,14 +93,14 @@ pub struct DaemonConsole {
}
impl DaemonConsole {
pub async fn new(glt: ZoneLookupTable) -> Result<DaemonConsole> {
pub async fn new(zlt: ZoneLookupTable) -> Result<DaemonConsole> {
let (service, sender, receiver) =
ChannelService::new("krata-console".to_string(), Some(0)).await?;
let task = service.launch().await?;
let listeners = Arc::new(Mutex::new(HashMap::new()));
let buffers = Arc::new(Mutex::new(HashMap::new()));
Ok(DaemonConsole {
glt,
zlt,
listeners,
buffers,
receiver,
@ -110,7 +110,7 @@ impl DaemonConsole {
}
pub async fn launch(mut self) -> Result<DaemonConsoleHandle> {
let glt = self.glt.clone();
let zlt = self.zlt.clone();
let listeners = self.listeners.clone();
let buffers = self.buffers.clone();
let sender = self.sender.clone();
@ -120,7 +120,7 @@ impl DaemonConsole {
}
});
Ok(DaemonConsoleHandle {
glt,
zlt,
listeners,
buffers,
sender,

View File

@ -1,4 +1,5 @@
use crate::db::zone::ZoneStore;
use crate::ip::assignment::IpAssignment;
use crate::{
command::DaemonCommand, console::DaemonConsoleHandle, devices::DaemonDeviceManager,
event::DaemonEventContext, idm::DaemonIdmHandle, metrics::idm_metric_to_api,
@ -68,12 +69,13 @@ impl From<ApiError> for Status {
#[derive(Clone)]
pub struct DaemonControlService {
glt: ZoneLookupTable,
zlt: ZoneLookupTable,
devices: DaemonDeviceManager,
events: DaemonEventContext,
console: DaemonConsoleHandle,
idm: DaemonIdmHandle,
zones: ZoneStore,
ip: IpAssignment,
zone_reconciler_notify: Sender<Uuid>,
packer: OciPackerService,
runtime: Runtime,
@ -82,23 +84,25 @@ pub struct DaemonControlService {
impl DaemonControlService {
#[allow(clippy::too_many_arguments)]
pub fn new(
glt: ZoneLookupTable,
zlt: ZoneLookupTable,
devices: DaemonDeviceManager,
events: DaemonEventContext,
console: DaemonConsoleHandle,
idm: DaemonIdmHandle,
zones: ZoneStore,
ip: IpAssignment,
zone_reconciler_notify: Sender<Uuid>,
packer: OciPackerService,
runtime: Runtime,
) -> Self {
Self {
glt,
zlt,
devices,
events,
console,
idm,
zones,
ip,
zone_reconciler_notify,
packer,
runtime,
@ -138,10 +142,29 @@ impl ControlService for DaemonControlService {
request: Request<HostStatusRequest>,
) -> Result<Response<HostStatusReply>, Status> {
let _ = request.into_inner();
let host_reservation =
self.ip
.retrieve(self.zlt.host_uuid())
.await
.map_err(|x| ApiError {
message: x.to_string(),
})?;
Ok(Response::new(HostStatusReply {
host_domid: self.glt.host_domid(),
host_uuid: self.glt.host_uuid().to_string(),
host_domid: self.zlt.host_domid(),
host_uuid: self.zlt.host_uuid().to_string(),
krata_version: DaemonCommand::version(),
host_ipv4: host_reservation
.as_ref()
.map(|x| format!("{}/{}", x.ipv4, x.ipv4_prefix))
.unwrap_or_default(),
host_ipv6: host_reservation
.as_ref()
.map(|x| format!("{}/{}", x.ipv6, x.ipv6_prefix))
.unwrap_or_default(),
host_mac: host_reservation
.as_ref()
.map(|x| x.mac.to_string().to_lowercase().replace('-', ":"))
.unwrap_or_default(),
}))
}
@ -168,7 +191,7 @@ impl ControlService for DaemonControlService {
exit_status: None,
error_status: None,
resource_status: None,
host: self.glt.host_uuid().to_string(),
host: self.zlt.host_uuid().to_string(),
domid: u32::MAX,
}),
spec: Some(spec),
@ -531,13 +554,13 @@ impl ControlService for DaemonControlService {
) -> Result<Response<Self::SnoopIdmStream>, Status> {
let _ = request.into_inner();
let mut messages = self.idm.snoop();
let glt = self.glt.clone();
let zlt = self.zlt.clone();
let output = try_stream! {
while let Ok(event) = messages.recv().await {
let Some(from_uuid) = glt.lookup_uuid_by_domid(event.from).await else {
let Some(from_uuid) = zlt.lookup_uuid_by_domid(event.from).await else {
continue;
};
let Some(to_uuid) = glt.lookup_uuid_by_domid(event.to).await else {
let Some(to_uuid) = zlt.lookup_uuid_by_domid(event.to).await else {
continue;
};
yield SnoopIdmReply { from: from_uuid.to_string(), to: to_uuid.to_string(), packet: Some(event.packet) };

View File

@ -31,7 +31,7 @@ type ClientMap = Arc<Mutex<HashMap<u32, IdmInternalClient>>>;
#[derive(Clone)]
pub struct DaemonIdmHandle {
glt: ZoneLookupTable,
zlt: ZoneLookupTable,
clients: ClientMap,
feeds: BackendFeedMap,
tx_sender: Sender<(u32, IdmTransportPacket)>,
@ -45,7 +45,7 @@ impl DaemonIdmHandle {
}
pub async fn client(&self, uuid: Uuid) -> Result<IdmInternalClient> {
let Some(domid) = self.glt.lookup_domid_by_uuid(&uuid).await else {
let Some(domid) = self.zlt.lookup_domid_by_uuid(&uuid).await else {
return Err(anyhow!("unable to find domain {}", uuid));
};
self.client_by_domid(domid).await
@ -72,7 +72,7 @@ pub struct DaemonIdmSnoopPacket {
}
pub struct DaemonIdm {
glt: ZoneLookupTable,
zlt: ZoneLookupTable,
clients: ClientMap,
feeds: BackendFeedMap,
tx_sender: Sender<(u32, IdmTransportPacket)>,
@ -84,7 +84,7 @@ pub struct DaemonIdm {
}
impl DaemonIdm {
pub async fn new(glt: ZoneLookupTable) -> Result<DaemonIdm> {
pub async fn new(zlt: ZoneLookupTable) -> Result<DaemonIdm> {
debug!("allocating channel service for idm");
let (service, tx_raw_sender, rx_receiver) =
ChannelService::new("krata-channel".to_string(), None).await?;
@ -98,7 +98,7 @@ impl DaemonIdm {
let feeds = Arc::new(Mutex::new(HashMap::new()));
Ok(DaemonIdm {
glt,
zlt,
rx_receiver,
tx_receiver,
tx_sender,
@ -111,7 +111,7 @@ impl DaemonIdm {
}
pub async fn launch(mut self) -> Result<DaemonIdmHandle> {
let glt = self.glt.clone();
let zlt = self.zlt.clone();
let clients = self.clients.clone();
let feeds = self.feeds.clone();
let tx_sender = self.tx_sender.clone();
@ -124,7 +124,7 @@ impl DaemonIdm {
}
});
Ok(DaemonIdmHandle {
glt,
zlt,
clients,
feeds,
tx_sender,

View File

@ -36,13 +36,13 @@ impl IpAssignment {
store: IpReservationStore,
) -> Result<Self> {
let mut state = IpAssignment::fetch_current_state(&store).await?;
let reservation = if let Some(reservation) = store.read(host_uuid).await? {
let gateway_reservation = if let Some(reservation) = store.read(Uuid::nil()).await? {
reservation
} else {
IpAssignment::allocate(
&mut state,
&store,
host_uuid,
Uuid::nil(),
ipv4_network,
ipv6_network,
None,
@ -51,12 +51,27 @@ impl IpAssignment {
)
.await?
};
if store.read(host_uuid).await?.is_none() {
let _ = IpAssignment::allocate(
&mut state,
&store,
host_uuid,
ipv4_network,
ipv6_network,
Some(gateway_reservation.gateway_ipv4),
Some(gateway_reservation.gateway_ipv6),
Some(gateway_reservation.gateway_mac),
)
.await?;
}
let assignment = IpAssignment {
ipv4_network,
ipv6_network,
gateway_ipv4: reservation.ipv4,
gateway_ipv6: reservation.ipv6,
gateway_mac: reservation.gateway_mac,
gateway_ipv4: gateway_reservation.ipv4,
gateway_ipv6: gateway_reservation.ipv6,
gateway_mac: gateway_reservation.mac,
store,
state: Arc::new(RwLock::new(state)),
};
@ -92,13 +107,17 @@ impl IpAssignment {
.filter(|ip| {
let last = ip.octets()[3];
// filter for IPs ending in .1 to .250 because .250+ can have special meaning
last > 0 && last < 250
(1..250).contains(&last)
})
.find(|ip| !state.ipv4.contains_key(ip));
let found_ipv6: Option<Ipv6Addr> = ipv6_network
.iter()
.filter(|ip| !ip.is_loopback() && !ip.is_multicast())
.filter(|ip| {
let last = ip.octets()[15];
last > 0
})
.find(|ip| !state.ipv6.contains_key(ip));
let Some(ipv4) = found_ipv4 else {
@ -114,7 +133,7 @@ impl IpAssignment {
};
let mut mac = MacAddr6::random();
mac.set_local(false);
mac.set_local(true);
mac.set_multicast(false);
let reservation = IpReservation {

View File

@ -45,9 +45,10 @@ pub mod zlt;
pub struct Daemon {
store: String,
_config: Arc<DaemonConfig>,
glt: ZoneLookupTable,
zlt: ZoneLookupTable,
devices: DaemonDeviceManager,
zones: ZoneStore,
ip: IpAssignment,
events: DaemonEventContext,
zone_reconciler_task: JoinHandle<()>,
zone_reconciler_notify: Sender<Uuid>,
@ -127,7 +128,7 @@ impl Daemon {
let ipv4_network = Ipv4Network::from_str(&config.network.ipv4.subnet)?;
let ipv6_network = Ipv6Network::from_str(&config.network.ipv6.subnet)?;
let ip_reservation_store = IpReservationStore::open(database)?;
let ip_assignment =
let ip =
IpAssignment::new(host_uuid, ipv4_network, ipv6_network, ip_reservation_store).await?;
debug!("initializing zone reconciler");
let zone_reconciler = ZoneReconciler::new(
@ -141,7 +142,7 @@ impl Daemon {
kernel_path,
initrd_path,
addons_path,
ip_assignment,
ip.clone(),
config.clone(),
)?;
@ -161,9 +162,10 @@ impl Daemon {
Ok(Self {
store,
_config: config,
glt: zlt,
zlt,
devices,
zones,
ip,
events,
zone_reconciler_task,
zone_reconciler_notify,
@ -178,12 +180,13 @@ impl Daemon {
pub async fn listen(&mut self, addr: ControlDialAddress) -> Result<()> {
debug!("starting control service");
let control_service = DaemonControlService::new(
self.glt.clone(),
self.zlt.clone(),
self.devices.clone(),
self.events.clone(),
self.console.clone(),
self.idm.clone(),
self.zones.clone(),
self.ip.clone(),
self.zone_reconciler_notify.clone(),
self.packer.clone(),
self.runtime.clone(),

View File

@ -380,9 +380,9 @@ pub fn ip_reservation_to_network_status(ip: &IpReservation) -> ZoneNetworkStatus
ZoneNetworkStatus {
zone_ipv4: format!("{}/{}", ip.ipv4, ip.ipv4_prefix),
zone_ipv6: format!("{}/{}", ip.ipv6, ip.ipv6_prefix),
zone_mac: ip.mac.to_string().replace('-', ":"),
zone_mac: ip.mac.to_string().to_lowercase().replace('-', ":"),
gateway_ipv4: format!("{}/{}", ip.gateway_ipv4, ip.ipv4_prefix),
gateway_ipv6: format!("{}/{}", ip.gateway_ipv6, ip.ipv6_prefix),
gateway_mac: ip.gateway_mac.to_string().replace('-', ":"),
gateway_mac: ip.gateway_mac.to_string().to_lowercase().replace('-', ":"),
}
}

View File

@ -45,6 +45,9 @@ message HostStatusReply {
string host_uuid = 1;
uint32 host_domid = 2;
string krata_version = 3;
string host_ipv4 = 4;
string host_ipv6 = 5;
string host_mac = 6;
}
message CreateZoneRequest {

View File

@ -127,7 +127,8 @@ impl NetworkBackend {
let (tx_sender, tx_receiver) = channel::<BytesMut>(TX_CHANNEL_BUFFER_LEN);
let mut udev = ChannelDevice::new(mtu, Medium::Ethernet, tx_sender.clone());
let mac = self.metadata.gateway.mac;
let nat = Nat::new(mtu, proxy, mac, addresses.clone(), tx_sender.clone())?;
let local_cidrs = addresses.clone();
let nat = Nat::new(mtu, proxy, mac, local_cidrs, tx_sender.clone())?;
let hardware_addr = HardwareAddress::Ethernet(mac);
let config = Config::new(hardware_addr);
let mut iface = Interface::new(config, &mut udev, Instant::now());

View File

@ -1,21 +1,15 @@
use std::{
io::ErrorKind,
net::{IpAddr, Ipv4Addr},
};
use std::{io::ErrorKind, net::IpAddr};
use advmac::MacAddr6;
use anyhow::{anyhow, Result};
use bytes::BytesMut;
use futures::TryStreamExt;
use log::error;
use smoltcp::wire::EthernetAddress;
use smoltcp::wire::{EthernetAddress, Ipv4Cidr, Ipv6Cidr};
use tokio::{select, task::JoinHandle};
use tokio_tun::Tun;
use crate::vbridge::{BridgeJoinHandle, VirtualBridge};
const HOST_IPV4_ADDR: Ipv4Addr = Ipv4Addr::new(10, 75, 0, 1);
#[derive(Debug)]
enum HostBridgeProcessSelect {
Send(Option<BytesMut>),
@ -27,7 +21,14 @@ pub struct HostBridge {
}
impl HostBridge {
pub async fn new(mtu: usize, interface: String, bridge: &VirtualBridge) -> Result<HostBridge> {
pub async fn new(
mtu: usize,
interface: String,
bridge: &VirtualBridge,
ipv4: Ipv4Cidr,
ipv6: Ipv6Cidr,
mac: EthernetAddress,
) -> Result<HostBridge> {
let tun = Tun::builder()
.name(&interface)
.tap(true)
@ -38,10 +39,6 @@ impl HostBridge {
let (connection, handle, _) = rtnetlink::new_connection()?;
tokio::spawn(connection);
let mut mac = MacAddr6::random();
mac.set_local(true);
mac.set_multicast(false);
let mut links = handle.link().get().match_name(interface.clone()).execute();
let link = links.try_next().await?;
if link.is_none() {
@ -54,25 +51,32 @@ impl HostBridge {
handle
.address()
.add(link.header.index, IpAddr::V4(HOST_IPV4_ADDR), 16)
.add(
link.header.index,
IpAddr::V4(ipv4.address().into()),
ipv4.prefix_len(),
)
.execute()
.await?;
handle
.address()
.add(link.header.index, IpAddr::V6(mac.to_link_local_ipv6()), 10)
.add(
link.header.index,
IpAddr::V6(ipv6.address().into()),
ipv6.prefix_len(),
)
.execute()
.await?;
handle
.link()
.set(link.header.index)
.address(mac.to_array().to_vec())
.address(mac.0.to_vec())
.up()
.execute()
.await?;
let mac = EthernetAddress(mac.to_array());
let bridge_handle = bridge.join(mac).await?;
let task = tokio::task::spawn(async move {

View File

@ -1,17 +1,21 @@
use std::{collections::HashMap, time::Duration};
use std::{collections::HashMap, str::FromStr, time::Duration};
use anyhow::Result;
use anyhow::{anyhow, Result};
use autonet::{AutoNetworkChangeset, AutoNetworkWatcher, NetworkMetadata};
use futures::{future::join_all, TryFutureExt};
use hbridge::HostBridge;
use krata::{
client::ControlClientProvider,
dial::ControlDialAddress,
v1::{common::Zone, control::control_service_client::ControlServiceClient},
v1::{
common::Zone,
control::{control_service_client::ControlServiceClient, HostStatusRequest},
},
};
use log::warn;
use smoltcp::wire::{EthernetAddress, Ipv4Cidr, Ipv6Cidr};
use tokio::{task::JoinHandle, time::sleep};
use tonic::transport::Channel;
use tonic::{transport::Channel, Request};
use uuid::Uuid;
use vbridge::VirtualBridge;
@ -41,10 +45,27 @@ pub struct NetworkService {
impl NetworkService {
pub async fn new(control_address: ControlDialAddress) -> Result<NetworkService> {
let control = ControlClientProvider::dial(control_address).await?;
let mut control = ControlClientProvider::dial(control_address).await?;
let host_status = control
.host_status(Request::new(HostStatusRequest {}))
.await?
.into_inner();
let host_ipv4 = Ipv4Cidr::from_str(&host_status.host_ipv4)
.map_err(|_| anyhow!("failed to parse host ipv4 cidr"))?;
let host_ipv6 = Ipv6Cidr::from_str(&host_status.host_ipv6)
.map_err(|_| anyhow!("failed to parse host ipv6 cidr"))?;
let host_mac = EthernetAddress::from_str(&host_status.host_mac)
.map_err(|_| anyhow!("failed to parse host mac address"))?;
let bridge = VirtualBridge::new()?;
let hbridge =
HostBridge::new(HOST_BRIDGE_MTU + EXTRA_MTU, "krata0".to_string(), &bridge).await?;
let hbridge = HostBridge::new(
HOST_BRIDGE_MTU + EXTRA_MTU,
"krata0".to_string(),
&bridge,
host_ipv4,
host_ipv6,
host_mac,
)
.await?;
Ok(NetworkService {
control,
zones: HashMap::new(),