feat: fast guest lookup table and host identification

This commit is contained in:
Alex Zenla
2024-04-22 02:13:01 +00:00
parent 4e0d843de7
commit fd0abae5a9
14 changed files with 283 additions and 96 deletions

View File

@ -0,0 +1,22 @@
use anyhow::Result;
use clap::Parser;
use krata::v1::control::{control_service_client::ControlServiceClient, IdentifyHostRequest};
use tonic::{transport::Channel, Request};
#[derive(Parser)]
#[command(about = "Identify information about the host")]
pub struct IdentifyHostCommand {}
impl IdentifyHostCommand {
pub async fn run(self, mut client: ControlServiceClient<Channel>) -> Result<()> {
let response = client
.identify_host(Request::new(IdentifyHostRequest {}))
.await?
.into_inner();
println!("Host UUID: {}", response.host_uuid);
println!("Host Domain: {}", response.host_domid);
println!("Krata Version: {}", response.krata_version);
Ok(())
}
}

View File

@ -83,8 +83,8 @@ impl IdmSnoopCommand {
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
pub struct IdmSnoopLine { pub struct IdmSnoopLine {
pub from: u32, pub from: String,
pub to: u32, pub to: String,
pub packet: IdmSnoopData, pub packet: IdmSnoopData,
} }

View File

@ -1,5 +1,6 @@
pub mod attach; pub mod attach;
pub mod destroy; pub mod destroy;
pub mod identify_host;
pub mod idm_snoop; pub mod idm_snoop;
pub mod launch; pub mod launch;
pub mod list; pub mod list;
@ -20,9 +21,10 @@ use krata::{
use tonic::{transport::Channel, Request}; use tonic::{transport::Channel, Request};
use self::{ use self::{
attach::AttachCommand, destroy::DestroyCommand, idm_snoop::IdmSnoopCommand, attach::AttachCommand, destroy::DestroyCommand, identify_host::IdentifyHostCommand,
launch::LauchCommand, list::ListCommand, logs::LogsCommand, metrics::MetricsCommand, idm_snoop::IdmSnoopCommand, launch::LauchCommand, list::ListCommand, logs::LogsCommand,
pull::PullCommand, resolve::ResolveCommand, top::TopCommand, watch::WatchCommand, metrics::MetricsCommand, pull::PullCommand, resolve::ResolveCommand, top::TopCommand,
watch::WatchCommand,
}; };
#[derive(Parser)] #[derive(Parser)]
@ -56,6 +58,7 @@ pub enum Commands {
Metrics(MetricsCommand), Metrics(MetricsCommand),
IdmSnoop(IdmSnoopCommand), IdmSnoop(IdmSnoopCommand),
Top(TopCommand), Top(TopCommand),
IdentifyHost(IdentifyHostCommand),
} }
impl ControlCommand { impl ControlCommand {
@ -107,6 +110,10 @@ impl ControlCommand {
Commands::Pull(pull) => { Commands::Pull(pull) => {
pull.run(client).await?; pull.run(client).await?;
} }
Commands::IdentifyHost(identify) => {
identify.run(client).await?;
}
} }
Ok(()) Ok(())
} }

View File

@ -1,21 +1,9 @@
use anyhow::Result; use anyhow::Result;
use clap::Parser; use clap::Parser;
use env_logger::Env; use env_logger::Env;
use krata::dial::ControlDialAddress; use kratad::command::DaemonCommand;
use kratad::Daemon;
use log::LevelFilter; use log::LevelFilter;
use std::{ use std::sync::{atomic::AtomicBool, Arc};
str::FromStr,
sync::{atomic::AtomicBool, Arc},
};
#[derive(Parser)]
struct DaemonCommand {
#[arg(short, long, default_value = "unix:///var/lib/krata/daemon.socket")]
listen: String,
#[arg(short, long, default_value = "/var/lib/krata")]
store: String,
}
#[tokio::main(flavor = "multi_thread", worker_threads = 10)] #[tokio::main(flavor = "multi_thread", worker_threads = 10)]
async fn main() -> Result<()> { async fn main() -> Result<()> {
@ -24,12 +12,8 @@ async fn main() -> Result<()> {
.init(); .init();
mask_sighup()?; mask_sighup()?;
let args = DaemonCommand::parse(); let command = DaemonCommand::parse();
let addr = ControlDialAddress::from_str(&args.listen)?; command.run().await
let mut daemon = Daemon::new(args.store.clone()).await?;
daemon.listen(addr).await?;
Ok(())
} }
fn mask_sighup() -> Result<()> { fn mask_sighup() -> Result<()> {

View File

@ -0,0 +1,36 @@
use anyhow::Result;
use clap::{CommandFactory, Parser};
use krata::dial::ControlDialAddress;
use std::str::FromStr;
use crate::Daemon;
#[derive(Parser)]
#[command(version, about = "Krata hypervisor daemon")]
pub struct DaemonCommand {
#[arg(
short,
long,
default_value = "unix:///var/lib/krata/daemon.socket",
help = "Listen address"
)]
listen: String,
#[arg(short, long, default_value = "/var/lib/krata", help = "Storage path")]
store: String,
}
impl DaemonCommand {
pub async fn run(self) -> Result<()> {
let addr = ControlDialAddress::from_str(&self.listen)?;
let mut daemon = Daemon::new(self.store.clone()).await?;
daemon.listen(addr).await?;
Ok(())
}
pub fn version() -> String {
DaemonCommand::command()
.get_version()
.unwrap_or("unknown")
.to_string()
}
}

View File

@ -1,6 +1,6 @@
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use anyhow::Result; use anyhow::{anyhow, Result};
use circular_buffer::CircularBuffer; use circular_buffer::CircularBuffer;
use kratart::channel::ChannelService; use kratart::channel::ChannelService;
use log::error; use log::error;
@ -11,6 +11,9 @@ use tokio::{
}, },
task::JoinHandle, task::JoinHandle,
}; };
use uuid::Uuid;
use crate::glt::GuestLookupTable;
const CONSOLE_BUFFER_SIZE: usize = 1024 * 1024; const CONSOLE_BUFFER_SIZE: usize = 1024 * 1024;
type RawConsoleBuffer = CircularBuffer<CONSOLE_BUFFER_SIZE, u8>; type RawConsoleBuffer = CircularBuffer<CONSOLE_BUFFER_SIZE, u8>;
@ -21,6 +24,7 @@ type BufferMap = Arc<Mutex<HashMap<u32, ConsoleBuffer>>>;
#[derive(Clone)] #[derive(Clone)]
pub struct DaemonConsoleHandle { pub struct DaemonConsoleHandle {
glt: GuestLookupTable,
listeners: ListenerMap, listeners: ListenerMap,
buffers: BufferMap, buffers: BufferMap,
sender: Sender<(u32, Vec<u8>)>, sender: Sender<(u32, Vec<u8>)>,
@ -50,9 +54,12 @@ impl DaemonConsoleAttachHandle {
impl DaemonConsoleHandle { impl DaemonConsoleHandle {
pub async fn attach( pub async fn attach(
&self, &self,
domid: u32, uuid: Uuid,
sender: Sender<Vec<u8>>, sender: Sender<Vec<u8>>,
) -> Result<DaemonConsoleAttachHandle> { ) -> Result<DaemonConsoleAttachHandle> {
let Some(domid) = self.glt.lookup_domid_by_uuid(&uuid).await else {
return Err(anyhow!("unable to find domain {}", uuid));
};
let buffers = self.buffers.lock().await; let buffers = self.buffers.lock().await;
let buffer = buffers.get(&domid).map(|x| x.to_vec()).unwrap_or_default(); let buffer = buffers.get(&domid).map(|x| x.to_vec()).unwrap_or_default();
drop(buffers); drop(buffers);
@ -77,6 +84,7 @@ impl Drop for DaemonConsoleHandle {
} }
pub struct DaemonConsole { pub struct DaemonConsole {
glt: GuestLookupTable,
listeners: ListenerMap, listeners: ListenerMap,
buffers: BufferMap, buffers: BufferMap,
receiver: Receiver<(u32, Option<Vec<u8>>)>, receiver: Receiver<(u32, Option<Vec<u8>>)>,
@ -85,13 +93,14 @@ pub struct DaemonConsole {
} }
impl DaemonConsole { impl DaemonConsole {
pub async fn new() -> Result<DaemonConsole> { pub async fn new(glt: GuestLookupTable) -> Result<DaemonConsole> {
let (service, sender, receiver) = let (service, sender, receiver) =
ChannelService::new("krata-console".to_string(), Some(0)).await?; ChannelService::new("krata-console".to_string(), Some(0)).await?;
let task = service.launch().await?; let task = service.launch().await?;
let listeners = Arc::new(Mutex::new(HashMap::new())); let listeners = Arc::new(Mutex::new(HashMap::new()));
let buffers = Arc::new(Mutex::new(HashMap::new())); let buffers = Arc::new(Mutex::new(HashMap::new()));
Ok(DaemonConsole { Ok(DaemonConsole {
glt,
listeners, listeners,
buffers, buffers,
receiver, receiver,
@ -101,6 +110,7 @@ impl DaemonConsole {
} }
pub async fn launch(mut self) -> Result<DaemonConsoleHandle> { pub async fn launch(mut self) -> Result<DaemonConsoleHandle> {
let glt = self.glt.clone();
let listeners = self.listeners.clone(); let listeners = self.listeners.clone();
let buffers = self.buffers.clone(); let buffers = self.buffers.clone();
let sender = self.sender.clone(); let sender = self.sender.clone();
@ -110,6 +120,7 @@ impl DaemonConsole {
} }
}); });
Ok(DaemonConsoleHandle { Ok(DaemonConsoleHandle {
glt,
listeners, listeners,
buffers, buffers,
sender, sender,

View File

@ -10,9 +10,10 @@ use krata::{
control::{ control::{
control_service_server::ControlService, ConsoleDataReply, ConsoleDataRequest, control_service_server::ControlService, ConsoleDataReply, ConsoleDataRequest,
CreateGuestReply, CreateGuestRequest, DestroyGuestReply, DestroyGuestRequest, CreateGuestReply, CreateGuestRequest, DestroyGuestReply, DestroyGuestRequest,
ListGuestsReply, ListGuestsRequest, PullImageReply, PullImageRequest, IdentifyHostReply, IdentifyHostRequest, ListGuestsReply, ListGuestsRequest,
ReadGuestMetricsReply, ReadGuestMetricsRequest, ResolveGuestReply, ResolveGuestRequest, PullImageReply, PullImageRequest, ReadGuestMetricsReply, ReadGuestMetricsRequest,
SnoopIdmReply, SnoopIdmRequest, WatchEventsReply, WatchEventsRequest, ResolveGuestReply, ResolveGuestRequest, SnoopIdmReply, SnoopIdmRequest,
WatchEventsReply, WatchEventsRequest,
}, },
}, },
}; };
@ -32,7 +33,8 @@ use tonic::{Request, Response, Status, Streaming};
use uuid::Uuid; use uuid::Uuid;
use crate::{ use crate::{
console::DaemonConsoleHandle, db::GuestStore, event::DaemonEventContext, idm::DaemonIdmHandle, command::DaemonCommand, console::DaemonConsoleHandle, db::GuestStore,
event::DaemonEventContext, glt::GuestLookupTable, idm::DaemonIdmHandle,
metrics::idm_metric_to_api, oci::convert_oci_progress, metrics::idm_metric_to_api, oci::convert_oci_progress,
}; };
@ -56,6 +58,7 @@ impl From<ApiError> for Status {
#[derive(Clone)] #[derive(Clone)]
pub struct DaemonControlService { pub struct DaemonControlService {
glt: GuestLookupTable,
events: DaemonEventContext, events: DaemonEventContext,
console: DaemonConsoleHandle, console: DaemonConsoleHandle,
idm: DaemonIdmHandle, idm: DaemonIdmHandle,
@ -66,6 +69,7 @@ pub struct DaemonControlService {
impl DaemonControlService { impl DaemonControlService {
pub fn new( pub fn new(
glt: GuestLookupTable,
events: DaemonEventContext, events: DaemonEventContext,
console: DaemonConsoleHandle, console: DaemonConsoleHandle,
idm: DaemonIdmHandle, idm: DaemonIdmHandle,
@ -74,6 +78,7 @@ impl DaemonControlService {
packer: OciPackerService, packer: OciPackerService,
) -> Self { ) -> Self {
Self { Self {
glt,
events, events,
console, console,
idm, idm,
@ -108,6 +113,18 @@ impl ControlService for DaemonControlService {
type SnoopIdmStream = type SnoopIdmStream =
Pin<Box<dyn Stream<Item = Result<SnoopIdmReply, Status>> + Send + 'static>>; Pin<Box<dyn Stream<Item = Result<SnoopIdmReply, Status>> + Send + 'static>>;
async fn identify_host(
&self,
request: Request<IdentifyHostRequest>,
) -> Result<Response<IdentifyHostReply>, Status> {
let _ = request.into_inner();
Ok(Response::new(IdentifyHostReply {
host_domid: self.glt.host_domid(),
host_uuid: self.glt.host_uuid().to_string(),
krata_version: DaemonCommand::version(),
}))
}
async fn create_guest( async fn create_guest(
&self, &self,
request: Request<CreateGuestRequest>, request: Request<CreateGuestRequest>,
@ -130,6 +147,7 @@ impl ControlService for DaemonControlService {
network: None, network: None,
exit_info: None, exit_info: None,
error_info: None, error_info: None,
host: self.glt.host_uuid().to_string(),
domid: u32::MAX, domid: u32::MAX,
}), }),
spec: Some(spec), spec: Some(spec),
@ -230,36 +248,10 @@ impl ControlService for DaemonControlService {
let uuid = Uuid::from_str(&request.guest_id).map_err(|error| ApiError { let uuid = Uuid::from_str(&request.guest_id).map_err(|error| ApiError {
message: error.to_string(), message: error.to_string(),
})?; })?;
let guest = self
.guests
.read(uuid)
.await
.map_err(|error| ApiError {
message: error.to_string(),
})?
.ok_or_else(|| ApiError {
message: "guest did not exist in the database".to_string(),
})?;
let Some(ref state) = guest.state else {
return Err(ApiError {
message: "guest did not have state".to_string(),
}
.into());
};
let domid = state.domid;
if domid == 0 {
return Err(ApiError {
message: "invalid domid on the guest".to_string(),
}
.into());
}
let (sender, mut receiver) = channel(100); let (sender, mut receiver) = channel(100);
let console = self let console = self
.console .console
.attach(domid, sender) .attach(uuid, sender)
.await .await
.map_err(|error| ApiError { .map_err(|error| ApiError {
message: format!("failed to attach to console: {}", error), message: format!("failed to attach to console: {}", error),
@ -309,33 +301,7 @@ impl ControlService for DaemonControlService {
let uuid = Uuid::from_str(&request.guest_id).map_err(|error| ApiError { let uuid = Uuid::from_str(&request.guest_id).map_err(|error| ApiError {
message: error.to_string(), message: error.to_string(),
})?; })?;
let guest = self let client = self.idm.client(uuid).await.map_err(|error| ApiError {
.guests
.read(uuid)
.await
.map_err(|error| ApiError {
message: error.to_string(),
})?
.ok_or_else(|| ApiError {
message: "guest did not exist in the database".to_string(),
})?;
let Some(ref state) = guest.state else {
return Err(ApiError {
message: "guest did not have state".to_string(),
}
.into());
};
let domid = state.domid;
if domid == 0 {
return Err(ApiError {
message: "invalid domid on the guest".to_string(),
}
.into());
}
let client = self.idm.client(domid).await.map_err(|error| ApiError {
message: error.to_string(), message: error.to_string(),
})?; })?;
@ -448,9 +414,16 @@ impl ControlService for DaemonControlService {
) -> Result<Response<Self::SnoopIdmStream>, Status> { ) -> Result<Response<Self::SnoopIdmStream>, Status> {
let _ = request.into_inner(); let _ = request.into_inner();
let mut messages = self.idm.snoop(); let mut messages = self.idm.snoop();
let glt = self.glt.clone();
let output = try_stream! { let output = try_stream! {
while let Ok(event) = messages.recv().await { while let Ok(event) = messages.recv().await {
yield SnoopIdmReply { from: event.from, to: event.to, packet: Some(event.packet) }; let Some(from_uuid) = glt.lookup_uuid_by_domid(event.from).await else {
continue;
};
let Some(to_uuid) = glt.lookup_uuid_by_domid(event.to).await else {
continue;
};
yield SnoopIdmReply { from: from_uuid.to_string(), to: to_uuid.to_string(), packet: Some(event.packet) };
} }
}; };
Ok(Response::new(Box::pin(output) as Self::SnoopIdmStream)) Ok(Response::new(Box::pin(output) as Self::SnoopIdmStream))

View File

@ -93,7 +93,7 @@ impl DaemonEventGenerator {
match status { match status {
GuestStatus::Started => { GuestStatus::Started => {
if let Entry::Vacant(e) = self.idms.entry(domid) { if let Entry::Vacant(e) = self.idms.entry(domid) {
let client = self.idm.client(domid).await?; let client = self.idm.client_by_domid(domid).await?;
let mut receiver = client.subscribe().await?; let mut receiver = client.subscribe().await?;
let sender = self.idm_sender.clone(); let sender = self.idm_sender.clone();
let task = tokio::task::spawn(async move { let task = tokio::task::spawn(async move {
@ -136,6 +136,7 @@ impl DaemonEventGenerator {
network: guest.state.clone().unwrap_or_default().network, network: guest.state.clone().unwrap_or_default().network,
exit_info: Some(GuestExitInfo { code }), exit_info: Some(GuestExitInfo { code }),
error_info: None, error_info: None,
host: guest.state.clone().map(|x| x.host).unwrap_or_default(),
domid: guest.state.clone().map(|x| x.domid).unwrap_or(u32::MAX), domid: guest.state.clone().map(|x| x.domid).unwrap_or(u32::MAX),
}); });

69
crates/daemon/src/glt.rs Normal file
View File

@ -0,0 +1,69 @@
use std::{collections::HashMap, sync::Arc};
use tokio::sync::RwLock;
use uuid::Uuid;
struct GuestLookupTableState {
domid_to_uuid: HashMap<u32, Uuid>,
uuid_to_domid: HashMap<Uuid, u32>,
}
impl GuestLookupTableState {
pub fn new(host_uuid: Uuid) -> Self {
let mut domid_to_uuid = HashMap::new();
let mut uuid_to_domid = HashMap::new();
domid_to_uuid.insert(0, host_uuid);
uuid_to_domid.insert(host_uuid, 0);
GuestLookupTableState {
domid_to_uuid,
uuid_to_domid,
}
}
}
#[derive(Clone)]
pub struct GuestLookupTable {
host_domid: u32,
host_uuid: Uuid,
state: Arc<RwLock<GuestLookupTableState>>,
}
impl GuestLookupTable {
pub fn new(host_domid: u32, host_uuid: Uuid) -> Self {
GuestLookupTable {
host_domid,
host_uuid,
state: Arc::new(RwLock::new(GuestLookupTableState::new(host_uuid))),
}
}
pub fn host_uuid(&self) -> Uuid {
self.host_uuid
}
pub fn host_domid(&self) -> u32 {
self.host_domid
}
pub async fn lookup_uuid_by_domid(&self, domid: u32) -> Option<Uuid> {
let state = self.state.read().await;
state.domid_to_uuid.get(&domid).cloned()
}
pub async fn lookup_domid_by_uuid(&self, uuid: &Uuid) -> Option<u32> {
let state = self.state.read().await;
state.uuid_to_domid.get(uuid).cloned()
}
pub async fn associate(&self, uuid: Uuid, domid: u32) {
let mut state = self.state.write().await;
state.uuid_to_domid.insert(uuid, domid);
state.domid_to_uuid.insert(domid, uuid);
}
pub async fn remove(&self, uuid: Uuid, domid: u32) {
let mut state = self.state.write().await;
state.uuid_to_domid.remove(&uuid);
state.domid_to_uuid.remove(&domid);
}
}

View File

@ -22,12 +22,16 @@ use tokio::{
}, },
task::JoinHandle, task::JoinHandle,
}; };
use uuid::Uuid;
use crate::glt::GuestLookupTable;
type BackendFeedMap = Arc<Mutex<HashMap<u32, Sender<IdmTransportPacket>>>>; type BackendFeedMap = Arc<Mutex<HashMap<u32, Sender<IdmTransportPacket>>>>;
type ClientMap = Arc<Mutex<HashMap<u32, IdmInternalClient>>>; type ClientMap = Arc<Mutex<HashMap<u32, IdmInternalClient>>>;
#[derive(Clone)] #[derive(Clone)]
pub struct DaemonIdmHandle { pub struct DaemonIdmHandle {
glt: GuestLookupTable,
clients: ClientMap, clients: ClientMap,
feeds: BackendFeedMap, feeds: BackendFeedMap,
tx_sender: Sender<(u32, IdmTransportPacket)>, tx_sender: Sender<(u32, IdmTransportPacket)>,
@ -40,7 +44,14 @@ impl DaemonIdmHandle {
self.snoop_sender.subscribe() self.snoop_sender.subscribe()
} }
pub async fn client(&self, domid: u32) -> Result<IdmInternalClient> { pub async fn client(&self, uuid: Uuid) -> Result<IdmInternalClient> {
let Some(domid) = self.glt.lookup_domid_by_uuid(&uuid).await else {
return Err(anyhow!("unable to find domain {}", uuid));
};
self.client_by_domid(domid).await
}
pub async fn client_by_domid(&self, domid: u32) -> Result<IdmInternalClient> {
client_or_create(domid, &self.tx_sender, &self.clients, &self.feeds).await client_or_create(domid, &self.tx_sender, &self.clients, &self.feeds).await
} }
} }
@ -61,6 +72,7 @@ pub struct DaemonIdmSnoopPacket {
} }
pub struct DaemonIdm { pub struct DaemonIdm {
glt: GuestLookupTable,
clients: ClientMap, clients: ClientMap,
feeds: BackendFeedMap, feeds: BackendFeedMap,
tx_sender: Sender<(u32, IdmTransportPacket)>, tx_sender: Sender<(u32, IdmTransportPacket)>,
@ -72,7 +84,7 @@ pub struct DaemonIdm {
} }
impl DaemonIdm { impl DaemonIdm {
pub async fn new() -> Result<DaemonIdm> { pub async fn new(glt: GuestLookupTable) -> Result<DaemonIdm> {
let (service, tx_raw_sender, rx_receiver) = let (service, tx_raw_sender, rx_receiver) =
ChannelService::new("krata-channel".to_string(), None).await?; ChannelService::new("krata-channel".to_string(), None).await?;
let (tx_sender, tx_receiver) = channel(100); let (tx_sender, tx_receiver) = channel(100);
@ -81,6 +93,7 @@ impl DaemonIdm {
let clients = Arc::new(Mutex::new(HashMap::new())); let clients = Arc::new(Mutex::new(HashMap::new()));
let feeds = Arc::new(Mutex::new(HashMap::new())); let feeds = Arc::new(Mutex::new(HashMap::new()));
Ok(DaemonIdm { Ok(DaemonIdm {
glt,
rx_receiver, rx_receiver,
tx_receiver, tx_receiver,
tx_sender, tx_sender,
@ -93,6 +106,7 @@ impl DaemonIdm {
} }
pub async fn launch(mut self) -> Result<DaemonIdmHandle> { pub async fn launch(mut self) -> Result<DaemonIdmHandle> {
let glt = self.glt.clone();
let clients = self.clients.clone(); let clients = self.clients.clone();
let feeds = self.feeds.clone(); let feeds = self.feeds.clone();
let tx_sender = self.tx_sender.clone(); let tx_sender = self.tx_sender.clone();
@ -105,6 +119,7 @@ impl DaemonIdm {
} }
}); });
Ok(DaemonIdmHandle { Ok(DaemonIdmHandle {
glt,
clients, clients,
feeds, feeds,
tx_sender, tx_sender,

View File

@ -5,6 +5,7 @@ use console::{DaemonConsole, DaemonConsoleHandle};
use control::DaemonControlService; use control::DaemonControlService;
use db::GuestStore; use db::GuestStore;
use event::{DaemonEventContext, DaemonEventGenerator}; use event::{DaemonEventContext, DaemonEventGenerator};
use glt::GuestLookupTable;
use idm::{DaemonIdm, DaemonIdmHandle}; use idm::{DaemonIdm, DaemonIdmHandle};
use krata::{dial::ControlDialAddress, v1::control::control_service_server::ControlServiceServer}; use krata::{dial::ControlDialAddress, v1::control::control_service_server::ControlServiceServer};
use krataoci::{packer::service::OciPackerService, registry::OciPlatform}; use krataoci::{packer::service::OciPackerService, registry::OciPlatform};
@ -21,10 +22,12 @@ use tokio_stream::wrappers::UnixListenerStream;
use tonic::transport::{Identity, Server, ServerTlsConfig}; use tonic::transport::{Identity, Server, ServerTlsConfig};
use uuid::Uuid; use uuid::Uuid;
pub mod command;
pub mod console; pub mod console;
pub mod control; pub mod control;
pub mod db; pub mod db;
pub mod event; pub mod event;
pub mod glt;
pub mod idm; pub mod idm;
pub mod metrics; pub mod metrics;
pub mod oci; pub mod oci;
@ -32,6 +35,7 @@ pub mod reconcile;
pub struct Daemon { pub struct Daemon {
store: String, store: String,
glt: GuestLookupTable,
guests: GuestStore, guests: GuestStore,
events: DaemonEventContext, events: DaemonEventContext,
guest_reconciler_task: JoinHandle<()>, guest_reconciler_task: JoinHandle<()>,
@ -51,22 +55,43 @@ impl Daemon {
image_cache_dir.push("image"); image_cache_dir.push("image");
fs::create_dir_all(&image_cache_dir).await?; fs::create_dir_all(&image_cache_dir).await?;
let mut host_uuid_path = PathBuf::from(store.clone());
host_uuid_path.push("host.uuid");
let host_uuid = if host_uuid_path.is_file() {
let content = fs::read_to_string(&host_uuid_path).await?;
Uuid::from_str(content.trim()).ok()
} else {
None
};
let host_uuid = if let Some(host_uuid) = host_uuid {
host_uuid
} else {
let generated = Uuid::new_v4();
let mut string = generated.to_string();
string.push('\n');
fs::write(&host_uuid_path, string).await?;
generated
};
let packer = OciPackerService::new(None, &image_cache_dir, OciPlatform::current()).await?; let packer = OciPackerService::new(None, &image_cache_dir, OciPlatform::current()).await?;
let runtime = Runtime::new(store.clone()).await?; let runtime = Runtime::new(store.clone()).await?;
let glt = GuestLookupTable::new(0, host_uuid);
let guests_db_path = format!("{}/guests.db", store); let guests_db_path = format!("{}/guests.db", store);
let guests = GuestStore::open(&PathBuf::from(guests_db_path))?; let guests = GuestStore::open(&PathBuf::from(guests_db_path))?;
let (guest_reconciler_notify, guest_reconciler_receiver) = let (guest_reconciler_notify, guest_reconciler_receiver) =
channel::<Uuid>(GUEST_RECONCILER_QUEUE_LEN); channel::<Uuid>(GUEST_RECONCILER_QUEUE_LEN);
let idm = DaemonIdm::new().await?; let idm = DaemonIdm::new(glt.clone()).await?;
let idm = idm.launch().await?; let idm = idm.launch().await?;
let console = DaemonConsole::new().await?; let console = DaemonConsole::new(glt.clone()).await?;
let console = console.launch().await?; let console = console.launch().await?;
let (events, generator) = let (events, generator) =
DaemonEventGenerator::new(guests.clone(), guest_reconciler_notify.clone(), idm.clone()) DaemonEventGenerator::new(guests.clone(), guest_reconciler_notify.clone(), idm.clone())
.await?; .await?;
let runtime_for_reconciler = runtime.dupe().await?; let runtime_for_reconciler = runtime.dupe().await?;
let guest_reconciler = GuestReconciler::new( let guest_reconciler = GuestReconciler::new(
glt.clone(),
guests.clone(), guests.clone(),
events.clone(), events.clone(),
runtime_for_reconciler, runtime_for_reconciler,
@ -79,6 +104,7 @@ impl Daemon {
Ok(Self { Ok(Self {
store, store,
glt,
guests, guests,
events, events,
guest_reconciler_task, guest_reconciler_task,
@ -92,6 +118,7 @@ impl Daemon {
pub async fn listen(&mut self, addr: ControlDialAddress) -> Result<()> { pub async fn listen(&mut self, addr: ControlDialAddress) -> Result<()> {
let control_service = DaemonControlService::new( let control_service = DaemonControlService::new(
self.glt.clone(),
self.events.clone(), self.events.clone(),
self.console.clone(), self.console.clone(),
self.idm.clone(), self.idm.clone(),

View File

@ -30,6 +30,7 @@ use uuid::Uuid;
use crate::{ use crate::{
db::GuestStore, db::GuestStore,
event::{DaemonEvent, DaemonEventContext}, event::{DaemonEvent, DaemonEventContext},
glt::GuestLookupTable,
}; };
const PARALLEL_LIMIT: u32 = 5; const PARALLEL_LIMIT: u32 = 5;
@ -53,6 +54,7 @@ impl Drop for GuestReconcilerEntry {
#[derive(Clone)] #[derive(Clone)]
pub struct GuestReconciler { pub struct GuestReconciler {
glt: GuestLookupTable,
guests: GuestStore, guests: GuestStore,
events: DaemonEventContext, events: DaemonEventContext,
runtime: Runtime, runtime: Runtime,
@ -64,6 +66,7 @@ pub struct GuestReconciler {
impl GuestReconciler { impl GuestReconciler {
pub fn new( pub fn new(
glt: GuestLookupTable,
guests: GuestStore, guests: GuestStore,
events: DaemonEventContext, events: DaemonEventContext,
runtime: Runtime, runtime: Runtime,
@ -71,6 +74,7 @@ impl GuestReconciler {
guest_reconciler_notify: Sender<Uuid>, guest_reconciler_notify: Sender<Uuid>,
) -> Result<Self> { ) -> Result<Self> {
Ok(Self { Ok(Self {
glt,
guests, guests,
events, events,
runtime, runtime,
@ -123,6 +127,23 @@ impl GuestReconciler {
trace!("reconciling runtime"); trace!("reconciling runtime");
let runtime_guests = self.runtime.list().await?; let runtime_guests = self.runtime.list().await?;
let stored_guests = self.guests.list().await?; let stored_guests = self.guests.list().await?;
let non_existent_guests = runtime_guests
.iter()
.filter(|x| !stored_guests.iter().any(|g| *g.0 == x.uuid))
.collect::<Vec<_>>();
for guest in non_existent_guests {
warn!("destroying unknown runtime guest {}", guest.uuid);
if let Err(error) = self.runtime.destroy(guest.uuid).await {
error!(
"failed to destroy unknown runtime guest {}: {}",
guest.uuid, error
);
}
self.guests.remove(guest.uuid).await?;
}
for (uuid, mut stored_guest) in stored_guests { for (uuid, mut stored_guest) in stored_guests {
let previous_guest = stored_guest.clone(); let previous_guest = stored_guest.clone();
let runtime_guest = runtime_guests.iter().find(|x| x.uuid == uuid); let runtime_guest = runtime_guests.iter().find(|x| x.uuid == uuid);
@ -136,6 +157,7 @@ impl GuestReconciler {
} }
Some(runtime) => { Some(runtime) => {
self.glt.associate(uuid, runtime.domid).await;
let mut state = stored_guest.state.as_mut().cloned().unwrap_or_default(); let mut state = stored_guest.state.as_mut().cloned().unwrap_or_default();
if let Some(code) = runtime.state.exit_code { if let Some(code) = runtime.state.exit_code {
state.status = GuestStatus::Exited.into(); state.status = GuestStatus::Exited.into();
@ -283,12 +305,14 @@ impl GuestReconciler {
debug: false, debug: false,
}) })
.await?; .await?;
self.glt.associate(uuid, info.domid).await;
info!("started guest {}", uuid); info!("started guest {}", uuid);
guest.state = Some(GuestState { guest.state = Some(GuestState {
status: GuestStatus::Started.into(), status: GuestStatus::Started.into(),
network: Some(guestinfo_to_networkstate(&info)), network: Some(guestinfo_to_networkstate(&info)),
exit_info: None, exit_info: None,
error_info: None, error_info: None,
host: self.glt.host_uuid().to_string(),
domid: info.domid, domid: info.domid,
}); });
Ok(GuestReconcilerResult::Changed { rerun: false }) Ok(GuestReconcilerResult::Changed { rerun: false })
@ -308,13 +332,20 @@ impl GuestReconciler {
trace!("failed to destroy runtime guest {}: {}", uuid, error); trace!("failed to destroy runtime guest {}: {}", uuid, error);
} }
let domid = guest.state.as_ref().map(|x| x.domid);
if let Some(domid) = domid {
self.glt.remove(uuid, domid).await;
}
info!("destroyed guest {}", uuid); info!("destroyed guest {}", uuid);
guest.state = Some(GuestState { guest.state = Some(GuestState {
status: GuestStatus::Destroyed.into(), status: GuestStatus::Destroyed.into(),
network: None, network: None,
exit_info: None, exit_info: None,
error_info: None, error_info: None,
domid: guest.state.as_ref().map(|x| x.domid).unwrap_or(u32::MAX), host: self.glt.host_uuid().to_string(),
domid: domid.unwrap_or(u32::MAX),
}); });
Ok(GuestReconcilerResult::Changed { rerun: false }) Ok(GuestReconcilerResult::Changed { rerun: false })
} }

View File

@ -62,7 +62,8 @@ message GuestState {
GuestNetworkState network = 2; GuestNetworkState network = 2;
GuestExitInfo exit_info = 3; GuestExitInfo exit_info = 3;
GuestErrorInfo error_info = 4; GuestErrorInfo error_info = 4;
uint32 domid = 5; string host = 5;
uint32 domid = 6;
} }
enum GuestStatus { enum GuestStatus {

View File

@ -10,6 +10,8 @@ import "krata/idm/transport.proto";
import "krata/v1/common.proto"; import "krata/v1/common.proto";
service ControlService { service ControlService {
rpc IdentifyHost(IdentifyHostRequest) returns (IdentifyHostReply);
rpc CreateGuest(CreateGuestRequest) returns (CreateGuestReply); rpc CreateGuest(CreateGuestRequest) returns (CreateGuestReply);
rpc DestroyGuest(DestroyGuestRequest) returns (DestroyGuestReply); rpc DestroyGuest(DestroyGuestRequest) returns (DestroyGuestReply);
rpc ResolveGuest(ResolveGuestRequest) returns (ResolveGuestReply); rpc ResolveGuest(ResolveGuestRequest) returns (ResolveGuestReply);
@ -24,6 +26,14 @@ service ControlService {
rpc PullImage(PullImageRequest) returns (stream PullImageReply); rpc PullImage(PullImageRequest) returns (stream PullImageReply);
} }
message IdentifyHostRequest {}
message IdentifyHostReply {
string host_uuid = 1;
uint32 host_domid = 2;
string krata_version = 3;
}
message CreateGuestRequest { message CreateGuestRequest {
krata.v1.common.GuestSpec spec = 1; krata.v1.common.GuestSpec spec = 1;
} }
@ -84,8 +94,8 @@ message ReadGuestMetricsReply {
message SnoopIdmRequest {} message SnoopIdmRequest {}
message SnoopIdmReply { message SnoopIdmReply {
uint32 from = 1; string from = 1;
uint32 to = 2; string to = 2;
krata.idm.transport.IdmTransportPacket packet = 3; krata.idm.transport.IdmTransportPacket packet = 3;
} }