krata: implement guest reconciliation

crates/kratad/Cargo.toml

@@ -14,6 +14,8 @@ futures = { workspace = true }
 krata = { path = "../krata" }
 kratart = { path = "../kratart" }
 log = { workspace = true }
+prost = { workspace = true }
+redb = { workspace = true }
 serde = { workspace = true }
 serde_yaml = { workspace = true }
 signal-hook = { workspace = true }
@@ -28,3 +30,6 @@ name = "kratad"
 [[bin]]
 name = "kratad"
 path = "bin/daemon.rs"
+
+[build-dependencies]
+prost-build = { workspace = true }

crates/kratad/bin/daemon.rs

@@ -4,7 +4,6 @@ use env_logger::Env;
 use krata::dial::ControlDialAddress;
 use kratad::Daemon;
 use kratart::Runtime;
-use log::error;
 use std::{
     str::FromStr,
     sync::{atomic::AtomicBool, Arc},
@@ -16,8 +15,6 @@ struct Args {
     listen: String,
     #[arg(short, long, default_value = "/var/lib/krata")]
     store: String,
-    #[arg(long, default_value = "false")]
-    no_load_guest_tab: bool,
 }
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 10)]
@@ -29,11 +26,6 @@ async fn main() -> Result<()> {
     let addr = ControlDialAddress::from_str(&args.listen)?;
     let runtime = Runtime::new(args.store.clone()).await?;
     let mut daemon = Daemon::new(args.store.clone(), runtime).await?;
-    if !args.no_load_guest_tab {
-        if let Err(error) = daemon.load_guest_tab().await {
-            error!("failed to load guest tab: {}", error);
-        }
-    }
     daemon.listen(addr).await?;
     Ok(())
 }

crates/kratad/build.rs

@@ -0,0 +1,8 @@
+use std::io::Result;
+
+fn main() -> Result<()> {
+    prost_build::Config::new()
+        .extern_path(".krata.common", "::krata::common")
+        .compile_protos(&["proto/kratad/db.proto"], &["proto/", "../krata/proto"])?;
+    Ok(())
+}

crates/kratad/proto/kratad/db.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+package kratad.db;
+
+import "krata/common.proto";
+
+message GuestEntry {
+    string id = 1;
+    krata.common.Guest guest = 2;
+}

crates/kratad/src/control.rs

@@ -1,22 +1,29 @@
-use std::{io, pin::Pin};
+use std::{io, pin::Pin, str::FromStr};
 
 use async_stream::try_stream;
 use futures::Stream;
-use krata::control::{
-    control_service_server::ControlService, guest_image_spec::Image, ConsoleDataReply,
-    ConsoleDataRequest, DestroyGuestReply, DestroyGuestRequest, GuestImageSpec, GuestInfo,
-    GuestNetworkInfo, GuestOciImageSpec, LaunchGuestReply, LaunchGuestRequest, ListGuestsReply,
-    ListGuestsRequest, WatchEventsReply, WatchEventsRequest,
+use krata::{
+    common::{Guest, GuestState, GuestStatus},
+    control::{
+        control_service_server::ControlService, ConsoleDataReply, ConsoleDataRequest,
+        CreateGuestReply, CreateGuestRequest, DestroyGuestReply, DestroyGuestRequest,
+        ListGuestsReply, ListGuestsRequest, WatchEventsReply, WatchEventsRequest,
+    },
 };
+use kratart::Runtime;
 use tokio::{
     io::{AsyncReadExt, AsyncWriteExt},
     select,
+    sync::mpsc::Sender,
 };
 use tokio_stream::StreamExt;
 use tonic::{Request, Response, Status, Streaming};
+use uuid::Uuid;
 
-use crate::event::DaemonEventContext;
-use kratart::{launch::GuestLaunchRequest, Runtime};
+use crate::{
+    db::{proto::GuestEntry, GuestStore},
+    event::DaemonEventContext,
+};
 
 pub struct ApiError {
     message: String,
@@ -40,11 +47,23 @@ impl From<ApiError> for Status {
 pub struct RuntimeControlService {
     events: DaemonEventContext,
     runtime: Runtime,
+    guests: GuestStore,
+    guest_reconciler_notify: Sender<Uuid>,
 }
 
 impl RuntimeControlService {
-    pub fn new(events: DaemonEventContext, runtime: Runtime) -> Self {
-        Self { events, runtime }
+    pub fn new(
+        events: DaemonEventContext,
+        runtime: Runtime,
+        guests: GuestStore,
+        guest_reconciler_notify: Sender<Uuid>,
+    ) -> Self {
+        Self {
+            events,
+            runtime,
+            guests,
+            guest_reconciler_notify,
+        }
     }
 }
 
@@ -61,45 +80,46 @@ impl ControlService for RuntimeControlService {
     type WatchEventsStream =
         Pin<Box<dyn Stream<Item = Result<WatchEventsReply, Status>> + Send + 'static>>;
 
-    async fn launch_guest(
+    async fn create_guest(
         &self,
-        request: Request<LaunchGuestRequest>,
-    ) -> Result<Response<LaunchGuestReply>, Status> {
+        request: Request<CreateGuestRequest>,
+    ) -> Result<Response<CreateGuestReply>, Status> {
         let request = request.into_inner();
-        let Some(image) = request.image else {
+        let Some(spec) = request.spec else {
             return Err(ApiError {
-                message: "image spec not provider".to_string(),
+                message: "guest spec not provided".to_string(),
             }
             .into());
         };
-        let oci = match image.image {
-            Some(Image::Oci(oci)) => oci,
-            None => {
-                return Err(ApiError {
-                    message: "image spec not provided".to_string(),
-                }
-                .into())
-            }
-        };
-        let guest: GuestInfo = convert_guest_info(
-            self.runtime
-                .launch(GuestLaunchRequest {
-                    name: if request.name.is_empty() {
-                        None
-                    } else {
-                        Some(&request.name)
-                    },
-                    image: &oci.image,
-                    vcpus: request.vcpus,
-                    mem: request.mem,
-                    env: empty_vec_optional(request.env),
-                    run: empty_vec_optional(request.run),
-                    debug: false,
-                })
-                .await
-                .map_err(ApiError::from)?,
-        );
-        Ok(Response::new(LaunchGuestReply { guest: Some(guest) }))
+        let uuid = Uuid::new_v4();
+        self.guests
+            .update(
+                uuid,
+                GuestEntry {
+                    id: uuid.to_string(),
+                    guest: Some(Guest {
+                        id: uuid.to_string(),
+                        state: Some(GuestState {
+                            status: GuestStatus::Start.into(),
+                            exit_info: None,
+                            error_info: None,
+                        }),
+                        spec: Some(spec),
+                        network: None,
+                    }),
+                },
+            )
+            .await
+            .map_err(ApiError::from)?;
+        self.guest_reconciler_notify
+            .send(uuid)
+            .await
+            .map_err(|x| ApiError {
+                message: x.to_string(),
+            })?;
+        Ok(Response::new(CreateGuestReply {
+            guest_id: uuid.to_string(),
+        }))
     }
 
     async fn destroy_guest(
@@ -107,10 +127,42 @@ impl ControlService for RuntimeControlService {
         request: Request<DestroyGuestRequest>,
     ) -> Result<Response<DestroyGuestReply>, Status> {
         let request = request.into_inner();
-        self.runtime
-            .destroy(&request.guest_id)
+        let uuid = Uuid::from_str(&request.guest_id).map_err(|error| ApiError {
+            message: error.to_string(),
+        })?;
+        let Some(mut entry) = self.guests.read(uuid).await.map_err(ApiError::from)? else {
+            return Err(ApiError {
+                message: "guest not found".to_string(),
+            }
+            .into());
+        };
+        let Some(ref mut guest) = entry.guest else {
+            return Err(ApiError {
+                message: "guest not found".to_string(),
+            }
+            .into());
+        };
+
+        guest.state = Some(guest.state.as_mut().cloned().unwrap_or_default());
+
+        if guest.state.as_ref().unwrap().status() == GuestStatus::Destroyed {
+            return Err(ApiError {
+                message: "guest already destroyed".to_string(),
+            }
+            .into());
+        }
+
+        guest.state.as_mut().unwrap().status = GuestStatus::Destroy.into();
+        self.guests
+            .update(uuid, entry)
             .await
             .map_err(ApiError::from)?;
+        self.guest_reconciler_notify
+            .send(uuid)
+            .await
+            .map_err(|x| ApiError {
+                message: x.to_string(),
+            })?;
         Ok(Response::new(DestroyGuestReply {}))
     }
 
@@ -119,11 +171,11 @@ impl ControlService for RuntimeControlService {
         request: Request<ListGuestsRequest>,
     ) -> Result<Response<ListGuestsReply>, Status> {
         let _ = request.into_inner();
-        let guests = self.runtime.list().await.map_err(ApiError::from)?;
+        let guests = self.guests.list().await.map_err(ApiError::from)?;
         let guests = guests
-            .into_iter()
-            .map(convert_guest_info)
-            .collect::<Vec<GuestInfo>>();
+            .into_values()
+            .filter_map(|entry| entry.guest)
+            .collect::<Vec<Guest>>();
         Ok(Response::new(ListGuestsReply { guests }))
     }
 
@@ -191,25 +243,3 @@ impl ControlService for RuntimeControlService {
         Ok(Response::new(Box::pin(output) as Self::WatchEventsStream))
     }
 }
-
-fn empty_vec_optional<T>(value: Vec<T>) -> Option<Vec<T>> {
-    if value.is_empty() {
-        None
-    } else {
-        Some(value)
-    }
-}
-
-fn convert_guest_info(value: kratart::GuestInfo) -> GuestInfo {
-    GuestInfo {
-        name: value.name.unwrap_or_default(),
-        id: value.uuid.to_string(),
-        image: Some(GuestImageSpec {
-            image: Some(Image::Oci(GuestOciImageSpec { image: value.image })),
-        }),
-        network: Some(GuestNetworkInfo {
-            ipv4: value.ipv4.map(|x| x.ip().to_string()).unwrap_or_default(),
-            ipv6: value.ipv6.map(|x| x.ip().to_string()).unwrap_or_default(),
-        }),
-    }
-}

crates/kratad/src/db/mod.rs

@@ -0,0 +1,82 @@
+pub mod proto;
+
+use std::{collections::HashMap, path::Path, sync::Arc};
+
+use self::proto::GuestEntry;
+use anyhow::Result;
+use log::error;
+use prost::Message;
+use redb::{Database, ReadableTable, TableDefinition};
+use uuid::Uuid;
+
+const GUESTS: TableDefinition<u128, &[u8]> = TableDefinition::new("guests");
+
+#[derive(Clone)]
+pub struct GuestStore {
+    database: Arc<Database>,
+}
+
+impl GuestStore {
+    pub fn open(path: &Path) -> Result<Self> {
+        let database = Database::create(path)?;
+        let write = database.begin_write()?;
+        let _ = write.open_table(GUESTS);
+        write.commit()?;
+        Ok(GuestStore {
+            database: Arc::new(database),
+        })
+    }
+
+    pub async fn read(&self, id: Uuid) -> Result<Option<GuestEntry>> {
+        let read = self.database.begin_read()?;
+        let table = read.open_table(GUESTS)?;
+        let Some(entry) = table.get(id.to_u128_le())? else {
+            return Ok(None);
+        };
+        let bytes = entry.value();
+        Ok(Some(GuestEntry::decode(bytes)?))
+    }
+
+    pub async fn list(&self) -> Result<HashMap<Uuid, GuestEntry>> {
+        let mut guests: HashMap<Uuid, GuestEntry> = HashMap::new();
+        let read = self.database.begin_read()?;
+        let table = read.open_table(GUESTS)?;
+        for result in table.iter()? {
+            let (key, value) = result?;
+            let uuid = Uuid::from_u128_le(key.value());
+            let state = match GuestEntry::decode(value.value()) {
+                Ok(state) => state,
+                Err(error) => {
+                    error!(
+                        "found invalid guest state in database for uuid {}: {}",
+                        uuid, error
+                    );
+                    continue;
+                }
+            };
+            guests.insert(uuid, state);
+        }
+        Ok(guests)
+    }
+
+    pub async fn update(&self, id: Uuid, entry: GuestEntry) -> Result<()> {
+        let write = self.database.begin_write()?;
+        {
+            let mut table = write.open_table(GUESTS)?;
+            let bytes = entry.encode_to_vec();
+            table.insert(id.to_u128_le(), bytes.as_slice())?;
+        }
+        write.commit()?;
+        Ok(())
+    }
+
+    pub async fn remove(&self, id: Uuid) -> Result<()> {
+        let write = self.database.begin_write()?;
+        {
+            let mut table = write.open_table(GUESTS)?;
+            table.remove(id.to_u128_le())?;
+        }
+        write.commit()?;
+        Ok(())
+    }
+}

crates/kratad/src/db/proto.rs

@@ -0,0 +1 @@
+include!(concat!(env!("OUT_DIR"), "/kratad.db.rs"));

crates/kratad/src/event.rs

@@ -1,13 +1,22 @@
 use std::{collections::HashMap, time::Duration};
 
 use anyhow::Result;
-use krata::control::{GuestDestroyedEvent, GuestExitedEvent, GuestLaunchedEvent};
-use log::{error, info, warn};
-use tokio::{sync::broadcast, task::JoinHandle, time};
+use krata::{
+    common::{GuestExitInfo, GuestState, GuestStatus},
+    control::watch_events_reply::Event,
+};
+use log::error;
+use tokio::{
+    sync::{broadcast, mpsc::Sender},
+    task::JoinHandle,
+    time,
+};
 use uuid::Uuid;
 
 use kratart::{GuestInfo, Runtime};
 
+use crate::db::GuestStore;
+
 pub type DaemonEvent = krata::control::watch_events_reply::Event;
 
 const EVENT_CHANNEL_QUEUE_LEN: usize = 1000;
@@ -21,21 +30,34 @@ impl DaemonEventContext {
     pub fn subscribe(&self) -> broadcast::Receiver<DaemonEvent> {
         self.sender.subscribe()
     }
+
+    pub fn send(&self, event: DaemonEvent) -> Result<()> {
+        let _ = self.sender.send(event);
+        Ok(())
+    }
 }
 
 pub struct DaemonEventGenerator {
     runtime: Runtime,
+    guests: GuestStore,
+    guest_reconciler_notify: Sender<Uuid>,
     last: HashMap<Uuid, GuestInfo>,
-    sender: broadcast::Sender<DaemonEvent>,
+    _sender: broadcast::Sender<Event>,
 }
 
 impl DaemonEventGenerator {
-    pub async fn new(runtime: Runtime) -> Result<(DaemonEventContext, DaemonEventGenerator)> {
+    pub async fn new(
+        guests: GuestStore,
+        guest_reconciler_notify: Sender<Uuid>,
+        runtime: Runtime,
+    ) -> Result<(DaemonEventContext, DaemonEventGenerator)> {
         let (sender, _) = broadcast::channel(EVENT_CHANNEL_QUEUE_LEN);
         let generator = DaemonEventGenerator {
             runtime,
+            guests,
+            guest_reconciler_notify,
             last: HashMap::new(),
-            sender: sender.clone(),
+            _sender: sender.clone(),
         };
         let context = DaemonEventContext { sender };
         Ok((context, generator))
@@ -51,24 +73,7 @@ impl DaemonEventGenerator {
             map
         };
 
-        let mut events: Vec<DaemonEvent> = Vec::new();
-        let mut exits: Vec<GuestExitedEvent> = Vec::new();
-
-        for uuid in guests.keys() {
-            if !self.last.contains_key(uuid) {
-                events.push(DaemonEvent::GuestLaunched(GuestLaunchedEvent {
-                    guest_id: uuid.to_string(),
-                }));
-            }
-        }
-
-        for uuid in self.last.keys() {
-            if !guests.contains_key(uuid) {
-                events.push(DaemonEvent::GuestDestroyed(GuestDestroyedEvent {
-                    guest_id: uuid.to_string(),
-                }));
-            }
-        }
+        let mut exits: Vec<(Uuid, i32)> = Vec::new();
 
         for (uuid, guest) in &guests {
             let Some(last) = self.last.get(uuid) else {
@@ -83,23 +88,27 @@ impl DaemonEventGenerator {
                 continue;
             };
 
-            let exit = GuestExitedEvent {
-                guest_id: uuid.to_string(),
-                code,
-            };
+            exits.push((*uuid, code));
+        }
 
-            exits.push(exit.clone());
-            events.push(DaemonEvent::GuestExited(exit));
+        for (uuid, code) in exits {
+            if let Some(mut entry) = self.guests.read(uuid).await? {
+                let Some(ref mut guest) = entry.guest else {
+                    continue;
+                };
+
+                guest.state = Some(GuestState {
+                    status: GuestStatus::Exited.into(),
+                    exit_info: Some(GuestExitInfo { code }),
+                    error_info: None,
+                });
+
+                self.guests.update(uuid, entry).await?;
+                self.guest_reconciler_notify.send(uuid).await?;
+            }
         }
 
         self.last = guests;
-
-        for event in events {
-            let _ = self.sender.send(event);
-        }
-
-        self.process_exit_auto_destroy(exits).await?;
-
         Ok(())
     }
 
@@ -115,21 +124,4 @@ impl DaemonEventGenerator {
             }
         }))
     }
-
-    async fn process_exit_auto_destroy(&mut self, exits: Vec<GuestExitedEvent>) -> Result<()> {
-        for exit in exits {
-            if let Err(error) = self.runtime.destroy(&exit.guest_id).await {
-                warn!(
-                    "failed to auto-destroy exited guest {}: {}",
-                    exit.guest_id, error
-                );
-            } else {
-                info!(
-                    "auto-destroyed guest {}: exited with status {}",
-                    exit.guest_id, exit.code
-                );
-            }
-        }
-        Ok(())
-    }
 }

crates/kratad/src/lib.rs

@@ -2,100 +2,72 @@ use std::{net::SocketAddr, path::PathBuf, str::FromStr};
 
 use anyhow::Result;
 use control::RuntimeControlService;
+use db::GuestStore;
 use event::{DaemonEventContext, DaemonEventGenerator};
 use krata::{control::control_service_server::ControlServiceServer, dial::ControlDialAddress};
-use kratart::{launch::GuestLaunchRequest, Runtime};
-use log::{info, warn};
-use tab::Tab;
-use tokio::{fs, net::UnixListener, task::JoinHandle};
+use kratart::Runtime;
+use log::info;
+use reconcile::GuestReconciler;
+use tokio::{
+    net::UnixListener,
+    sync::mpsc::{channel, Sender},
+    task::JoinHandle,
+};
 use tokio_stream::wrappers::UnixListenerStream;
 use tonic::transport::{Identity, Server, ServerTlsConfig};
+use uuid::Uuid;
 
 pub mod control;
+pub mod db;
 pub mod event;
-pub mod tab;
+pub mod reconcile;
 
 pub struct Daemon {
     store: String,
     runtime: Runtime,
+    guests: GuestStore,
     events: DaemonEventContext,
-    task: JoinHandle<()>,
+    guest_reconciler_task: JoinHandle<()>,
+    guest_reconciler_notify: Sender<Uuid>,
+    generator_task: JoinHandle<()>,
 }
 
+const GUEST_RECONCILER_QUEUE_LEN: usize = 1000;
+
 impl Daemon {
     pub async fn new(store: String, runtime: Runtime) -> Result<Self> {
+        let guests_db_path = format!("{}/guests.db", store);
+        let guests = GuestStore::open(&PathBuf::from(guests_db_path))?;
         let runtime_for_events = runtime.dupe().await?;
-        let (events, generator) = DaemonEventGenerator::new(runtime_for_events).await?;
+        let (guest_reconciler_notify, guest_reconciler_receiver) =
+            channel::<Uuid>(GUEST_RECONCILER_QUEUE_LEN);
+        let (events, generator) = DaemonEventGenerator::new(
+            guests.clone(),
+            guest_reconciler_notify.clone(),
+            runtime_for_events,
+        )
+        .await?;
+        let runtime_for_reconciler = runtime.dupe().await?;
+        let guest_reconciler =
+            GuestReconciler::new(guests.clone(), events.clone(), runtime_for_reconciler)?;
         Ok(Self {
             store,
             runtime,
+            guests,
             events,
-            task: generator.launch().await?,
+            guest_reconciler_task: guest_reconciler.launch(guest_reconciler_receiver).await?,
+            guest_reconciler_notify,
+            generator_task: generator.launch().await?,
         })
     }
 
-    pub async fn load_guest_tab(&mut self) -> Result<()> {
-        let tab_path = PathBuf::from(format!("{}/guests.yml", self.store));
-
-        if !tab_path.exists() {
-            return Ok(());
-        }
-
-        info!("loading guest tab");
-
-        let tab_content = fs::read_to_string(tab_path).await?;
-        let tab: Tab = serde_yaml::from_str(&tab_content)?;
-        let running = self.runtime.list().await?;
-        for (name, guest) in tab.guests {
-            let existing = running
-                .iter()
-                .filter(|x| x.name.is_some())
-                .find(|run| *run.name.as_ref().unwrap() == name);
-
-            if let Some(existing) = existing {
-                info!("guest {} is already running: {}", name, existing.uuid);
-                continue;
-            }
-
-            let request = GuestLaunchRequest {
-                name: Some(&name),
-                image: &guest.image,
-                vcpus: guest.cpus,
-                mem: guest.mem,
-                env: if guest.env.is_empty() {
-                    None
-                } else {
-                    Some(
-                        guest
-                            .env
-                            .iter()
-                            .map(|(key, value)| format!("{}={}", key, value))
-                            .collect::<Vec<String>>(),
-                    )
-                },
-                run: if guest.run.is_empty() {
-                    None
-                } else {
-                    Some(guest.run)
-                },
-                debug: false,
-            };
-            match self.runtime.launch(request).await {
-                Err(error) => {
-                    warn!("failed to launch guest {}: {}", name, error);
-                }
-
-                Ok(info) => {
-                    info!("launched guest {}: {}", name, info.uuid);
-                }
-            }
-        }
-        info!("loaded guest tab");
-        Ok(())
-    }
-
     pub async fn listen(&mut self, addr: ControlDialAddress) -> Result<()> {
-        let control_service = RuntimeControlService::new(self.events.clone(), self.runtime.clone());
+        let control_service = RuntimeControlService::new(
+            self.events.clone(),
+            self.runtime.clone(),
+            self.guests.clone(),
+            self.guest_reconciler_notify.clone(),
+        );
 
         let mut server = Server::builder();
 
@@ -147,6 +119,7 @@ impl Daemon {
 
 impl Drop for Daemon {
     fn drop(&mut self) {
-        self.task.abort();
+        self.guest_reconciler_task.abort();
+        self.generator_task.abort();
     }
 }

crates/kratad/src/reconcile/mod.rs

@@ -0,0 +1,216 @@
+use anyhow::{anyhow, Result};
+use krata::{
+    common::{
+        guest_image_spec::Image, Guest, GuestErrorInfo, GuestNetworkState, GuestState, GuestStatus,
+    },
+    control::GuestChangedEvent,
+};
+use kratart::{launch::GuestLaunchRequest, Runtime};
+use log::{error, info, warn};
+use tokio::{sync::mpsc::Receiver, task::JoinHandle};
+use uuid::Uuid;
+
+use crate::{
+    db::GuestStore,
+    event::{DaemonEvent, DaemonEventContext},
+};
+
+pub struct GuestReconciler {
+    guests: GuestStore,
+    events: DaemonEventContext,
+    runtime: Runtime,
+}
+
+impl GuestReconciler {
+    pub fn new(guests: GuestStore, events: DaemonEventContext, runtime: Runtime) -> Result<Self> {
+        Ok(Self {
+            guests,
+            events,
+            runtime,
+        })
+    }
+
+    pub async fn launch(self, mut notify: Receiver<Uuid>) -> Result<JoinHandle<()>> {
+        Ok(tokio::task::spawn(async move {
+            if let Err(error) = self.reconcile_runtime().await {
+                error!("runtime reconciler failed: {}", error);
+            }
+
+            loop {
+                let Some(uuid) = notify.recv().await else {
+                    break;
+                };
+                if let Err(error) = self.reconcile(uuid).await {
+                    error!("guest reconciler failed: {}", error);
+                }
+            }
+        }))
+    }
+
+    pub async fn reconcile_runtime(&self) -> Result<()> {
+        let runtime_guests = self.runtime.list().await?;
+        let stored_guests = self.guests.list().await?;
+        for (uuid, mut stored_guest_entry) in stored_guests {
+            let Some(ref mut stored_guest) = stored_guest_entry.guest else {
+                warn!("removing unpopulated guest entry for guest {}", uuid);
+                self.guests.remove(uuid).await?;
+                continue;
+            };
+            let runtime_guest = runtime_guests.iter().find(|x| x.uuid == uuid);
+            match runtime_guest {
+                None => {
+                    let mut state = stored_guest.state.as_mut().cloned().unwrap_or_default();
+                    if state.status() == GuestStatus::Started {
+                        state.status = GuestStatus::Start.into();
+                    }
+                    stored_guest.state = Some(state);
+                    stored_guest.network = None;
+                    self.guests.update(uuid, stored_guest_entry).await?;
+                    if let Err(error) = self.reconcile(uuid).await {
+                        error!("failed to reconcile guest {}: {}", uuid, error);
+                    }
+                }
+
+                Some(_) => {
+                    let mut state = stored_guest.state.as_mut().cloned().unwrap_or_default();
+                    state.status = GuestStatus::Started.into();
+                    stored_guest.state = Some(state);
+                    stored_guest.network = None;
+                    self.guests.update(uuid, stored_guest_entry).await?;
+                    if let Err(error) = self.reconcile(uuid).await {
+                        error!("failed to reconcile guest {}: {}", uuid, error);
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn reconcile(&self, uuid: Uuid) -> Result<()> {
+        let Some(mut entry) = self.guests.read(uuid).await? else {
+            warn!(
+                "notified of reconcile for guest {} but it didn't exist",
+                uuid
+            );
+            return Ok(());
+        };
+
+        info!("reconciling guest {}", uuid);
+
+        let Some(ref mut guest) = entry.guest else {
+            return Ok(());
+        };
+
+        self.events
+            .send(DaemonEvent::GuestChanged(GuestChangedEvent {
+                guest: Some(guest.clone()),
+            }))?;
+
+        let result = match guest.state.as_ref().map(|x| x.status()).unwrap_or_default() {
+            GuestStatus::Start => self.start(uuid, guest).await.map(|_| true),
+
+            GuestStatus::Destroy | GuestStatus::Exited => {
+                self.destroy(uuid, guest).await.map(|_| true)
+            }
+
+            _ => Ok(false),
+        };
+
+        let changed = match result {
+            Ok(changed) => changed,
+            Err(error) => {
+                guest.state = Some(guest.state.as_mut().cloned().unwrap_or_default());
+                guest.state.as_mut().unwrap().error_info = Some(GuestErrorInfo {
+                    message: error.to_string(),
+                });
+                true
+            }
+        };
+
+        info!("reconciled guest {}", uuid);
+
+        let destroyed =
+            guest.state.as_ref().map(|x| x.status()).unwrap_or_default() == GuestStatus::Destroyed;
+
+        if changed {
+            let event = DaemonEvent::GuestChanged(GuestChangedEvent {
+                guest: Some(guest.clone()),
+            });
+
+            if destroyed {
+                self.guests.remove(uuid).await?;
+            } else {
+                self.guests.update(uuid, entry.clone()).await?;
+            }
+
+            self.events.send(event)?;
+        }
+
+        Ok(())
+    }
+
+    async fn start(&self, uuid: Uuid, guest: &mut Guest) -> Result<()> {
+        let Some(ref spec) = guest.spec else {
+            return Err(anyhow!("guest spec not specified"));
+        };
+
+        let Some(ref image) = spec.image else {
+            return Err(anyhow!("image spec not provided"));
+        };
+        let oci = match image.image {
+            Some(Image::Oci(ref oci)) => oci,
+            None => {
+                return Err(anyhow!("oci spec not specified"));
+            }
+        };
+
+        let info = self
+            .runtime
+            .launch(GuestLaunchRequest {
+                uuid: Some(uuid),
+                name: if spec.name.is_empty() {
+                    None
+                } else {
+                    Some(&spec.name)
+                },
+                image: &oci.image,
+                vcpus: spec.vcpus,
+                mem: spec.mem,
+                env: empty_vec_optional(spec.env.clone()),
+                run: empty_vec_optional(spec.run.clone()),
+                debug: false,
+            })
+            .await?;
+        info!("started guest {}", uuid);
+        guest.network = Some(GuestNetworkState {
+            ipv4: info.ipv4.map(|x| x.ip().to_string()).unwrap_or_default(),
+            ipv6: info.ipv6.map(|x| x.ip().to_string()).unwrap_or_default(),
+        });
+        guest.state = Some(GuestState {
+            status: GuestStatus::Started.into(),
+            exit_info: None,
+            error_info: None,
+        });
+        Ok(())
+    }
+
+    async fn destroy(&self, uuid: Uuid, guest: &mut Guest) -> Result<()> {
+        self.runtime.destroy(&uuid.to_string()).await?;
+        info!("destroyed guest {}", uuid);
+        guest.network = None;
+        guest.state = Some(GuestState {
+            status: GuestStatus::Destroyed.into(),
+            exit_info: None,
+            error_info: None,
+        });
+        Ok(())
+    }
+}
+
+fn empty_vec_optional<T>(value: Vec<T>) -> Option<Vec<T>> {
+    if value.is_empty() {
+        None
+    } else {
+        Some(value)
+    }
+}

crates/kratad/src/tab.rs

@@ -1,20 +0,0 @@
-use std::collections::HashMap;
-
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Tab {
-    #[serde(default)]
-    pub guests: HashMap<String, TabGuest>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct TabGuest {
-    pub image: String,
-    pub mem: u64,
-    pub cpus: u32,
-    #[serde(default)]
-    pub env: HashMap<String, String>,
-    #[serde(default)]
-    pub run: Vec<String>,
-}