mirrors/krata
1
0
Fork 0
mirror of https://github.com/edera-dev/krata.git synced 2025-11-04 07:39:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

krata: rework into daemon / controller structure

Browse Source
This commit is contained in:
Alex Zenla
2024-03-05 11:35:25 +00:00
parent 17889d1c64
commit 8653fd6249
45 changed files with 1597 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
controller/src/autoloop.rs
View File

@ -1,33 +0,0 @@
use anyhow::{anyhow, Result};
use loopdev::{LoopControl, LoopDevice};
use xenclient::BlockDeviceRef;
pub struct AutoLoop {
control: LoopControl,
}
impl AutoLoop {
pub fn new(control: LoopControl) -> AutoLoop {
AutoLoop { control }
}
pub fn loopify(&self, file: &str) -> Result<BlockDeviceRef> {
let device = self.control.next_free()?;
device.with().read_only(true).attach(file)?;
let path = device
.path()
.ok_or(anyhow!("unable to get loop device path"))?
.to_str()
.ok_or(anyhow!("unable to convert loop device path to string",))?
.to_string();
let major = device.major()?;
let minor = device.minor()?;
Ok(BlockDeviceRef { path, major, minor })
}
pub fn unloop(&self, device: &str) -> Result<()> {
let device = LoopDevice::open(device)?;
device.detach()?;
Ok(())
}
}

195
controller/src/client.rs Normal file
View File

@ -0,0 +1,195 @@
use std::{collections::HashMap, sync::Arc};
use anyhow::{anyhow, Result};
use krata::{
control::{Message, Request, RequestBox, Response},
stream::{ConnectionStreams, StreamContext},
};
use log::{trace, warn};
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
net::{unix, UnixStream},
select,
sync::{
mpsc::{channel, Receiver, Sender},
oneshot, Mutex,
},
task::JoinHandle,
};
use tokio_stream::{wrappers::LinesStream, StreamExt};
const QUEUE_MAX_LEN: usize = 100;
pub struct KrataClientTransport {
sender: Sender<Message>,
receiver: Receiver<Message>,
task: JoinHandle<()>,
}
impl Drop for KrataClientTransport {
fn drop(&mut self) {
self.task.abort();
}
}
impl KrataClientTransport {
pub async fn new(stream: UnixStream) -> Result<Self> {
let (read, write) = stream.into_split();
let (tx_sender, tx_receiver) = channel::<Message>(QUEUE_MAX_LEN);
let (rx_sender, rx_receiver) = channel::<Message>(QUEUE_MAX_LEN);
let task = tokio::task::spawn(async move {
if let Err(error) =
KrataClientTransport::process_unix_stream(read, write, rx_sender, tx_receiver).await
{
warn!("failed to process krata transport messages: {}", error);
}
});
Ok(Self {
sender: tx_sender,
receiver: rx_receiver,
task,
})
}
async fn process_unix_stream(
read: unix::OwnedReadHalf,
mut write: unix::OwnedWriteHalf,
rx_sender: Sender<Message>,
mut tx_receiver: Receiver<Message>,
) -> Result<()> {
let mut read = LinesStream::new(BufReader::new(read).lines());
loop {
select! {
x = tx_receiver.recv() => match x {
Some(message) => {
let mut line = serde_json::to_string(&message)?;
trace!("sending line '{}'", line);
line.push('\n');
write.write_all(line.as_bytes()).await?;
},
None => {
break;
}
},
x = read.next() => match x {
Some(Ok(line)) => {
let message = serde_json::from_str::<Message>(&line)?;
rx_sender.send(message).await?;
},
Some(Err(error)) => {
return Err(error.into());
},
None => {
break;
}
}
};
}
Ok(())
}
}
type RequestsMap = Arc<Mutex<HashMap<u64, oneshot::Sender<Response>>>>;
#[derive(Clone)]
pub struct KrataClient {
tx_sender: Sender<Message>,
next: Arc<Mutex<u64>>,
streams: ConnectionStreams,
requests: RequestsMap,
task: Arc<JoinHandle<()>>,
}
impl KrataClient {
pub async fn new(transport: KrataClientTransport) -> Result<Self> {
let tx_sender = transport.sender.clone();
let streams = ConnectionStreams::new(tx_sender.clone());
let requests = Arc::new(Mutex::new(HashMap::new()));
let task = {
let requests = requests.clone();
let streams = streams.clone();
tokio::task::spawn(async move {
if let Err(error) = KrataClient::process(transport, streams, requests).await {
warn!("failed to process krata client messages: {}", error);
}
})
};
Ok(Self {
tx_sender,
next: Arc::new(Mutex::new(0)),
requests,
streams,
task: Arc::new(task),
})
}
pub async fn send(&self, request: Request) -> Result<Response> {
let id = {
let mut next = self.next.lock().await;
let id = *next;
*next = id + 1;
id
};
let (sender, receiver) = oneshot::channel();
self.requests.lock().await.insert(id, sender);
self.tx_sender
.send(Message::Request(RequestBox { id, request }))
.await?;
let response = receiver.await?;
if let Response::Error(error) = response {
Err(anyhow!("krata error: {}", error.message))
} else {
Ok(response)
}
}
pub async fn acquire(&self, stream: u64) -> Result<StreamContext> {
self.streams.acquire(stream).await
}
async fn process(
mut transport: KrataClientTransport,
streams: ConnectionStreams,
requests: RequestsMap,
) -> Result<()> {
loop {
let Some(message) = transport.receiver.recv().await else {
break;
};
match message {
Message::Request(_) => {
return Err(anyhow!("received request from service"));
}
Message::Response(resp) => {
let Some(sender) = requests.lock().await.remove(&resp.id) else {
continue;
};
let _ = sender.send(resp.response);
}
Message::StreamUpdated(updated) => {
streams.incoming(updated).await?;
}
}
}
Ok(())
}
}
impl Drop for KrataClient {
fn drop(&mut self) {
if Arc::strong_count(&self.task) <= 1 {
self.task.abort();
}
}
}

86
controller/src/console.rs
View File

@ -4,71 +4,73 @@ use std::{
};
use anyhow::Result;
use futures::future::join_all;
use krata::{
control::{ConsoleStreamUpdate, StreamUpdate},
stream::StreamContext,
};
use log::debug;
use std::process::exit;
use termion::raw::IntoRawMode;
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncWriteExt},
select,
};
pub struct XenConsole {
xen_read_handle: File,
xen_write_handle: File,
stream: StreamContext,
}
impl XenConsole {
pub async fn new(tty: &str) -> Result<XenConsole> {
let xen_read_handle = File::options().read(true).write(false).open(tty).await?;
let xen_write_handle = File::options().read(false).write(true).open(tty).await?;
Ok(XenConsole {
xen_read_handle,
xen_write_handle,
})
pub async fn new(stream: StreamContext) -> Result<XenConsole> {
Ok(XenConsole { stream })
}
pub async fn attach(self) -> Result<()> {
let stdin = stdin();
let stdin = unsafe { File::from_raw_fd(stdin().as_raw_fd()) };
let terminal = stdout().into_raw_mode()?;
let stdout = unsafe { File::from_raw_fd(terminal.as_raw_fd()) };
let reader_task = tokio::task::spawn(async move {
if let Err(error) = XenConsole::copy_stdout(stdout, self.xen_read_handle).await {
debug!("failed to copy console output: {}", error);
}
});
let writer_task = tokio::task::spawn(async move {
if let Err(error) = XenConsole::intercept_stdin(
unsafe { File::from_raw_fd(stdin.as_raw_fd()) },
self.xen_write_handle,
)
.await
{
debug!("failed to intercept stdin: {}", error);
}
});
join_all(vec![reader_task, writer_task]).await;
if let Err(error) = XenConsole::process(stdin, stdout, self.stream).await {
debug!("failed to process console stream: {}", error);
}
Ok(())
}
async fn copy_stdout(mut stdout: File, mut console: File) -> Result<()> {
let mut buffer = vec![0u8; 256];
loop {
let size = console.read(&mut buffer).await?;
stdout.write_all(&buffer[0..size]).await?;
stdout.flush().await?;
}
}
async fn intercept_stdin(mut stdin: File, mut console: File) -> Result<()> {
async fn process(mut stdin: File, mut stdout: File, mut stream: StreamContext) -> Result<()> {
let mut buffer = vec![0u8; 60];
loop {
let size = stdin.read(&mut buffer).await?;
if size == 1 && buffer[0] == 0x1d {
exit(0);
}
console.write_all(&buffer[0..size]).await?;
select! {
x = stream.receiver.recv() => match x {
Some(StreamUpdate::ConsoleStream(update)) => {
stdout.write_all(&update.data).await?;
stdout.flush().await?;
},
None => {
break;
}
},
x = stdin.read(&mut buffer) => match x {
Ok(size) => {
if size == 1 && buffer[0] == 0x1d {
exit(0);
}
let data = buffer[0..size].to_vec();
stream.send(StreamUpdate::ConsoleStream(ConsoleStreamUpdate {
data,
})).await?;
},
Err(error) => {
return Err(error.into());
}
}
};
}
Ok(())
}
}

71
controller/src/ctl/cfgblk.rs
View File

@ -1,71 +0,0 @@
use crate::image::ImageInfo;
use anyhow::Result;
use backhand::{FilesystemWriter, NodeHeader};
use krata::LaunchInfo;
use log::trace;
use std::fs;
use std::fs::File;
use std::path::PathBuf;
use uuid::Uuid;
pub struct ConfigBlock<'a> {
pub image_info: &'a ImageInfo,
pub file: PathBuf,
pub dir: PathBuf,
}
impl ConfigBlock<'_> {
pub fn new<'a>(uuid: &Uuid, image_info: &'a ImageInfo) -> Result<ConfigBlock<'a>> {
let mut dir = std::env::temp_dir().clone();
dir.push(format!("krata-cfg-{}", uuid));
fs::create_dir_all(&dir)?;
let mut file = dir.clone();
file.push("config.squashfs");
Ok(ConfigBlock {
image_info,
file,
dir,
})
}
pub fn build(&self, launch_config: &LaunchInfo) -> Result<()> {
trace!("build launch_config={:?}", launch_config);
let manifest = self.image_info.config.to_string()?;
let launch = serde_json::to_string(launch_config)?;
let mut writer = FilesystemWriter::default();
writer.push_dir(
"/image",
NodeHeader {
permissions: 384,
uid: 0,
gid: 0,
mtime: 0,
},
)?;
writer.push_file(
manifest.as_bytes(),
"/image/config.json",
NodeHeader {
permissions: 384,
uid: 0,
gid: 0,
mtime: 0,
},
)?;
writer.push_file(
launch.as_bytes(),
"/launch.json",
NodeHeader {
permissions: 384,
uid: 0,
gid: 0,
mtime: 0,
},
)?;
let mut file = File::create(&self.file)?;
trace!("build write sqaushfs");
writer.write(&mut file)?;
trace!("build complete");
Ok(())
}
}

51
controller/src/ctl/console.rs
View File

@ -1,51 +0,0 @@
use std::{process::exit, time::Duration};
use anyhow::{anyhow, Result};
use log::warn;
use tokio::time::sleep;
use xenstore::client::XsdInterface;
use super::destroy::ControllerDestroy;
use crate::console::XenConsole;
use super::ControllerContext;
pub struct ControllerConsole<'a> {
context: &'a mut ControllerContext,
}
impl ControllerConsole<'_> {
pub fn new(context: &mut ControllerContext) -> ControllerConsole<'_> {
ControllerConsole { context }
}
pub async fn perform(&mut self, id: &str) -> Result<()> {
let info = self
.context
.resolve(id)
.await?
.ok_or_else(|| anyhow!("unable to resolve guest: {}", id))?;
let domid = info.domid;
let tty = self.context.xen.get_console_path(domid).await?;
let console = XenConsole::new(&tty).await?;
let dom_path = self.context.xen.store.get_domain_path(domid).await?;
tokio::task::spawn(async move {
if let Err(error) = console.attach().await {
warn!("failed to attach to console: {}", error);
}
});
let exit_code_path = format!("{}/krata/guest/exit-code", dom_path);
loop {
let Some(code) = self.context.xen.store.read_string(&exit_code_path).await? else {
sleep(Duration::from_secs(1)).await;
continue;
};
let mut destroy = ControllerDestroy::new(self.context);
destroy.perform(&domid.to_string()).await?;
exit(code.parse::<i32>()?);
}
}
}

64
controller/src/ctl/destroy.rs
View File

@ -1,64 +0,0 @@
use std::{fs, path::PathBuf};
use anyhow::{anyhow, Result};
use uuid::Uuid;
use xenstore::client::{XsdClient, XsdInterface};
use super::ControllerContext;
pub struct ControllerDestroy<'a> {
context: &'a mut ControllerContext,
}
impl ControllerDestroy<'_> {
pub fn new(context: &mut ControllerContext) -> ControllerDestroy<'_> {
ControllerDestroy { context }
}
pub async fn perform(&mut self, id: &str) -> Result<Uuid> {
let info = self
.context
.resolve(id)
.await?
.ok_or_else(|| anyhow!("unable to resolve guest: {}", id))?;
let domid = info.domid;
let mut store = XsdClient::open().await?;
let dom_path = store.get_domain_path(domid).await?;
let uuid = match store
.read_string(format!("{}/krata/uuid", dom_path).as_str())
.await?
{
None => {
return Err(anyhow!(
"domain {} was not found or not created by krata",
domid
))
}
Some(value) => value,
};
if uuid.is_empty() {
return Err(anyhow!("unable to find krata uuid based on the domain",));
}
let uuid = Uuid::parse_str(&uuid)?;
let loops = store
.read_string(format!("{}/krata/loops", dom_path).as_str())
.await?;
let loops = ControllerContext::parse_loop_set(&loops);
self.context.xen.destroy(domid).await?;
for info in &loops {
self.context.autoloop.unloop(&info.device)?;
match &info.delete {
None => {}
Some(delete) => {
let delete_path = PathBuf::from(delete);
if delete_path.is_file() || delete_path.is_symlink() {
fs::remove_file(&delete_path)?;
} else if delete_path.is_dir() {
fs::remove_dir_all(&delete_path)?;
}
}
}
}
Ok(uuid)
}
}

21
controller/src/ctl/launch.rs
View File

@ -1,21 +0,0 @@
use super::{ControllerContext, GuestInfo};
use crate::launch::{GuestLaunchRequest, GuestLauncher};
use anyhow::Result;
pub struct ControllerLaunch<'a> {
context: &'a mut ControllerContext,
}
impl ControllerLaunch<'_> {
pub fn new(context: &mut ControllerContext) -> ControllerLaunch<'_> {
ControllerLaunch { context }
}
pub async fn perform<'c, 'r>(
&'c mut self,
request: GuestLaunchRequest<'r>,
) -> Result<GuestInfo> {
let mut launcher = GuestLauncher::new()?;
launcher.launch(self.context, request).await
}
}

141
controller/src/ctl/mod.rs
View File

@ -1,141 +0,0 @@
pub mod cfgblk;
use crate::autoloop::AutoLoop;
use crate::image::cache::ImageCache;
use anyhow::{anyhow, Result};
use loopdev::LoopControl;
use std::fs;
use std::path::PathBuf;
use std::str::FromStr;
use uuid::Uuid;
use xenclient::XenClient;
use xenstore::client::XsdInterface;
pub mod console;
pub mod destroy;
pub mod launch;
pub struct ControllerContext {
pub image_cache: ImageCache,
pub autoloop: AutoLoop,
pub xen: XenClient,
}
pub struct ContainerLoopInfo {
pub device: String,
pub file: String,
pub delete: Option<String>,
}
pub struct GuestInfo {
pub uuid: Uuid,
pub domid: u32,
pub image: String,
pub loops: Vec<ContainerLoopInfo>,
pub ipv4: String,
pub ipv6: String,
}
impl ControllerContext {
pub async fn new(store_path: String) -> Result<ControllerContext> {
let mut image_cache_path = PathBuf::from(store_path);
image_cache_path.push("cache");
fs::create_dir_all(&image_cache_path)?;
let xen = XenClient::open().await?;
image_cache_path.push("image");
fs::create_dir_all(&image_cache_path)?;
let image_cache = ImageCache::new(&image_cache_path)?;
Ok(ControllerContext {
image_cache,
autoloop: AutoLoop::new(LoopControl::open()?),
xen,
})
}
pub async fn list(&mut self) -> Result<Vec<GuestInfo>> {
let mut containers: Vec<GuestInfo> = Vec::new();
for domid_candidate in self.xen.store.list("/local/domain").await? {
let dom_path = format!("/local/domain/{}", domid_candidate);
let uuid_string = match self
.xen
.store
.read_string(&format!("{}/krata/uuid", &dom_path))
.await?
{
None => continue,
Some(value) => value,
};
let domid =
u32::from_str(&domid_candidate).map_err(|_| anyhow!("failed to parse domid"))?;
let uuid = Uuid::from_str(&uuid_string)?;
let image = self
.xen
.store
.read_string(&format!("{}/krata/image", &dom_path))
.await?
.unwrap_or("unknown".to_string());
let loops = self
.xen
.store
.read_string(&format!("{}/krata/loops", &dom_path))
.await?;
let ipv4 = self
.xen
.store
.read_string(&format!("{}/krata/network/guest/ipv4", &dom_path))
.await?
.unwrap_or("unknown".to_string());
let ipv6: String = self
.xen
.store
.read_string(&format!("{}/krata/network/guest/ipv6", &dom_path))
.await?
.unwrap_or("unknown".to_string());
let loops = ControllerContext::parse_loop_set(&loops);
containers.push(GuestInfo {
uuid,
domid,
image,
loops,
ipv4,
ipv6,
});
}
Ok(containers)
}
pub async fn resolve(&mut self, id: &str) -> Result<Option<GuestInfo>> {
for container in self.list().await? {
let uuid_string = container.uuid.to_string();
let domid_string = container.domid.to_string();
if uuid_string == id || domid_string == id || id == format!("krata-{}", uuid_string) {
return Ok(Some(container));
}
}
Ok(None)
}
fn parse_loop_set(input: &Option<String>) -> Vec<ContainerLoopInfo> {
let Some(input) = input else {
return Vec::new();
};
let sets = input
.split(',')
.map(|x| x.to_string())
.map(|x| x.split(':').map(|v| v.to_string()).collect::<Vec<String>>())
.map(|x| (x[0].clone(), x[1].clone(), x[2].clone()))
.collect::<Vec<(String, String, String)>>();
sets.iter()
.map(|(device, file, delete)| ContainerLoopInfo {
device: device.clone(),
file: file.clone(),
delete: if delete == "none" {
None
} else {
Some(delete.clone())
},
})
.collect::<Vec<ContainerLoopInfo>>()
}
}

69
controller/src/image/cache.rs
View File

@ -1,69 +0,0 @@
use crate::image::{ImageInfo, Result};
use log::debug;
use oci_spec::image::{ImageConfiguration, ImageManifest};
use std::fs;
use std::path::{Path, PathBuf};
pub struct ImageCache {
cache_dir: PathBuf,
}
impl ImageCache {
pub fn new(cache_dir: &Path) -> Result<ImageCache> {
Ok(ImageCache {
cache_dir: cache_dir.to_path_buf(),
})
}
pub fn recall(&self, digest: &str) -> Result<Option<ImageInfo>> {
let mut squashfs_path = self.cache_dir.clone();
let mut config_path = self.cache_dir.clone();
let mut manifest_path = self.cache_dir.clone();
squashfs_path.push(format!("{}.squashfs", digest));
manifest_path.push(format!("{}.manifest.json", digest));
config_path.push(format!("{}.config.json", digest));
Ok(
if squashfs_path.exists() && manifest_path.exists() && config_path.exists() {
let squashfs_metadata = fs::metadata(&squashfs_path)?;
let manifest_metadata = fs::metadata(&manifest_path)?;
let config_metadata = fs::metadata(&config_path)?;
if squashfs_metadata.is_file()
&& manifest_metadata.is_file()
&& config_metadata.is_file()
{
let manifest_text = fs::read_to_string(&manifest_path)?;
let manifest: ImageManifest = serde_json::from_str(&manifest_text)?;
let config_text = fs::read_to_string(&config_path)?;
let config: ImageConfiguration = serde_json::from_str(&config_text)?;
debug!("cache hit digest={}", digest);
Some(ImageInfo::new(squashfs_path.clone(), manifest, config)?)
} else {
None
}
} else {
debug!("cache miss digest={}", digest);
None
},
)
}
pub fn store(&self, digest: &str, info: &ImageInfo) -> Result<ImageInfo> {
debug!("cache store digest={}", digest);
let mut squashfs_path = self.cache_dir.clone();
let mut manifest_path = self.cache_dir.clone();
let mut config_path = self.cache_dir.clone();
squashfs_path.push(format!("{}.squashfs", digest));
manifest_path.push(format!("{}.manifest.json", digest));
config_path.push(format!("{}.config.json", digest));
fs::copy(&info.image_squashfs, &squashfs_path)?;
let manifest_text = serde_json::to_string_pretty(&info.manifest)?;
fs::write(&manifest_path, manifest_text)?;
let config_text = serde_json::to_string_pretty(&info.config)?;
fs::write(&config_path, config_text)?;
ImageInfo::new(
squashfs_path.clone(),
info.manifest.clone(),
info.config.clone(),
)
}
}

138
controller/src/image/fetch.rs
View File

@ -1,138 +0,0 @@
use anyhow::{anyhow, Result};
use bytes::Bytes;
use oci_spec::image::{Arch, Descriptor, ImageIndex, ImageManifest, MediaType, Os, ToDockerV2S2};
use reqwest::{Client, RequestBuilder, Response};
use tokio::{fs::File, io::AsyncWriteExt};
use url::Url;
const MANIFEST_PICKER_PLATFORM: Os = Os::Linux;
const MANIFEST_PICKER_ARCHITECTURE: Arch = Arch::Amd64;
pub struct RegistryClient {
agent: Client,
url: Url,
}
impl RegistryClient {
pub fn new(url: Url) -> Result<RegistryClient> {
Ok(RegistryClient {
agent: Client::new(),
url,
})
}
async fn call(&mut self, req: RequestBuilder) -> Result<Response> {
self.agent.execute(req.build()?).await.map_err(|x| x.into())
}
pub async fn get_blob(&mut self, name: &str, descriptor: &Descriptor) -> Result<Bytes> {
let url = self
.url
.join(&format!("/v2/{}/blobs/{}", name, descriptor.digest()))?;
let response = self.call(self.agent.get(url.as_str())).await?;
Ok(response.bytes().await?)
}
pub async fn write_blob_to_file(
&mut self,
name: &str,
descriptor: &Descriptor,
mut dest: File,
) -> Result<u64> {
let url = self
.url
.join(&format!("/v2/{}/blobs/{}", name, descriptor.digest()))?;
let mut response = self.call(self.agent.get(url.as_str())).await?;
let mut size: u64 = 0;
while let Some(chunk) = response.chunk().await? {
dest.write_all(&chunk).await?;
size += chunk.len() as u64;
}
Ok(size)
}
async fn get_raw_manifest_with_digest(
&mut self,
name: &str,
reference: &str,
) -> Result<(ImageManifest, String)> {
let url = self
.url
.join(&format!("/v2/{}/manifests/{}", name, reference))?;
let accept = format!(
"{}, {}, {}, {}",
MediaType::ImageManifest.to_docker_v2s2()?,
MediaType::ImageManifest,
MediaType::ImageIndex,
MediaType::ImageIndex.to_docker_v2s2()?,
);
let response = self
.call(self.agent.get(url.as_str()).header("Accept", &accept))
.await?;
let digest = response
.headers()
.get("Docker-Content-Digest")
.ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))?
.to_str()?
.to_string();
let manifest = serde_json::from_str(&response.text().await?)?;
Ok((manifest, digest))
}
pub async fn get_manifest_with_digest(
&mut self,
name: &str,
reference: &str,
) -> Result<(ImageManifest, String)> {
let url = self
.url
.join(&format!("/v2/{}/manifests/{}", name, reference))?;
let accept = format!(
"{}, {}, {}, {}",
MediaType::ImageManifest.to_docker_v2s2()?,
MediaType::ImageManifest,
MediaType::ImageIndex,
MediaType::ImageIndex.to_docker_v2s2()?,
);
let response = self
.call(self.agent.get(url.as_str()).header("Accept", &accept))
.await?;
let content_type = response
.headers()
.get("Content-Type")
.ok_or_else(|| anyhow!("registry response did not have a Content-Type header"))?
.to_str()?;
if content_type == MediaType::ImageIndex.to_string()
|| content_type == MediaType::ImageIndex.to_docker_v2s2()?
{
let index = serde_json::from_str(&response.text().await?)?;
let descriptor = self
.pick_manifest(index)
.ok_or_else(|| anyhow!("unable to pick manifest from index"))?;
return self
.get_raw_manifest_with_digest(name, descriptor.digest())
.await;
}
let digest = response
.headers()
.get("Docker-Content-Digest")
.ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))?
.to_str()?
.to_string();
let manifest = serde_json::from_str(&response.text().await?)?;
Ok((manifest, digest))
}
fn pick_manifest(&mut self, index: ImageIndex) -> Option<Descriptor> {
for item in index.manifests() {
if let Some(platform) = item.platform() {
if *platform.os() == MANIFEST_PICKER_PLATFORM
&& *platform.architecture() == MANIFEST_PICKER_ARCHITECTURE
{
return Some(item.clone());
}
}
}
None
}
}

414
controller/src/image/mod.rs
View File

@ -1,414 +0,0 @@
pub mod cache;
pub mod fetch;
pub mod name;
use crate::image::cache::ImageCache;
use crate::image::fetch::RegistryClient;
use crate::image::name::ImageName;
use anyhow::{anyhow, Result};
use backhand::compression::Compressor;
use backhand::{FilesystemCompressor, FilesystemWriter, NodeHeader};
use flate2::read::GzDecoder;
use log::{debug, trace, warn};
use oci_spec::image::{Descriptor, ImageConfiguration, ImageManifest, MediaType, ToDockerV2S2};
use std::fs::File;
use std::io::{BufReader, Cursor};
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::{fs, io};
use tar::{Archive, Entry};
use uuid::Uuid;
use walkdir::WalkDir;
pub const IMAGE_SQUASHFS_VERSION: u64 = 1;
const LAYER_BUFFER_SIZE: usize = 128 * 1024;
// we utilize in-memory buffers when generating the squashfs for files
// under this size. for files of or above this size, we open a file.
// the file is then read during writing. we want to reduce the number
// of open files during squashfs generation, so this limit should be set
// to something that limits the number of files on average, at the expense
// of increased memory usage.
// TODO: it may be wise to, during crawling of the image layers, infer this
// value from the size to file count ratio of all layers.
const SQUASHFS_MEMORY_BUFFER_LIMIT: usize = 8 * 1024 * 1024;
pub struct ImageInfo {
pub image_squashfs: PathBuf,
pub manifest: ImageManifest,
pub config: ImageConfiguration,
}
impl ImageInfo {
fn new(
squashfs: PathBuf,
manifest: ImageManifest,
config: ImageConfiguration,
) -> Result<ImageInfo> {
Ok(ImageInfo {
image_squashfs: squashfs,
manifest,
config,
})
}
}
pub struct ImageCompiler<'a> {
cache: &'a ImageCache,
}
#[derive(Debug)]
enum LayerCompressionType {
None,
Gzip,
Zstd,
}
struct LayerFile {
digest: String,
compression: LayerCompressionType,
path: PathBuf,
}
impl LayerFile {
fn open_reader(&self) -> Result<Box<dyn io::Read>> {
Ok(match self.compression {
LayerCompressionType::None => Box::new(BufReader::with_capacity(
LAYER_BUFFER_SIZE,
File::open(&self.path)?,
)),
LayerCompressionType::Gzip => Box::new(GzDecoder::new(BufReader::with_capacity(
LAYER_BUFFER_SIZE,
File::open(&self.path)?,
))),
LayerCompressionType::Zstd => Box::new(zstd::Decoder::new(BufReader::with_capacity(
LAYER_BUFFER_SIZE,
File::open(&self.path)?,
))?),
})
}
}
impl ImageCompiler<'_> {
pub fn new(cache: &ImageCache) -> Result<ImageCompiler> {
Ok(ImageCompiler { cache })
}
pub async fn compile(&self, image: &ImageName) -> Result<ImageInfo> {
debug!("compile image={image}");
let mut tmp_dir = std::env::temp_dir().clone();
tmp_dir.push(format!("krata-compile-{}", Uuid::new_v4()));
let mut image_dir = tmp_dir.clone();
image_dir.push("image");
fs::create_dir_all(&image_dir)?;
let mut layer_dir = tmp_dir.clone();
layer_dir.push("layer");
fs::create_dir_all(&layer_dir)?;
let mut squash_file = tmp_dir.clone();
squash_file.push("image.squashfs");
let info = self
.download_and_compile(image, &layer_dir, &image_dir, &squash_file)
.await?;
fs::remove_dir_all(&tmp_dir)?;
Ok(info)
}
async fn download_and_compile(
&self,
image: &ImageName,
layer_dir: &Path,
image_dir: &PathBuf,
squash_file: &PathBuf,
) -> Result<ImageInfo> {
debug!(
"download manifest image={image}, image_dir={}",
image_dir.to_str().unwrap()
);
let mut client = RegistryClient::new(image.registry_url()?)?;
let (manifest, digest) = client
.get_manifest_with_digest(&image.name, &image.reference)
.await?;
let cache_key = format!(
"manifest={}:squashfs-version={}\n",
digest, IMAGE_SQUASHFS_VERSION
);
let cache_digest = sha256::digest(cache_key);
if let Some(cached) = self.cache.recall(&cache_digest)? {
return Ok(cached);
}
debug!(
"download config digest={} size={}",
manifest.config().digest(),
manifest.config().size(),
);
let config_bytes = client.get_blob(&image.name, manifest.config()).await?;
let config: ImageConfiguration = serde_json::from_slice(&config_bytes)?;
let mut layers: Vec<LayerFile> = Vec::new();
for layer in manifest.layers() {
layers.push(
self.download_layer(image, layer, layer_dir, &mut client)
.await?,
);
}
for layer in layers {
debug!(
"process layer digest={} compression={:?}",
&layer.digest, layer.compression
);
let mut archive = Archive::new(layer.open_reader()?);
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?;
let Some(name) = path.file_name() else {
return Err(anyhow!("unable to get file name"));
};
let Some(name) = name.to_str() else {
return Err(anyhow!("unable to get file name as string"));
};
if name.starts_with(".wh.") {
self.process_whiteout_entry(&entry, name, &layer, image_dir)?;
} else {
self.process_write_entry(&mut entry, &layer, image_dir)?;
}
}
fs::remove_file(&layer.path)?;
}
self.squash(image_dir, squash_file)?;
let info = ImageInfo::new(squash_file.clone(), manifest.clone(), config)?;
self.cache.store(&cache_digest, &info)
}
fn process_whiteout_entry<T: io::Read>(
&self,
entry: &Entry<T>,
name: &str,
layer: &LayerFile,
image_dir: &PathBuf,
) -> Result<()> {
let dst = self.check_safe_entry(entry, image_dir)?;
let mut dst = dst.clone();
dst.pop();
let opaque = name == ".wh..wh..opq";
if !opaque {
dst.push(name);
self.check_safe_path(&dst, image_dir)?;
}
trace!(
"whiteout entry layer={} path={:?}",
&layer.digest,
entry.path()?
);
if opaque {
if dst.is_dir() {
for entry in fs::read_dir(dst)? {
let entry = entry?;
let path = entry.path();
if path.is_symlink() || path.is_file() {
fs::remove_file(&path)?;
} else if path.is_dir() {
fs::remove_dir_all(&path)?;
} else {
return Err(anyhow!("opaque whiteout entry did not exist"));
}
}
} else {
warn!(
"whiteout entry missing locally layer={} path={:?} local={:?}",
&layer.digest,
entry.path()?,
dst,
);
}
} else if dst.is_file() || dst.is_symlink() {
fs::remove_file(&dst)?;
} else if dst.is_dir() {
fs::remove_dir(&dst)?;
} else {
warn!(
"whiteout entry missing locally layer={} path={:?} local={:?}",
&layer.digest,
entry.path()?,
dst,
);
}
Ok(())
}
fn process_write_entry<T: io::Read>(
&self,
entry: &mut Entry<T>,
layer: &LayerFile,
image_dir: &PathBuf,
) -> Result<()> {
trace!(
"unpack entry layer={} path={:?} type={:?}",
&layer.digest,
entry.path()?,
entry.header().entry_type()
);
entry.unpack_in(image_dir)?;
Ok(())
}
fn check_safe_entry<T: io::Read>(
&self,
entry: &Entry<T>,
image_dir: &PathBuf,
) -> Result<PathBuf> {
let mut dst = image_dir.clone();
dst.push(entry.path()?);
if let Some(name) = dst.file_name() {
if let Some(name) = name.to_str() {
if name.starts_with(".wh.") {
let copy = dst.clone();
dst.pop();
self.check_safe_path(&dst, image_dir)?;
return Ok(copy);
}
}
}
self.check_safe_path(&dst, image_dir)?;
Ok(dst)
}
fn check_safe_path(&self, dst: &PathBuf, image_dir: &PathBuf) -> Result<()> {
let resolved = path_clean::clean(dst);
if !resolved.starts_with(image_dir) {
return Err(anyhow!("layer attempts to work outside image dir"));
}
Ok(())
}
async fn download_layer(
&self,
image: &ImageName,
layer: &Descriptor,
layer_dir: &Path,
client: &mut RegistryClient,
) -> Result<LayerFile> {
debug!(
"download layer digest={} size={}",
layer.digest(),
layer.size()
);
let mut layer_path = layer_dir.to_path_buf();
layer_path.push(layer.digest());
let mut tmp_path = layer_dir.to_path_buf();
tmp_path.push(format!("{}.tmp", layer.digest()));
{
let file = tokio::fs::File::create(&layer_path).await?;
let size = client.write_blob_to_file(&image.name, layer, file).await?;
if layer.size() as u64 != size {
return Err(anyhow!(
"downloaded layer size differs from size in manifest",
));
}
}
let mut media_type = layer.media_type().clone();
// docker layer compatibility
if media_type.to_string() == MediaType::ImageLayerGzip.to_docker_v2s2()? {
media_type = MediaType::ImageLayerGzip;
}
let compression = match media_type {
MediaType::ImageLayer => LayerCompressionType::None,
MediaType::ImageLayerGzip => LayerCompressionType::Gzip,
MediaType::ImageLayerZstd => LayerCompressionType::Zstd,
other => return Err(anyhow!("found layer with unknown media type: {}", other)),
};
Ok(LayerFile {
digest: layer.digest().clone(),
compression,
path: layer_path,
})
}
fn squash(&self, image_dir: &PathBuf, squash_file: &PathBuf) -> Result<()> {
let mut writer = FilesystemWriter::default();
writer.set_compressor(FilesystemCompressor::new(Compressor::Gzip, None)?);
let walk = WalkDir::new(image_dir).follow_links(false);
for entry in walk {
let entry = entry?;
let rel = entry
.path()
.strip_prefix(image_dir)?
.to_str()
.ok_or_else(|| anyhow!("failed to strip prefix of tmpdir"))?;
let rel = format!("/{}", rel);
trace!("squash write {}", rel);
let typ = entry.file_type();
let metadata = fs::symlink_metadata(entry.path())?;
let uid = metadata.uid();
let gid = metadata.gid();
let mode = metadata.permissions().mode();
let mtime = metadata.mtime();
if rel == "/" {
writer.set_root_uid(uid);
writer.set_root_gid(gid);
writer.set_root_mode(mode as u16);
continue;
}
let header = NodeHeader {
permissions: mode as u16,
uid,
gid,
mtime: mtime as u32,
};
if typ.is_symlink() {
let symlink = fs::read_link(entry.path())?;
let symlink = symlink
.to_str()
.ok_or_else(|| anyhow!("failed to read symlink"))?;
writer.push_symlink(symlink, rel, header)?;
} else if typ.is_dir() {
writer.push_dir(rel, header)?;
} else if typ.is_file() {
if metadata.size() >= SQUASHFS_MEMORY_BUFFER_LIMIT as u64 {
let reader =
BufReader::with_capacity(LAYER_BUFFER_SIZE, File::open(entry.path())?);
writer.push_file(reader, rel, header)?;
} else {
let cursor = Cursor::new(fs::read(entry.path())?);
writer.push_file(cursor, rel, header)?;
}
} else if typ.is_block_device() {
let device = metadata.dev();
writer.push_block_device(device as u32, rel, header)?;
} else if typ.is_char_device() {
let device = metadata.dev();
writer.push_char_device(device as u32, rel, header)?;
} else {
return Err(anyhow!("invalid file type"));
}
}
fs::remove_dir_all(image_dir)?;
let squash_file_path = squash_file
.to_str()
.ok_or_else(|| anyhow!("failed to convert squashfs string"))?;
let mut file = File::create(squash_file)?;
trace!("squash generate: {}", squash_file_path);
writer.write(&mut file)?;
Ok(())
}
}

89
controller/src/image/name.rs
View File

@ -1,89 +0,0 @@
use anyhow::Result;
use std::fmt;
use url::Url;
const DOCKER_HUB_MIRROR: &str = "mirror.gcr.io";
const DEFAULT_IMAGE_TAG: &str = "latest";
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ImageName {
pub hostname: String,
pub port: Option<u16>,
pub name: String,
pub reference: String,
}
impl fmt::Display for ImageName {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if let Some(port) = self.port {
write!(
f,
"{}:{}/{}:{}",
self.hostname, port, self.name, self.reference
)
} else {
write!(f, "{}/{}:{}", self.hostname, self.name, self.reference)
}
}
}
impl Default for ImageName {
fn default() -> Self {
Self::parse(&format!("{}", uuid::Uuid::new_v4().as_hyphenated()))
.expect("UUID hyphenated must be valid name")
}
}
impl ImageName {
pub fn parse(name: &str) -> Result<Self> {
let full_name = name.to_string();
let name = full_name.clone();
let (mut hostname, mut name) = name
.split_once('/')
.map(|x| (x.0.to_string(), x.1.to_string()))
.unwrap_or_else(|| (DOCKER_HUB_MIRROR.to_string(), format!("library/{}", name)));
// heuristic to find any docker hub image formats
// that may be in the hostname format. for example:
// abc/xyz:latest will trigger this if check, but abc.io/xyz:latest will not,
// and neither will abc/hello/xyz:latest
if !hostname.contains('.') && full_name.chars().filter(|x| *x == '/').count() == 1 {
name = format!("{}/{}", hostname, name);
hostname = DOCKER_HUB_MIRROR.to_string();
}
let (hostname, port) = if let Some((hostname, port)) = hostname
.split_once(':')
.map(|x| (x.0.to_string(), x.1.to_string()))
{
(hostname, Some(str::parse(&port)?))
} else {
(hostname, None)
};
let (name, reference) = name
.split_once(':')
.map(|x| (x.0.to_string(), x.1.to_string()))
.unwrap_or((name.to_string(), DEFAULT_IMAGE_TAG.to_string()));
Ok(ImageName {
hostname,
port,
name,
reference,
})
}
/// URL for OCI distribution API endpoint
pub fn registry_url(&self) -> Result<Url> {
let hostname = if let Some(port) = self.port {
format!("{}:{}", self.hostname, port)
} else {
self.hostname.clone()
};
let url = if self.hostname.starts_with("localhost") {
format!("http://{}", hostname)
} else {
format!("https://{}", hostname)
};
Ok(Url::parse(&url)?)
}
}

240
controller/src/launch/mod.rs
View File

@ -1,240 +0,0 @@
use std::{fs, net::Ipv4Addr, str::FromStr};
use advmac::MacAddr6;
use anyhow::{anyhow, Result};
use ipnetwork::Ipv4Network;
use krata::{
LaunchInfo, LaunchNetwork, LaunchNetworkIpv4, LaunchNetworkIpv6, LaunchNetworkResolver,
};
use uuid::Uuid;
use xenclient::{DomainConfig, DomainDisk, DomainNetworkInterface};
use xenstore::client::XsdInterface;
use crate::{
ctl::GuestInfo,
image::{cache::ImageCache, name::ImageName, ImageCompiler, ImageInfo},
};
use crate::ctl::{cfgblk::ConfigBlock, ControllerContext};
pub struct GuestLaunchRequest<'a> {
pub kernel_path: &'a str,
pub initrd_path: &'a str,
pub image: &'a str,
pub vcpus: u32,
pub mem: u64,
pub env: Option<Vec<String>>,
pub run: Option<Vec<String>>,
pub debug: bool,
}
pub struct GuestLauncher {}
impl GuestLauncher {
pub fn new() -> Result<Self> {
Ok(Self {})
}
pub async fn launch<'c, 'r>(
&mut self,
context: &'c mut ControllerContext,
request: GuestLaunchRequest<'r>,
) -> Result<GuestInfo> {
let uuid = Uuid::new_v4();
let name = format!("krata-{uuid}");
let image_info = self.compile(request.image, &context.image_cache).await?;
let mut gateway_mac = MacAddr6::random();
gateway_mac.set_local(true);
gateway_mac.set_multicast(false);
let mut container_mac = MacAddr6::random();
container_mac.set_local(true);
container_mac.set_multicast(false);
let guest_ipv4 = self.allocate_ipv4(context).await?;
let guest_ipv6 = container_mac.to_link_local_ipv6();
let gateway_ipv4 = "10.75.70.1";
let gateway_ipv6 = "fe80::1";
let ipv4_network_mask: u32 = 16;
let ipv6_network_mask: u32 = 10;
let launch_config = LaunchInfo {
network: Some(LaunchNetwork {
link: "eth0".to_string(),
ipv4: LaunchNetworkIpv4 {
address: format!("{}/{}", guest_ipv4, ipv4_network_mask),
gateway: gateway_ipv4.to_string(),
},
ipv6: LaunchNetworkIpv6 {
address: format!("{}/{}", guest_ipv6, ipv6_network_mask),
gateway: gateway_ipv6.to_string(),
},
resolver: LaunchNetworkResolver {
nameservers: vec![
"1.1.1.1".to_string(),
"1.0.0.1".to_string(),
"2606:4700:4700::1111".to_string(),
"2606:4700:4700::1001".to_string(),
],
},
}),
env: request.env,
run: request.run,
};
let cfgblk = ConfigBlock::new(&uuid, &image_info)?;
cfgblk.build(&launch_config)?;
let image_squashfs_path = image_info
.image_squashfs
.to_str()
.ok_or_else(|| anyhow!("failed to convert image squashfs path to string"))?;
let cfgblk_dir_path = cfgblk
.dir
.to_str()
.ok_or_else(|| anyhow!("failed to convert cfgblk directory path to string"))?;
let cfgblk_squashfs_path = cfgblk
.file
.to_str()
.ok_or_else(|| anyhow!("failed to convert cfgblk squashfs path to string"))?;
let image_squashfs_loop = context.autoloop.loopify(image_squashfs_path)?;
let cfgblk_squashfs_loop = context.autoloop.loopify(cfgblk_squashfs_path)?;
let cmdline_options = [
if request.debug { "debug" } else { "quiet" },
"elevator=noop",
];
let cmdline = cmdline_options.join(" ");
let container_mac_string = container_mac.to_string().replace('-', ":");
let gateway_mac_string = gateway_mac.to_string().replace('-', ":");
let config = DomainConfig {
backend_domid: 0,
name: &name,
max_vcpus: request.vcpus,
mem_mb: request.mem,
kernel_path: request.kernel_path,
initrd_path: request.initrd_path,
cmdline: &cmdline,
disks: vec![
DomainDisk {
vdev: "xvda",
block: &image_squashfs_loop,
writable: false,
},
DomainDisk {
vdev: "xvdb",
block: &cfgblk_squashfs_loop,
writable: false,
},
],
consoles: vec![],
vifs: vec![DomainNetworkInterface {
mac: &container_mac_string,
mtu: 1500,
bridge: None,
script: None,
}],
filesystems: vec![],
event_channels: vec![],
extra_keys: vec![
("krata/uuid".to_string(), uuid.to_string()),
(
"krata/loops".to_string(),
format!(
"{}:{}:none,{}:{}:{}",
&image_squashfs_loop.path,
image_squashfs_path,
&cfgblk_squashfs_loop.path,
cfgblk_squashfs_path,
cfgblk_dir_path,
),
),
("krata/image".to_string(), request.image.to_string()),
(
"krata/network/guest/ipv4".to_string(),
format!("{}/{}", guest_ipv4, ipv4_network_mask),
),
(
"krata/network/guest/ipv6".to_string(),
format!("{}/{}", guest_ipv6, ipv6_network_mask),
),
(
"krata/network/guest/mac".to_string(),
container_mac_string.clone(),
),
(
"krata/network/gateway/ipv4".to_string(),
format!("{}/{}", gateway_ipv4, ipv4_network_mask),
),
(
"krata/network/gateway/ipv6".to_string(),
format!("{}/{}", gateway_ipv6, ipv6_network_mask),
),
(
"krata/network/gateway/mac".to_string(),
gateway_mac_string.clone(),
),
],
extra_rw_paths: vec!["krata/guest".to_string()],
};
match context.xen.create(&config).await {
Ok(domid) => Ok(GuestInfo {
uuid,
domid,
image: request.image.to_string(),
loops: vec![],
ipv4: format!("{}/{}", guest_ipv4, ipv4_network_mask),
ipv6: format!("{}/{}", guest_ipv6, ipv6_network_mask),
}),
Err(error) => {
let _ = context.autoloop.unloop(&image_squashfs_loop.path);
let _ = context.autoloop.unloop(&cfgblk_squashfs_loop.path);
let _ = fs::remove_dir(&cfgblk.dir);
Err(error.into())
}
}
}
async fn compile(&self, image: &str, image_cache: &ImageCache) -> Result<ImageInfo> {
let image = ImageName::parse(image)?;
let compiler = ImageCompiler::new(image_cache)?;
compiler.compile(&image).await
}
async fn allocate_ipv4(&mut self, context: &mut ControllerContext) -> Result<Ipv4Addr> {
let network = Ipv4Network::new(Ipv4Addr::new(10, 75, 80, 0), 24)?;
let mut used: Vec<Ipv4Addr> = vec![];
for domid_candidate in context.xen.store.list("/local/domain").await? {
let dom_path = format!("/local/domain/{}", domid_candidate);
let ip_path = format!("{}/krata/network/guest/ipv4", dom_path);
let existing_ip = context.xen.store.read_string(&ip_path).await?;
if let Some(existing_ip) = existing_ip {
let ipv4_network = Ipv4Network::from_str(&existing_ip)?;
used.push(ipv4_network.ip());
}
}
let mut found: Option<Ipv4Addr> = None;
for ip in network.iter() {
let last = ip.octets()[3];
if last == 0 || last == 255 {
continue;
}
if !used.contains(&ip) {
found = Some(ip);
break;
}
}
if found.is_none() {
return Err(anyhow!(
"unable to find ipv4 to allocate to container, ipv4 addresses are exhausted"
));
}
Ok(found.unwrap())
}
}

5
controller/src/lib.rs
View File

@ -1,5 +1,2 @@
pub mod autoloop;
pub mod client;
pub mod console;
pub mod ctl;
pub mod image;
pub mod launch;