mirrors/krata
1
0
Fork 0
mirror of https://github.com/edera-dev/krata.git synced 2025-08-02 12:50:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: addons mounting and kernel modules support

Browse Source
This commit is contained in:
Alex Zenla 2024-04-25 16:06:57 -07:00
parent ef57de819f
commit 5dea89f644
No known key found for this signature in database
GPG Key ID: 067B238899B51269
14 changed files with 767 additions and 1885 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Cargo.lock generated
View File

@ -1481,6 +1481,7 @@ dependencies = [
"nix 0.28.0",
"oci-spec",
"path-absolutize",
"platform-info",
"rtnetlink",
"serde",
"serde_json",
@ -1560,6 +1561,7 @@ dependencies = [
"serde_json",
"tokio",
"uuid",
"walkdir",
]
[[package]]
@ -2070,6 +2072,16 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
[[package]]
name = "platform-info"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5ff316b9c4642feda973c18f0decd6c8b0919d4722566f6e4337cce0dd88217"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "portable-atomic"
version = "1.6.0"

1
Cargo.toml
View File

@ -57,6 +57,7 @@ oci-spec = "0.6.4"
once_cell = "1.19.0"
path-absolutize = "3.1.1"
path-clean = "1.0.1"
platform-info = "2.0.3"
prost = "0.12.4"
prost-build = "0.12.4"
prost-reflect-build = "0.13.0"

8
crates/daemon/src/lib.rs
View File

@ -88,8 +88,9 @@ impl Daemon {
generated
};
let initrd_path = detect_guest_file(&store, "initrd")?;
let kernel_path = detect_guest_file(&store, "kernel")?;
let initrd_path = detect_guest_path(&store, "initrd")?;
let kernel_path = detect_guest_path(&store, "kernel")?;
let addons_path = detect_guest_path(&store, "addons.squashfs")?;
let packer = OciPackerService::new(None, &image_cache_dir, OciPlatform::current()).await?;
let runtime = Runtime::new().await?;
@ -116,6 +117,7 @@ impl Daemon {
guest_reconciler_notify.clone(),
kernel_path,
initrd_path,
addons_path,
)?;
let guest_reconciler_task = guest_reconciler.launch(guest_reconciler_receiver).await?;
@ -204,7 +206,7 @@ impl Drop for Daemon {
}
}
fn detect_guest_file(store: &str, name: &str) -> Result<PathBuf> {
fn detect_guest_path(store: &str, name: &str) -> Result<PathBuf> {
let mut path = PathBuf::from(format!("{}/guest/{}", store, name));
if path.is_file() {
return Ok(path);

4
crates/daemon/src/reconcile/guest/mod.rs
View File

@ -64,6 +64,7 @@ pub struct GuestReconciler {
packer: OciPackerService,
kernel_path: PathBuf,
initrd_path: PathBuf,
addons_path: PathBuf,
tasks: Arc<Mutex<HashMap<Uuid, GuestReconcilerEntry>>>,
guest_reconciler_notify: Sender<Uuid>,
reconcile_lock: Arc<RwLock<()>>,
@ -81,6 +82,7 @@ impl GuestReconciler {
guest_reconciler_notify: Sender<Uuid>,
kernel_path: PathBuf,
initrd_path: PathBuf,
modules_path: PathBuf,
) -> Result<Self> {
Ok(Self {
devices,
@ -91,6 +93,7 @@ impl GuestReconciler {
packer,
kernel_path,
initrd_path,
addons_path: modules_path,
tasks: Arc::new(Mutex::new(HashMap::new())),
guest_reconciler_notify,
reconcile_lock: Arc::new(RwLock::with_max_readers((), PARALLEL_LIMIT)),
@ -278,6 +281,7 @@ impl GuestReconciler {
devices: &self.devices,
kernel_path: &self.kernel_path,
initrd_path: &self.initrd_path,
addons_path: &self.addons_path,
packer: &self.packer,
glt: &self.glt,
runtime: &self.runtime,

2
crates/daemon/src/reconcile/guest/start.rs
View File

@ -32,6 +32,7 @@ pub struct GuestStarter<'a> {
pub devices: &'a DaemonDeviceManager,
pub kernel_path: &'a Path,
pub initrd_path: &'a Path,
pub addons_path: &'a Path,
pub packer: &'a OciPackerService,
pub glt: &'a GuestLookupTable,
pub runtime: &'a Runtime,
@ -206,6 +207,7 @@ impl GuestStarter<'_> {
.collect::<HashMap<_, _>>(),
run: empty_vec_optional(task.command.clone()),
debug: false,
addons_image: Some(self.addons_path.to_path_buf()),
})
.await?;
self.glt.associate(uuid, info.domid).await;

1
crates/guest/Cargo.toml
View File

@ -21,6 +21,7 @@ log = { workspace = true }
nix = { workspace = true, features = ["ioctl", "process", "fs"] }
oci-spec = { workspace = true }
path-absolutize = { workspace = true }
platform-info = { workspace = true }
rtnetlink = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }

65
crates/guest/src/init.rs
View File

@ -12,6 +12,7 @@ use nix::ioctl_write_int_bad;
use nix::unistd::{dup2, execve, fork, ForkResult, Pid};
use oci_spec::image::{Config, ImageConfiguration};
use path_absolutize::Absolutize;
use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI};
use std::collections::HashMap;
use std::ffi::CString;
use std::fs::{File, OpenOptions, Permissions};
@ -50,6 +51,10 @@ const NEW_ROOT_DEV_PATH: &str = "/newroot/dev";
const IMAGE_CONFIG_JSON_PATH: &str = "/config/image/config.json";
const LAUNCH_CONFIG_JSON_PATH: &str = "/config/launch.json";
const ADDONS_DEVICE_PATH: &str = "/dev/xvdc";
const ADDONS_MOUNT_PATH: &str = "/addons";
const ADDONS_MODULES_PATH: &str = "/addons/modules";
ioctl_write_int_bad!(set_controlling_terminal, TIOCSCTTY);
pub struct GuestInit {}
@ -88,7 +93,10 @@ impl GuestInit {
self.mount_root_image(launch.root.format.clone()).await?;
self.mount_addons().await?;
self.mount_new_root().await?;
self.mount_kernel_modules().await?;
self.bind_new_root().await?;
if let Some(hostname) = launch.hostname.clone() {
@ -137,16 +145,60 @@ impl GuestInit {
self.create_dir("/root", Some(0o0700)).await?;
self.create_dir("/tmp", None).await?;
self.create_dir("/run", Some(0o0755)).await?;
self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755", None)
self.mount_kernel_fs("devtmpfs", "/dev", "mode=0755", None, None)
.await?;
self.mount_kernel_fs("proc", "/proc", "", None, None)
.await?;
self.mount_kernel_fs("sysfs", "/sys", "", None, None)
.await?;
self.create_dir("/dev/pts", Some(0o0755)).await?;
self.mount_kernel_fs("devpts", "/dev/pts", "", None, Some("/dev/ptmx"))
.await?;
self.mount_kernel_fs("proc", "/proc", "", None).await?;
self.mount_kernel_fs("sysfs", "/sys", "", None).await?;
fs::symlink("/proc/self/fd", "/dev/fd").await?;
fs::symlink("/proc/self/fd/0", "/dev/stdin").await?;
fs::symlink("/proc/self/fd/1", "/dev/stdout").await?;
fs::symlink("/proc/self/fd/2", "/dev/stderr").await?;
self.mount_kernel_fs("cgroup2", "/sys/fs/cgroup", "", Some(MountFlags::RELATIME))
.await?;
self.mount_kernel_fs(
"cgroup2",
"/sys/fs/cgroup",
"",
Some(MountFlags::RELATIME),
None,
)
.await?;
Ok(())
}
async fn mount_addons(&mut self) -> Result<()> {
if !fs::try_exists(ADDONS_DEVICE_PATH).await? {
return Ok(());
}
self.mount_image(
&PathBuf::from(ADDONS_DEVICE_PATH),
&PathBuf::from(ADDONS_MOUNT_PATH),
LaunchPackedFormat::Squashfs,
)
.await?;
Ok(())
}
async fn mount_kernel_modules(&mut self) -> Result<()> {
if !fs::try_exists(ADDONS_MODULES_PATH).await? {
return Ok(());
}
let Some(platform_info) = PlatformInfo::new().ok() else {
return Ok(());
};
let kernel_release = platform_info.release().to_string_lossy().to_string();
let modules_path = format!("/newroot/lib/modules/{}", kernel_release);
fs::create_dir_all(&modules_path).await?;
Mount::builder()
.fstype(FilesystemType::Manual("none"))
.flags(MountFlags::BIND | MountFlags::RDONLY)
.mount(ADDONS_MODULES_PATH, modules_path)?;
Ok(())
}
@ -170,13 +222,14 @@ impl GuestInit {
path: &str,
data: &str,
flags: Option<MountFlags>,
source: Option<&str>,
) -> Result<()> {
trace!("mounting kernel fs {} to {}", fstype, path);
Mount::builder()
.fstype(FilesystemType::Manual(fstype))
.flags(MountFlags::NOEXEC | MountFlags::NOSUID | flags.unwrap_or(MountFlags::empty()))
.data(data)
.mount(fstype, path)?;
.mount(source.unwrap_or(fstype), path)?;
Ok(())
}

1
crates/runtime/Cargo.toml
View File

@ -24,6 +24,7 @@ krata-xenclient = { path = "../xen/xenclient", version = "^0.0.10" }
krata-xenevtchn = { path = "../xen/xenevtchn", version = "^0.0.10" }
krata-xengnt = { path = "../xen/xengnt", version = "^0.0.10" }
krata-xenstore = { path = "../xen/xenstore", version = "^0.0.10" }
walkdir = { workspace = true }
[lib]
name = "kratart"

12
crates/runtime/src/cfgblk.rs
View File

@ -1,5 +1,6 @@
use anyhow::Result;
use backhand::{FilesystemWriter, NodeHeader};
use backhand::compression::Compressor;
use backhand::{FilesystemCompressor, FilesystemWriter, NodeHeader};
use krata::launchcfg::LaunchInfo;
use krataoci::packer::OciPackedImage;
use log::trace;
@ -8,14 +9,14 @@ use std::fs::File;
use std::path::PathBuf;
use uuid::Uuid;
pub struct ConfigBlock<'a> {
pub image: &'a OciPackedImage,
pub struct ConfigBlock {
pub image: OciPackedImage,
pub file: PathBuf,
pub dir: PathBuf,
}
impl ConfigBlock<'_> {
pub fn new<'a>(uuid: &Uuid, image: &'a OciPackedImage) -> Result<ConfigBlock<'a>> {
impl ConfigBlock {
pub fn new(uuid: &Uuid, image: OciPackedImage) -> Result<ConfigBlock> {
let mut dir = std::env::temp_dir().clone();
dir.push(format!("krata-cfg-{}", uuid));
fs::create_dir_all(&dir)?;
@ -29,6 +30,7 @@ impl ConfigBlock<'_> {
let config = self.image.config.raw();
let launch = serde_json::to_string(launch_config)?;
let mut writer = FilesystemWriter::default();
writer.set_compressor(FilesystemCompressor::new(Compressor::Gzip, None)?);
writer.push_dir(
"/image",
NodeHeader {

99
crates/runtime/src/launch.rs
View File

@ -1,5 +1,6 @@
use std::collections::HashMap;
use std::net::{IpAddr, Ipv6Addr};
use std::path::PathBuf;
use std::sync::Arc;
use std::{fs, net::Ipv4Addr, str::FromStr};
@ -38,6 +39,7 @@ pub struct GuestLaunchRequest {
pub pcis: Vec<PciDevice>,
pub debug: bool,
pub image: OciPackedImage,
pub addons_image: Option<PathBuf>,
}
pub struct GuestLauncher {
@ -105,8 +107,10 @@ impl GuestLauncher {
run: request.run,
};
let cfgblk = ConfigBlock::new(&uuid, &request.image)?;
cfgblk.build(&launch_config)?;
let cfgblk = ConfigBlock::new(&uuid, request.image.clone())?;
let cfgblk_file = cfgblk.file.clone();
let cfgblk_dir = cfgblk.dir.clone();
tokio::task::spawn_blocking(move || cfgblk.build(&launch_config)).await??;
let image_squashfs_path = request
.image
@ -114,18 +118,33 @@ impl GuestLauncher {
.to_str()
.ok_or_else(|| anyhow!("failed to convert image path to string"))?;
let cfgblk_dir_path = cfgblk
.dir
let cfgblk_dir_path = cfgblk_dir
.to_str()
.ok_or_else(|| anyhow!("failed to convert cfgblk directory path to string"))?;
let cfgblk_squashfs_path = cfgblk
.file
let cfgblk_squashfs_path = cfgblk_file
.to_str()
.ok_or_else(|| anyhow!("failed to convert cfgblk squashfs path to string"))?;
let addons_squashfs_path = request
.addons_image
.map(|x| x.to_str().map(|x| x.to_string()))
.map(|x| {
Some(x.ok_or_else(|| anyhow!("failed to convert addons squashfs path to string")))
})
.unwrap_or(None);
let addons_squashfs_path = if let Some(path) = addons_squashfs_path {
Some(path?)
} else {
None
};
let image_squashfs_loop = context.autoloop.loopify(image_squashfs_path)?;
let cfgblk_squashfs_loop = context.autoloop.loopify(cfgblk_squashfs_path)?;
let addons_squashfs_loop = if let Some(ref addons_squashfs_path) = addons_squashfs_path {
Some(context.autoloop.loopify(addons_squashfs_path)?)
} else {
None
};
let cmdline_options = [
if request.debug { "debug" } else { "quiet" },
"elevator=noop",
@ -135,19 +154,48 @@ impl GuestLauncher {
let guest_mac_string = container_mac.to_string().replace('-', ":");
let gateway_mac_string = gateway_mac.to_string().replace('-', ":");
let mut disks = vec![
DomainDisk {
vdev: "xvda".to_string(),
block: image_squashfs_loop.clone(),
writable: false,
},
DomainDisk {
vdev: "xvdb".to_string(),
block: cfgblk_squashfs_loop.clone(),
writable: false,
},
];
if let Some(ref addons) = addons_squashfs_loop {
disks.push(DomainDisk {
vdev: "xvdc".to_string(),
block: addons.clone(),
writable: false,
});
}
let mut loops = vec![
format!("{}:{}:none", image_squashfs_loop.path, image_squashfs_path),
format!(
"{}:{}:{}",
cfgblk_squashfs_loop.path, cfgblk_squashfs_path, cfgblk_dir_path
),
];
if let Some(ref addons) = addons_squashfs_loop {
loops.push(format!(
"{}:{}:none",
addons.path,
addons_squashfs_path
.clone()
.ok_or_else(|| anyhow!("addons squashfs path missing"))?
));
}
let mut extra_keys = vec![
("krata/uuid".to_string(), uuid.to_string()),
(
"krata/loops".to_string(),
format!(
"{}:{}:none,{}:{}:{}",
&image_squashfs_loop.path,
image_squashfs_path,
&cfgblk_squashfs_loop.path,
cfgblk_squashfs_path,
cfgblk_dir_path,
),
),
("krata/loops".to_string(), loops.join(",")),
(
"krata/network/guest/ipv4".to_string(),
format!("{}/{}", guest_ipv4, ipv4_network_mask),
@ -187,18 +235,7 @@ impl GuestLauncher {
initrd: request.initrd,
cmdline,
use_console_backend: Some("krata-console".to_string()),
disks: vec![
DomainDisk {
vdev: "xvda".to_string(),
block: image_squashfs_loop.clone(),
writable: false,
},
DomainDisk {
vdev: "xvdb".to_string(),
block: cfgblk_squashfs_loop.clone(),
writable: false,
},
],
disks,
channels: vec![DomainChannel {
typ: "krata-channel".to_string(),
initialized: false,
@ -245,7 +282,7 @@ impl GuestLauncher {
Err(error) => {
let _ = context.autoloop.unloop(&image_squashfs_loop.path).await;
let _ = context.autoloop.unloop(&cfgblk_squashfs_loop.path).await;
let _ = fs::remove_dir(&cfgblk.dir);
let _ = fs::remove_dir(&cfgblk_dir);
Err(error.into())
}
}

1
hack/dist/bundle.sh vendored
View File

@ -33,6 +33,7 @@ cd "${BUNDLE_DIR}"
cp "${KRATA_DIR}/target/initrd/initrd-${TARGET_ARCH}" initrd
cp "${KRATA_DIR}/target/kernel/kernel-${TARGET_ARCH}" kernel
cp "${KRATA_DIR}/target/kernel/addons-${TARGET_ARCH}.squashfs" addons.squashfs
cp "${KRATA_DIR}/resources/systemd/kratad.service" kratad.service
cp "${KRATA_DIR}/resources/systemd/kratanet.service" kratanet.service
cp "${KRATA_DIR}/resources/bundle/install.sh" install.sh

1
hack/dist/systar.sh vendored
View File

@ -40,6 +40,7 @@ fi
mkdir -p usr/share/krata/guest
mv ../krata/kernel ../krata/initrd usr/share/krata/guest
mv ../krata/addons.squashfs usr/share/krata/guest/addons.squashfs
tar czf "${SYSTAR}" --owner 0 --group 0 .

21
hack/kernel/build.sh
View File

@ -3,7 +3,7 @@ set -e
REAL_SCRIPT="$(realpath "${0}")"
cd "$(dirname "${REAL_SCRIPT}")/../.."
KRATA_DIR="${PWD}"
KRATA_DIR="$(realpath "${PWD}")"
KERNEL_DIR="${KRATA_DIR}/kernel"
cd "${KRATA_DIR}"
@ -62,7 +62,24 @@ then
IMAGE_TARGET="Image.gz"
fi
make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" -j"${KRATA_KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" "${IMAGE_TARGET}"
make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" -j"${KRATA_KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" "${IMAGE_TARGET}" modules
MODULES_INSTALL_PATH="${OUTPUT_DIR}/modules-install-${TARGET_ARCH_STANDARD}"
MODULES_OUTPUT_PATH="${OUTPUT_DIR}/addons-${TARGET_ARCH_STANDARD}/modules"
ADDONS_SQUASHFS_PATH="${OUTPUT_DIR}/addons-${TARGET_ARCH_STANDARD}.squashfs"
rm -rf "${MODULES_INSTALL_PATH}"
rm -rf "${MODULES_OUTPUT_PATH}"
rm -rf "${MODULES_SQUASHFS_PATH}"
make -C "${KERNEL_SRC}" ARCH="${TARGET_ARCH_KERNEL}" -j"${KRATA_KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" INSTALL_MOD_PATH="${MODULES_INSTALL_PATH}" modules_install
KERNEL_MODULES_VER="$(ls "${MODULES_INSTALL_PATH}/lib/modules")"
mkdir -p "${MODULES_OUTPUT_PATH}"
mv "${MODULES_INSTALL_PATH}/lib/modules/${KERNEL_MODULES_VER}" "${MODULES_OUTPUT_PATH}"
rm -rf "${MODULES_INSTALL_PATH}"
unlink "${MODULES_OUTPUT_PATH}/build"
mksquashfs "${MODULES_OUTPUT_PATH}" "${ADDONS_SQUASHFS_PATH}"
if [ "${TARGET_ARCH_STANDARD}" = "x86_64" ]
then

2424
kernel/krata-x86_64.config
View File

File diff suppressed because it is too large Load Diff