feature(zone): kernel command line control on launch (#351)

.github/workflows/check.yml

@@ -26,9 +26,8 @@ jobs:
         rustup component add rustfmt
     - name: install linux dependencies
       run: ./hack/ci/install-linux-deps.sh
-    # Temporarily ignored: https://github.com/edera-dev/krata/issues/206
     - name: cargo fmt
-      run: ./hack/build/cargo.sh fmt --all -- --check || true
+      run: ./hack/build/cargo.sh fmt --all -- --check
   shellcheck:
     name: shellcheck
     runs-on: ubuntu-latest

crates/ctl/src/cli/host/hv_console.rs

@@ -7,9 +7,8 @@ use krata::v1::control::{
 use tonic::{transport::Channel, Request};
 
 #[derive(Parser)]
-#[command(about = "Display hypervisor diagnostic messages")]
+#[command(about = "Display hypervisor console output")]
-pub struct HostHvConsoleCommand {
+pub struct HostHvConsoleCommand {}
-}
 
 impl HostHvConsoleCommand {
     pub async fn run(self, mut client: ControlServiceClient<Channel>) -> Result<()> {

crates/ctl/src/cli/host/mod.rs

@@ -6,14 +6,14 @@ use krata::events::EventStream;
 use krata::v1::control::control_service_client::ControlServiceClient;
 
 use crate::cli::host::cpu_topology::HostCpuTopologyCommand;
+use crate::cli::host::hv_console::HostHvConsoleCommand;
 use crate::cli::host::identify::HostStatusCommand;
 use crate::cli::host::idm_snoop::HostIdmSnoopCommand;
-use crate::cli::host::hv_console::HostHvConsoleCommand;
 
 pub mod cpu_topology;
+pub mod hv_console;
 pub mod identify;
 pub mod idm_snoop;
-pub mod hv_console;
 
 #[derive(Parser)]
 #[command(about = "Manage the host of the isolation engine")]

crates/ctl/src/cli/mod.rs

@@ -31,6 +31,7 @@ pub struct ControlCommand {
     command: ControlCommands,
 }
 
+#[allow(clippy::large_enum_variant)]
 #[derive(Parser)]
 pub enum ControlCommands {
     Zone(ZoneCommand),
@@ -48,7 +49,11 @@ impl ControlCommand {
 }
 
 impl ControlCommands {
-    pub async fn run(self, client: ControlServiceClient<Channel>, events: EventStream) -> Result<()> {
+    pub async fn run(
+        self,
+        client: ControlServiceClient<Channel>,
+        events: EventStream,
+    ) -> Result<()> {
         match self {
             ControlCommands::Zone(zone) => zone.run(client, events).await,
 

crates/ctl/src/cli/zone/launch.rs

@@ -6,8 +6,8 @@ use krata::{
     events::EventStream,
     v1::{
         common::{
-            zone_image_spec::Image, OciImageFormat, ZoneImageSpec, ZoneOciImageSpec,
+            zone_image_spec::Image, OciImageFormat, ZoneImageSpec, ZoneKernelOptionsSpec,
-            ZoneResourceSpec, ZoneSpec, ZoneSpecDevice, ZoneState, ZoneTaskSpec,
+            ZoneOciImageSpec, ZoneResourceSpec, ZoneSpec, ZoneSpecDevice, ZoneState, ZoneTaskSpec,
             ZoneTaskSpecEnvVar,
         },
         control::{
@@ -91,6 +91,10 @@ pub struct ZoneLaunchCommand {
     initrd: Option<String>,
     #[arg(short = 'w', long, help = "Working directory")]
     working_directory: Option<String>,
+    #[arg(long, help = "Enable verbose logging on the kernel")]
+    kernel_verbose: bool,
+    #[arg(long, help = "Additional kernel cmdline options")]
+    kernel_cmdline_append: Option<String>,
     #[arg(help = "Container image for zone to use")]
     oci: String,
     #[arg(
@@ -166,6 +170,10 @@ impl ZoneLaunchCommand {
                     .iter()
                     .map(|name| ZoneSpecDevice { name: name.clone() })
                     .collect(),
+                kernel_options: Some(ZoneKernelOptionsSpec {
+                    verbose: self.kernel_verbose,
+                    cmdline_append: self.kernel_cmdline_append.clone().unwrap_or_default(),
+                }),
             }),
         };
         let response = client

crates/ctl/src/cli/zone/mod.rs

@@ -26,7 +26,7 @@ pub mod logs;
 pub mod metrics;
 pub mod resolve;
 pub mod top;
-mod update_resources;
+pub mod update_resources;
 pub mod watch;
 
 #[derive(Parser)]
@@ -46,6 +46,7 @@ impl ZoneCommand {
     }
 }
 
+#[allow(clippy::large_enum_variant)]
 #[derive(Subcommand)]
 pub enum ZoneCommands {
     Attach(ZoneAttachCommand),

crates/daemon/src/control.rs

@@ -25,10 +25,10 @@ use krata::{
             ExecInsideZoneRequest, GetHostCpuTopologyReply, GetHostCpuTopologyRequest,
             HostCpuTopologyInfo, HostStatusReply, HostStatusRequest, ListDevicesReply,
             ListDevicesRequest, ListZonesReply, ListZonesRequest, PullImageReply, PullImageRequest,
-            ReadZoneMetricsReply, ReadZoneMetricsRequest, ResolveZoneIdReply, ResolveZoneIdRequest,
+            ReadHypervisorConsoleReply, ReadHypervisorConsoleRequest, ReadZoneMetricsReply,
-            SnoopIdmReply, SnoopIdmRequest, UpdateZoneResourcesReply, UpdateZoneResourcesRequest,
+            ReadZoneMetricsRequest, ResolveZoneIdReply, ResolveZoneIdRequest, SnoopIdmReply,
+            SnoopIdmRequest, UpdateZoneResourcesReply, UpdateZoneResourcesRequest,
             WatchEventsReply, WatchEventsRequest, ZoneConsoleReply, ZoneConsoleRequest,
-            ReadHypervisorConsoleRequest, ReadHypervisorConsoleReply,
         },
     },
 };
@@ -716,9 +716,15 @@ impl ControlService for DaemonControlService {
         &self,
         _request: Request<ReadHypervisorConsoleRequest>,
     ) -> Result<Response<ReadHypervisorConsoleReply>, Status> {
-        let data = self.runtime.read_hypervisor_console(false).await.map_err(|error| ApiError {
+        let data = self
-            message: error.to_string(),
+            .runtime
-        })?;
+            .read_hypervisor_console(false)
-        Ok(Response::new(ReadHypervisorConsoleReply { data: data.to_string() }))
+            .await
+            .map_err(|error| ApiError {
+                message: error.to_string(),
+            })?;
+        Ok(Response::new(ReadHypervisorConsoleReply {
+            data: data.to_string(),
+        }))
     }
 }

crates/daemon/src/reconcile/zone/create.rs

@@ -184,10 +184,19 @@ impl ZoneCreator<'_> {
             initial_resources.max_cpus = initial_resources.target_cpus;
         }
         spec.initial_resources = Some(initial_resources);
+        let kernel_options = spec.kernel_options.clone().unwrap_or_default();
         let info = self
             .runtime
             .launch(ZoneLaunchRequest {
-                format: LaunchPackedFormat::Squashfs,
+                format: match image.format {
+                    OciPackedFormat::Squashfs => LaunchPackedFormat::Squashfs,
+                    OciPackedFormat::Erofs => LaunchPackedFormat::Erofs,
+                    _ => {
+                        return Err(anyhow!(
+                            "oci image is in an invalid format, which isn't compatible with launch"
+                        ));
+                    }
+                },
                 uuid: Some(uuid),
                 name: if spec.name.is_empty() {
                     None
@@ -208,7 +217,8 @@ impl ZoneCreator<'_> {
                     .map(|x| (x.key.clone(), x.value.clone()))
                     .collect::<HashMap<_, _>>(),
                 run: empty_vec_optional(task.command.clone()),
-                debug: false,
+                kernel_verbose: kernel_options.verbose,
+                kernel_cmdline_append: kernel_options.cmdline_append,
                 addons_image: Some(self.addons_path.to_path_buf()),
                 network: ZoneLaunchNetwork {
                     ipv4: reservation.ipv4.to_string(),

crates/krata/proto/krata/v1/common.proto

@@ -25,6 +25,7 @@ message ZoneSpec {
     ZoneTaskSpec task = 6;
     repeated ZoneSpecAnnotation annotations = 7;
     repeated ZoneSpecDevice devices = 8;
+    ZoneKernelOptionsSpec kernel_options = 9;
 }
 
 message ZoneResourceSpec {
@@ -40,6 +41,11 @@ message ZoneImageSpec {
     }
 }
 
+message ZoneKernelOptionsSpec {
+    bool verbose = 1;
+    string cmdline_append = 2;
+}
+
 enum OciImageFormat {
     OCI_IMAGE_FORMAT_UNKNOWN = 0;
     OCI_IMAGE_FORMAT_SQUASHFS = 1;

crates/runtime/src/launch.rs

@@ -37,7 +37,8 @@ pub struct ZoneLaunchRequest {
     pub env: HashMap<String, String>,
     pub run: Option<Vec<String>>,
     pub pcis: Vec<PciDevice>,
-    pub debug: bool,
+    pub kernel_verbose: bool,
+    pub kernel_cmdline_append: String,
     pub image: OciPackedImage,
     pub addons_image: Option<PathBuf>,
     pub network: ZoneLaunchNetwork,
@@ -139,9 +140,14 @@ impl ZoneLauncher {
             None
         };
         let mut cmdline_options = ["console=hvc0"].to_vec();
-        if !request.debug {
+        if !request.kernel_verbose {
             cmdline_options.push("quiet");
         }
+
+        if !request.kernel_cmdline_append.is_empty() {
+            cmdline_options.push(&request.kernel_cmdline_append);
+        }
+
         let cmdline = cmdline_options.join(" ");
 
         let zone_mac_string = request.network.zone_mac.to_string().replace('-', ":");

crates/runtime/src/lib.rs

@@ -307,11 +307,13 @@ impl Runtime {
     }
 
     pub async fn read_hypervisor_console(&self, clear: bool) -> Result<Arc<str>> {
-        let index = 0 as u32;
+        let index = 0_u32;
-        let (rawbuf, newindex) = self.context
+        let (rawbuf, newindex) = self
-                                     .xen
+            .context
-                                     .call
+            .xen
-                                     .read_console_ring_raw(clear, index).await?;
+            .call
+            .read_console_ring_raw(clear, index)
+            .await?;
         let buf = std::str::from_utf8(&rawbuf[..newindex as usize])?;
         Ok(Arc::from(buf))
     }

crates/xen/xencall/examples/console_read.rs

@@ -6,7 +6,7 @@ async fn main() -> Result<()> {
     env_logger::init();
 
     let call = XenCall::open(0)?;
-    let index = 0 as u32;
+    let index = 0_u32;
     let (buf, newindex) = call.read_console_ring_raw(false, index).await?;
 
     match std::str::from_utf8(&buf[..newindex as usize]) {

crates/xen/xencall/src/lib.rs

@@ -26,8 +26,8 @@ use std::sync::Arc;
 use std::time::Duration;
 use sys::{
     CpuId, E820Entry, ForeignMemoryMap, PhysdevMapPirq, Sysctl, SysctlCputopo, SysctlCputopoinfo,
-    SysctlPhysinfo, SysctlPmOp, SysctlPmOpValue, SysctlSetCpuFreqGov, SysctlValue, SysctlReadconsole,
+    SysctlPhysinfo, SysctlPmOp, SysctlPmOpValue, SysctlReadconsole, SysctlSetCpuFreqGov,
-    VcpuGuestContextAny, HYPERVISOR_PHYSDEV_OP, HYPERVISOR_SYSCTL, PHYSDEVOP_MAP_PIRQ,
+    SysctlValue, VcpuGuestContextAny, HYPERVISOR_PHYSDEV_OP, HYPERVISOR_SYSCTL, PHYSDEVOP_MAP_PIRQ,
     XEN_DOMCTL_MAX_INTERFACE_VERSION, XEN_DOMCTL_MIN_INTERFACE_VERSION, XEN_MEM_SET_MEMORY_MAP,
     XEN_SYSCTL_CPUTOPOINFO, XEN_SYSCTL_MAX_INTERFACE_VERSION, XEN_SYSCTL_MIN_INTERFACE_VERSION,
     XEN_SYSCTL_PHYSINFO, XEN_SYSCTL_PM_OP, XEN_SYSCTL_PM_OP_DISABLE_TURBO,
@@ -1088,7 +1088,11 @@ impl XenCall {
         Ok(())
     }
 
-    pub async fn read_console_ring_raw(&self, clear: bool, index: u32) -> Result<([u8; 16384], u32)> {
+    pub async fn read_console_ring_raw(
+        &self,
+        clear: bool,
+        index: u32,
+    ) -> Result<([u8; 16384], u32)> {
         let mut u8buf = [0u8; 16384];
         let mut sysctl = Sysctl {
             cmd: XEN_SYSCTL_READCONSOLE,
@@ -1098,7 +1102,7 @@ impl XenCall {
                     clear: clear as u8,
                     incremental: 1,
                     pad: 0,
-                    index: index,
+                    index,
                     buffer: addr_of_mut!(u8buf) as u64,
                     count: 16384,
                 },
@@ -1109,9 +1113,7 @@ impl XenCall {
         // Safety: We are passing a SysctlReadconsole struct as part of the hypercall, and
         // calling the hypercall is known to not change the underlying value outside changing
         // the values on some SysctlReadconsole fields.
-        let newindex = unsafe {
+        let newindex = unsafe { sysctl.value.console.index };
-            sysctl.value.console.index
-        };
         Ok((u8buf, newindex))
     }
 }