Power management core functionality (#217)

* feat(power-management-core): add core power management control messages for kratad

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): expose xen hypercall client publicly

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): add indexmap to kratart crate dependencies

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): implement power management core in kratart

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): bubble up runtime context in daemon/control service

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): expose performance/efficiency core data in protobuf

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): fix up some protobuf message names

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): fix up performance core heuristic

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): implement GetHostCpuTopology RPC

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): hackfix to get sysctls working with tokio

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): borrow the PowerManagementContext when calling functions belonging to it

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): remove GetHostPowerManagementPolicy RPC for now

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): implement SetHostPowerManagementPolicy RPC

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): add cpu-topology command

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* feat(power-management-core): appease format checking

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>

* fix(runtime): cpu topology corrections

---------

Signed-off-by: Ariadne Conill <ariadne@ariadne.space>
Co-authored-by: Alex Zenla <alex@edera.dev>

Cargo.lock

@@ -1545,11 +1545,13 @@ dependencies = [
  "anyhow",
  "backhand",
  "env_logger",
+ "indexmap 2.2.6",
  "ipnetwork",
  "krata",
  "krata-advmac",
  "krata-loopdev",
  "krata-oci",
+ "krata-xencall",
  "krata-xenclient",
  "krata-xenevtchn",
  "krata-xengnt",

crates/ctl/src/cli/cpu_topology.rs

@@ -0,0 +1,46 @@
+use anyhow::Result;
+use clap::Parser;
+use krata::v1::control::{control_service_client::ControlServiceClient, HostCpuTopologyRequest};
+
+use tonic::{transport::Channel, Request};
+
+fn class_to_str(input: i32) -> String {
+    match input {
+        0 => "Standard".to_string(),
+        1 => "Performance".to_string(),
+        2 => "Efficiency".to_string(),
+        _ => "???".to_string(),
+    }
+}
+
+#[derive(Parser)]
+#[command(about = "Display information about a host's CPU topology")]
+pub struct CpuTopologyCommand {}
+
+impl CpuTopologyCommand {
+    pub async fn run(self, mut client: ControlServiceClient<Channel>) -> Result<()> {
+        println!(
+            "{0:<10} {1:<10} {2:<10} {3:<10} {4:<10} {5:<10}",
+            "CPUID", "Node", "Socket", "Core", "Thread", "Class"
+        );
+
+        let response = client
+            .get_host_cpu_topology(Request::new(HostCpuTopologyRequest {}))
+            .await?
+            .into_inner();
+
+        for (i, cpu) in response.cpus.iter().enumerate() {
+            println!(
+                "{0:<10} {1:<10} {2:<10} {3:<10} {4:<10} {5:<10}",
+                i,
+                cpu.node,
+                cpu.socket,
+                cpu.core,
+                cpu.thread,
+                class_to_str(cpu.class)
+            );
+        }
+
+        Ok(())
+    }
+}

crates/ctl/src/cli/mod.rs

@@ -1,4 +1,5 @@
 pub mod attach;
+pub mod cpu_topology;
 pub mod destroy;
 pub mod exec;
 pub mod identify_host;
@@ -23,9 +24,9 @@ use krata::{
 use tonic::{transport::Channel, Request};
 
 use self::{
-    attach::AttachCommand, destroy::DestroyCommand, exec::ExecCommand,
+    attach::AttachCommand, cpu_topology::CpuTopologyCommand, destroy::DestroyCommand,
-    identify_host::IdentifyHostCommand, idm_snoop::IdmSnoopCommand, launch::LaunchCommand,
+    exec::ExecCommand, identify_host::IdentifyHostCommand, idm_snoop::IdmSnoopCommand,
-    list::ListCommand, list_devices::ListDevicesCommand, logs::LogsCommand,
+    launch::LaunchCommand, list::ListCommand, list_devices::ListDevicesCommand, logs::LogsCommand,
     metrics::MetricsCommand, pull::PullCommand, resolve::ResolveCommand, top::TopCommand,
     watch::WatchCommand,
 };
@@ -61,6 +62,7 @@ pub enum Commands {
     Top(TopCommand),
     IdentifyHost(IdentifyHostCommand),
     Exec(ExecCommand),
+    CpuTopology(CpuTopologyCommand),
 }
 
 impl ControlCommand {
@@ -124,6 +126,10 @@ impl ControlCommand {
             Commands::ListDevices(list) => {
                 list.run(client, events).await?;
             }
+
+            Commands::CpuTopology(cpu_topology) => {
+                cpu_topology.run(client).await?;
+            }
         }
         Ok(())
     }

crates/daemon/bin/daemon.rs

@@ -20,20 +20,10 @@ async fn main() -> Result<()> {
         .filter(Some("backhand::filesystem::writer"), LevelFilter::Warn);
 
     if let Ok(f_addr) = std::env::var("KRATA_FLUENT_ADDR") {
-        println!("KRATA_FLUENT_ADDR set to {f_addr}");
         let target = SocketAddr::from_str(f_addr.as_str())?;
         builder.target(Target::Pipe(Box::new(TcpStream::connect(target)?)));
     }
 
-    let ev = std::env::vars()
-        .into_iter()
-        .fold(String::new(), |mut acc, (k, v)| {
-            acc.push_str(&format!("{k}={v}\n"));
-            acc
-        });
-
-    std::fs::write("/var/log/krata/ev", ev)?;
-
     builder.init();
 
     mask_sighup()?;

crates/daemon/src/control.rs

@@ -11,11 +11,12 @@ use krata::{
         control::{
             control_service_server::ControlService, ConsoleDataReply, ConsoleDataRequest,
             CreateGuestReply, CreateGuestRequest, DestroyGuestReply, DestroyGuestRequest,
-            DeviceInfo, ExecGuestReply, ExecGuestRequest, IdentifyHostReply, IdentifyHostRequest,
+            DeviceInfo, ExecGuestReply, ExecGuestRequest, HostCpuTopologyInfo,
-            ListDevicesReply, ListDevicesRequest, ListGuestsReply, ListGuestsRequest,
+            HostCpuTopologyReply, HostCpuTopologyRequest, HostPowerManagementPolicy,
-            PullImageReply, PullImageRequest, ReadGuestMetricsReply, ReadGuestMetricsRequest,
+            IdentifyHostReply, IdentifyHostRequest, ListDevicesReply, ListDevicesRequest,
-            ResolveGuestReply, ResolveGuestRequest, SnoopIdmReply, SnoopIdmRequest,
+            ListGuestsReply, ListGuestsRequest, PullImageReply, PullImageRequest,
-            WatchEventsReply, WatchEventsRequest,
+            ReadGuestMetricsReply, ReadGuestMetricsRequest, ResolveGuestReply, ResolveGuestRequest,
+            SnoopIdmReply, SnoopIdmRequest, WatchEventsReply, WatchEventsRequest,
         },
     },
 };
@@ -24,6 +25,7 @@ use krataoci::{
     packer::{service::OciPackerService, OciPackedFormat, OciPackedImage},
     progress::{OciProgress, OciProgressContext},
 };
+use kratart::Runtime;
 use std::{pin::Pin, str::FromStr};
 use tokio::{
     select,
@@ -68,6 +70,7 @@ pub struct DaemonControlService {
     guests: GuestStore,
     guest_reconciler_notify: Sender<Uuid>,
     packer: OciPackerService,
+    runtime: Runtime,
 }
 
 impl DaemonControlService {
@@ -81,6 +84,7 @@ impl DaemonControlService {
         guests: GuestStore,
         guest_reconciler_notify: Sender<Uuid>,
         packer: OciPackerService,
+        runtime: Runtime,
     ) -> Self {
         Self {
             glt,
@@ -91,6 +95,7 @@ impl DaemonControlService {
             guests,
             guest_reconciler_notify,
             packer,
+            runtime,
         }
     }
 }
@@ -548,4 +553,57 @@ impl ControlService for DaemonControlService {
         }
         Ok(Response::new(ListDevicesReply { devices }))
     }
+
+    async fn get_host_cpu_topology(
+        &self,
+        request: Request<HostCpuTopologyRequest>,
+    ) -> Result<Response<HostCpuTopologyReply>, Status> {
+        let _ = request.into_inner();
+        let power = self
+            .runtime
+            .power_management_context()
+            .await
+            .map_err(ApiError::from)?;
+        let cputopo = power.cpu_topology().await.map_err(ApiError::from)?;
+        let mut cpus = vec![];
+
+        for cpu in cputopo {
+            cpus.push(HostCpuTopologyInfo {
+                core: cpu.core,
+                socket: cpu.socket,
+                node: cpu.node,
+                thread: cpu.thread,
+                class: cpu.class as i32,
+            })
+        }
+
+        Ok(Response::new(HostCpuTopologyReply { cpus }))
+    }
+
+    async fn set_host_power_management_policy(
+        &self,
+        request: Request<HostPowerManagementPolicy>,
+    ) -> Result<Response<HostPowerManagementPolicy>, Status> {
+        let policy = request.into_inner();
+        let power = self
+            .runtime
+            .power_management_context()
+            .await
+            .map_err(ApiError::from)?;
+        let scheduler = &policy.scheduler;
+
+        power
+            .set_smt_policy(policy.smt_awareness)
+            .await
+            .map_err(ApiError::from)?;
+        power
+            .set_scheduler_policy(scheduler)
+            .await
+            .map_err(ApiError::from)?;
+
+        Ok(Response::new(HostPowerManagementPolicy {
+            scheduler: scheduler.to_string(),
+            smt_awareness: policy.smt_awareness,
+        }))
+    }
 }

crates/daemon/src/lib.rs

@@ -50,6 +50,7 @@ pub struct Daemon {
     idm: DaemonIdmHandle,
     console: DaemonConsoleHandle,
     packer: OciPackerService,
+    runtime: Runtime,
 }
 
 const GUEST_RECONCILER_QUEUE_LEN: usize = 1000;
@@ -136,6 +137,7 @@ impl Daemon {
             idm,
             console,
             packer,
+            runtime,
         })
     }
 
@@ -149,6 +151,7 @@ impl Daemon {
             self.guests.clone(),
             self.guest_reconciler_notify.clone(),
             self.packer.clone(),
+            self.runtime.clone(),
         );
 
         let mut server = Server::builder();

crates/krata/proto/krata/v1/control.proto

@@ -27,6 +27,9 @@ service ControlService {
     rpc WatchEvents(WatchEventsRequest) returns (stream WatchEventsReply);
 
     rpc PullImage(PullImageRequest) returns (stream PullImageReply);
+
+    rpc GetHostCpuTopology(HostCpuTopologyRequest) returns (HostCpuTopologyReply);
+    rpc SetHostPowerManagementPolicy(HostPowerManagementPolicy) returns (HostPowerManagementPolicy);
 }
 
 message IdentifyHostRequest {}
@@ -200,3 +203,30 @@ message ListDevicesRequest {}
 message ListDevicesReply {
     repeated DeviceInfo devices = 1;
 }
+
+enum HostCpuTopologyClass {
+    CPU_CLASS_STANDARD = 0;
+    CPU_CLASS_PERFORMANCE = 1;
+    CPU_CLASS_EFFICIENCY = 2;
+}
+
+message HostCpuTopologyInfo {
+    uint32 core = 1;
+    uint32 socket = 2;
+    uint32 node = 3;
+    uint32 thread = 4;
+    HostCpuTopologyClass class = 5;
+}
+
+message HostCpuTopologyRequest {}
+
+message HostCpuTopologyReply {
+    repeated HostCpuTopologyInfo cpus = 1;
+}
+
+message HostPowerManagementPolicyRequest {}
+
+message HostPowerManagementPolicy {
+    string scheduler = 1;
+    bool smt_awareness = 2;
+}

crates/loopdev/src/lib.rs

@@ -223,7 +223,6 @@ impl LoopDevice {
         self.device
             .metadata()
             .map(|m| unsafe { libc::major(m.rdev()) })
-            .map(|m| m as u32)
     }
 
     /// Return the minor device node number.
@@ -231,7 +230,6 @@ impl LoopDevice {
         self.device
             .metadata()
             .map(|m| unsafe { libc::minor(m.rdev()) })
-            .map(|m| m as u32)
     }
 }
 
@@ -327,11 +325,7 @@ impl AttachOptions<'_> {
     }
 
     pub fn direct_io(&self) -> bool {
-        if (self.info.lo_flags & LO_FLAGS_DIRECT_IO) == LO_FLAGS_DIRECT_IO {
+        (self.info.lo_flags & LO_FLAGS_DIRECT_IO) == LO_FLAGS_DIRECT_IO
-            true
-        } else {
-            false
-        }
     }
 
     pub fn attach(&self, backing_file: impl AsRef<Path>) -> io::Result<()> {

crates/runtime/Cargo.toml

@@ -20,12 +20,14 @@ serde_json = { workspace = true }
 tokio = { workspace = true }
 uuid = { workspace = true }
 krata-loopdev = { path = "../loopdev", version = "^0.0.11" }
+krata-xencall = { path = "../xen/xencall", version = "^0.0.11" }
 krata-xenclient = { path = "../xen/xenclient", version = "^0.0.11" }
 krata-xenevtchn = { path = "../xen/xenevtchn", version = "^0.0.11" }
 krata-xengnt = { path = "../xen/xengnt", version = "^0.0.11" }
 krata-xenplatform = { path = "../xen/xenplatform", version = "^0.0.11" }
 krata-xenstore = { path = "../xen/xenstore", version = "^0.0.11" }
 walkdir = { workspace = true }
+indexmap = { workspace = true }
 
 [lib]
 name = "kratart"

crates/runtime/src/lib.rs

@@ -13,6 +13,7 @@ use xenstore::{XsdClient, XsdInterface};
 use self::{
     autoloop::AutoLoop,
     launch::{GuestLaunchRequest, GuestLauncher},
+    power::PowerManagementContext,
 };
 
 pub mod autoloop;
@@ -20,6 +21,7 @@ pub mod cfgblk;
 pub mod channel;
 pub mod ip;
 pub mod launch;
+pub mod power;
 
 #[cfg(target_arch = "x86_64")]
 type RuntimePlatform = xenplatform::x86pv::X86PvPlatform;
@@ -321,4 +323,9 @@ impl Runtime {
     pub async fn dupe(&self) -> Result<Runtime> {
         Runtime::new(self.host_uuid).await
     }
+
+    pub async fn power_management_context(&self) -> Result<PowerManagementContext> {
+        let context = RuntimeContext::new(self.host_uuid).await?;
+        Ok(PowerManagementContext { context })
+    }
 }

crates/runtime/src/power.rs

@@ -0,0 +1,162 @@
+use anyhow::Result;
+use indexmap::IndexMap;
+use xencall::sys::{CpuId, SysctlCputopo};
+
+use crate::RuntimeContext;
+
+#[derive(Clone)]
+pub struct PowerManagementContext {
+    pub context: RuntimeContext,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum CpuClass {
+    Standard,
+    Performance,
+    Efficiency,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct CpuTopologyInfo {
+    pub core: u32,
+    pub socket: u32,
+    pub node: u32,
+    pub thread: u32,
+    pub class: CpuClass,
+}
+
+fn labelled_topo(input: &[SysctlCputopo]) -> Vec<CpuTopologyInfo> {
+    let mut cores: IndexMap<(u32, u32, u32), Vec<CpuTopologyInfo>> = IndexMap::new();
+    let mut pe_cores = false;
+    let mut last: Option<SysctlCputopo> = None;
+
+    for item in input {
+        if cores.is_empty() {
+            cores.insert(
+                (item.core, item.socket, item.node),
+                vec![CpuTopologyInfo {
+                    core: item.core,
+                    socket: item.socket,
+                    thread: 0,
+                    node: item.node,
+                    class: CpuClass::Standard,
+                }],
+            );
+            last = Some(*item);
+            continue;
+        }
+
+        if last
+            .map(|last| (item.core - last.core) >= 2)
+            .unwrap_or(false)
+        {
+            // detect if performance cores seem to be kicking in.
+            if let Some(last) = last {
+                if let Some(list) = cores.get_mut(&(last.core, last.socket, last.node)) {
+                    for other in list {
+                        other.class = CpuClass::Performance;
+                    }
+                }
+            }
+            let list = cores
+                .entry((item.core, item.socket, item.node))
+                .or_default();
+            for old in &mut *list {
+                old.class = CpuClass::Performance;
+            }
+            list.push(CpuTopologyInfo {
+                core: item.core,
+                socket: item.socket,
+                thread: 0,
+                node: item.node,
+                class: CpuClass::Performance,
+            });
+            pe_cores = true;
+        } else if pe_cores && last.map(|last| item.core == last.core + 1).unwrap_or(false) {
+            // detect efficiency cores if P/E cores are in use.
+            if let Some(last) = last {
+                if let Some(list) = cores.get_mut(&(last.core, last.socket, last.node)) {
+                    for other in list {
+                        other.class = CpuClass::Efficiency;
+                    }
+                }
+            }
+            let list = cores
+                .entry((item.core, item.socket, item.node))
+                .or_default();
+            list.push(CpuTopologyInfo {
+                core: item.core,
+                socket: item.socket,
+                thread: 0,
+                node: item.node,
+                class: CpuClass::Efficiency,
+            });
+        } else {
+            let list = cores
+                .entry((item.core, item.socket, item.node))
+                .or_default();
+            if list.is_empty() {
+                list.push(CpuTopologyInfo {
+                    core: item.core,
+                    socket: item.socket,
+                    thread: 0,
+                    node: item.node,
+                    class: CpuClass::Standard,
+                });
+            } else {
+                list.push(CpuTopologyInfo {
+                    core: item.core,
+                    socket: item.socket,
+                    thread: 0,
+                    node: item.node,
+                    class: list
+                        .first()
+                        .map(|first| first.class)
+                        .unwrap_or(CpuClass::Standard),
+                });
+            }
+        }
+        last = Some(*item);
+    }
+
+    for threads in cores.values_mut() {
+        for (index, thread) in threads.iter_mut().enumerate() {
+            thread.thread = index as u32;
+        }
+    }
+
+    cores.into_values().flatten().collect::<Vec<_>>()
+}
+
+impl PowerManagementContext {
+    /// Get the CPU topology, with SMT awareness.
+    /// Also translates Intel p-core/e-core nonsense: non-sequential core identifiers
+    /// are treated as p-cores, while e-cores behave as standard cores.
+    /// If there is a p-core/e-core split, then CPU class will be defined as
+    /// `CpuClass::Performance` or `CpuClass::Efficiency`, else `CpuClass::Standard`.
+    pub async fn cpu_topology(&self) -> Result<Vec<CpuTopologyInfo>> {
+        let xentopo = self.context.xen.call.cpu_topology().await?;
+        let logicaltopo = labelled_topo(&xentopo);
+        Ok(logicaltopo)
+    }
+
+    /// Enable or disable SMT awareness in the scheduler.
+    pub async fn set_smt_policy(&self, enable: bool) -> Result<()> {
+        self.context
+            .xen
+            .call
+            .set_turbo_mode(CpuId::All, enable)
+            .await?;
+        Ok(())
+    }
+
+    /// Set scheduler policy name.
+    pub async fn set_scheduler_policy(&self, policy: impl AsRef<str>) -> Result<()> {
+        self.context
+            .xen
+            .call
+            .set_cpufreq_gov(CpuId::All, policy)
+            .await?;
+        Ok(())
+    }
+}

crates/xen/xencall/examples/cputopo.rs

@@ -0,0 +1,19 @@
+use xencall::error::Result;
+use xencall::sys::CpuId;
+use xencall::XenCall;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    env_logger::init();
+
+    let call = XenCall::open(0)?;
+    let physinfo = call.phys_info().await?;
+    println!("{:?}", physinfo);
+    let topology = call.cpu_topology().await?;
+    println!("{:?}", topology);
+    call.set_cpufreq_gov(CpuId::All, "performance").await?;
+    call.set_cpufreq_gov(CpuId::Single(0), "performance")
+        .await?;
+    call.set_turbo_mode(CpuId::All, true).await?;
+    Ok(())
+}

crates/xen/xencall/src/lib.rs

@@ -98,7 +98,7 @@ impl XenCall {
                 value: SysctlValue {
                     cputopoinfo: SysctlCputopoinfo {
                         num_cpus: 0,
-                        handle: null_mut(),
+                        handle: 0,
                     },
                 },
             };
@@ -958,7 +958,7 @@ impl XenCall {
             value: SysctlValue {
                 cputopoinfo: SysctlCputopoinfo {
                     num_cpus: 0,
-                    handle: null_mut(),
+                    handle: 0,
                 },
             },
         };
@@ -979,7 +979,7 @@ impl XenCall {
             value: SysctlValue {
                 cputopoinfo: SysctlCputopoinfo {
                     num_cpus: cpus,
-                    handle: topos.as_mut_ptr(),
+                    handle: topos.as_mut_ptr() as c_ulong,
                 },
             },
         };

crates/xen/xencall/src/sys.rs

@@ -747,7 +747,7 @@ pub struct SysctlPmOp {
 #[derive(Clone, Copy, Debug)]
 pub struct SysctlCputopoinfo {
     pub num_cpus: u32,
-    pub handle: *mut SysctlCputopo,
+    pub handle: c_ulong,
 }
 
 #[repr(C)]

crates/xen/xenclient/src/lib.rs

@@ -21,7 +21,7 @@ pub mod tx;
 #[derive(Clone)]
 pub struct XenClient<P: BootSetupPlatform> {
     pub store: XsdClient,
-    call: XenCall,
+    pub call: XenCall,
     domain_manager: Arc<BaseDomainManager<P>>,
 }