From 3e9470afaa7274a6fd2be9f023d3b4ba5085ecb7 Mon Sep 17 00:00:00 2001 From: Alex Zenla Date: Mon, 15 Jan 2024 18:50:12 -0800 Subject: [PATCH] more debugging of page table issues --- xencall/src/domctl.rs | 38 +++++++++++++++++++++++++--- xencall/src/memory.rs | 45 +++++++++++++++++++++++++++++++-- xencall/src/sys.rs | 26 +++++++++++++++++++ xenclient/examples/boot.rs | 1 - xenclient/src/boot.rs | 51 ++++++++++++++++++++++++++++++++------ xenclient/src/mem.rs | 26 +++++++++++++++++++ xenclient/src/x86.rs | 50 +++++++++++++++++++++++++++++++++++++ 7 files changed, 223 insertions(+), 14 deletions(-) diff --git a/xencall/src/domctl.rs b/xencall/src/domctl.rs index aa601d1..314c3f2 100644 --- a/xencall/src/domctl.rs +++ b/xencall/src/domctl.rs @@ -1,9 +1,10 @@ use crate::sys::{ AddressSize, ArchDomainConfig, CreateDomain, DomCtl, DomCtlValue, DomCtlVcpuContext, - GetDomainInfo, HypercallInit, MaxMem, MaxVcpus, VcpuGuestContext, VcpuGuestContextAny, - HYPERVISOR_DOMCTL, XEN_DOMCTL_CREATEDOMAIN, XEN_DOMCTL_DESTROYDOMAIN, XEN_DOMCTL_GETDOMAININFO, - XEN_DOMCTL_GETVCPUCONTEXT, XEN_DOMCTL_HYPERCALL_INIT, XEN_DOMCTL_INTERFACE_VERSION, - XEN_DOMCTL_MAX_MEM, XEN_DOMCTL_MAX_VCPUS, XEN_DOMCTL_PAUSEDOMAIN, XEN_DOMCTL_SETVCPUCONTEXT, + GetDomainInfo, GetPageFrameInfo3, HypercallInit, MaxMem, MaxVcpus, VcpuGuestContext, + VcpuGuestContextAny, HYPERVISOR_DOMCTL, XEN_DOMCTL_CREATEDOMAIN, XEN_DOMCTL_DESTROYDOMAIN, + XEN_DOMCTL_GETDOMAININFO, XEN_DOMCTL_GETPAGEFRAMEINFO3, XEN_DOMCTL_GETVCPUCONTEXT, + XEN_DOMCTL_HYPERCALL_INIT, XEN_DOMCTL_INTERFACE_VERSION, XEN_DOMCTL_MAX_MEM, + XEN_DOMCTL_MAX_VCPUS, XEN_DOMCTL_PAUSEDOMAIN, XEN_DOMCTL_SETVCPUCONTEXT, XEN_DOMCTL_SET_ADDRESS_SIZE, }; use crate::{XenCall, XenCallError}; @@ -11,6 +12,7 @@ use log::trace; use std::ffi::c_ulong; use std::os::fd::AsRawFd; use std::ptr::addr_of_mut; +use std::slice; pub struct DomainControl<'a> { call: &'a XenCall, @@ -214,6 +216,34 @@ impl DomainControl<'_> { Ok(()) } + pub fn get_page_frame_info( + &self, + domid: u32, + frames: &[u64], + ) -> Result, XenCallError> { + let mut buffer: Vec = frames.to_vec(); + let mut domctl = DomCtl { + cmd: XEN_DOMCTL_GETPAGEFRAMEINFO3, + interface_version: XEN_DOMCTL_INTERFACE_VERSION, + domid, + value: DomCtlValue { + get_page_frame_info: GetPageFrameInfo3 { + num: buffer.len() as u64, + array: buffer.as_mut_ptr() as c_ulong, + }, + }, + }; + self.call + .hypercall1(HYPERVISOR_DOMCTL, addr_of_mut!(domctl) as c_ulong)?; + let slice = unsafe { + slice::from_raw_parts_mut( + domctl.value.get_page_frame_info.array as *mut u64, + domctl.value.get_page_frame_info.num as usize, + ) + }; + Ok(slice.to_vec()) + } + pub fn hypercall_init(&self, domid: u32, gmfn: u64) -> Result<(), XenCallError> { trace!( "domctl fd={} hypercall_init domid={} gmfn={}", diff --git a/xencall/src/memory.rs b/xencall/src/memory.rs index 6e73604..304a3df 100644 --- a/xencall/src/memory.rs +++ b/xencall/src/memory.rs @@ -1,10 +1,11 @@ use crate::sys::{ - MemoryReservation, MultiCallEntry, HYPERVISOR_MEMORY_OP, XEN_MEM_POPULATE_PHYSMAP, + MemoryMap, MemoryReservation, MmuExtOp, MultiCallEntry, HYPERVISOR_MEMORY_OP, + HYPERVISOR_MMUEXT_OP, XEN_MEM_MEMORY_MAP, XEN_MEM_POPULATE_PHYSMAP, }; use crate::{XenCall, XenCallError}; use log::trace; -use std::ffi::c_ulong; +use std::ffi::{c_uint, c_ulong}; use std::os::fd::AsRawFd; use std::ptr::addr_of_mut; @@ -17,6 +18,26 @@ impl MemoryControl<'_> { MemoryControl { call } } + pub fn get_memory_map(&self, size_of_entry: usize) -> Result, XenCallError> { + let mut memory_map = MemoryMap { + count: 0, + buffer: 0, + }; + self.call.hypercall2( + HYPERVISOR_MEMORY_OP, + XEN_MEM_MEMORY_MAP as c_ulong, + addr_of_mut!(memory_map) as c_ulong, + )?; + let mut buffer = vec![0u8; memory_map.count as usize * size_of_entry]; + memory_map.buffer = buffer.as_mut_ptr() as c_ulong; + self.call.hypercall2( + HYPERVISOR_MEMORY_OP, + XEN_MEM_MEMORY_MAP as c_ulong, + addr_of_mut!(memory_map) as c_ulong, + )?; + Ok(buffer) + } + pub fn populate_physmap( &self, domid: u32, @@ -62,4 +83,24 @@ impl MemoryControl<'_> { let extents = extent_starts[0..code as usize].to_vec(); Ok(extents) } + + pub fn mmuext( + &self, + domid: u32, + cmd: c_uint, + arg1: u64, + arg2: u64, + ) -> Result<(), XenCallError> { + let mut ops = MmuExtOp { cmd, arg1, arg2 }; + + self.call + .hypercall4( + HYPERVISOR_MMUEXT_OP, + addr_of_mut!(ops) as c_ulong, + 1, + 0, + domid as c_ulong, + ) + .map(|_| ()) + } } diff --git a/xencall/src/sys.rs b/xencall/src/sys.rs index 73c6966..3875ff5 100644 --- a/xencall/src/sys.rs +++ b/xencall/src/sys.rs @@ -223,6 +223,7 @@ pub union DomCtlValue { pub hypercall_init: HypercallInit, pub vcpu_context: DomCtlVcpuContext, pub address_size: AddressSize, + pub get_page_frame_info: GetPageFrameInfo3, pub pad: [u8; 128], } @@ -287,6 +288,13 @@ pub struct GetDomainInfo { pub arch: ArchDomainConfig, } +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub struct GetPageFrameInfo3 { + pub num: u64, + pub array: c_ulong, +} + #[repr(C)] #[derive(Copy, Clone, Debug)] pub struct ArchDomainConfig { @@ -342,6 +350,14 @@ pub struct MultiCallEntry { } pub const XEN_MEM_POPULATE_PHYSMAP: u32 = 6; +pub const XEN_MEM_MEMORY_MAP: u32 = 9; + +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub struct MemoryMap { + pub count: c_uint, + pub buffer: c_ulong, +} #[repr(C)] #[derive(Copy, Clone, Debug)] @@ -456,3 +472,13 @@ impl Default for VcpuGuestContext { pub union VcpuGuestContextAny { pub value: VcpuGuestContext, } + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MmuExtOp { + pub cmd: c_uint, + pub arg1: c_ulong, + pub arg2: c_ulong, +} + +pub const MMUEXT_PIN_L4_TABLE: u32 = 3; diff --git a/xenclient/examples/boot.rs b/xenclient/examples/boot.rs index 9be607e..4d5406a 100644 --- a/xenclient/examples/boot.rs +++ b/xenclient/examples/boot.rs @@ -19,7 +19,6 @@ fn main() -> Result<(), XenClientError> { let call = XenCall::open()?; let domctl = DomainControl::new(&call); let domid = domctl.create_domain(CreateDomain::default())?; - domctl.pause_domain(domid)?; domctl.set_max_vcpus(domid, 1)?; let result = boot(domid, kernel_image_path.as_str(), &call, &domctl); domctl.destroy_domain(domid)?; diff --git a/xenclient/src/boot.rs b/xenclient/src/boot.rs index b3d3fe8..2bd0bcd 100644 --- a/xenclient/src/boot.rs +++ b/xenclient/src/boot.rs @@ -4,19 +4,20 @@ use crate::sys::{ XEN_PAGE_SHIFT, }; use crate::x86::{ - PageTable, PageTableMapping, StartInfo, MAX_GUEST_CMDLINE, X86_GUEST_MAGIC, X86_PAGE_SHIFT, - X86_PAGE_SIZE, X86_PAGE_TABLE_MAX_MAPPINGS, X86_PGTABLE_LEVELS, X86_PGTABLE_LEVEL_SHIFT, - X86_VIRT_MASK, + PageTable, PageTableMapping, SharedInfo, StartInfo, MAX_GUEST_CMDLINE, X86_GUEST_MAGIC, + X86_PAGE_SHIFT, X86_PAGE_SIZE, X86_PAGE_TABLE_MAX_MAPPINGS, X86_PGTABLE_LEVELS, + X86_PGTABLE_LEVEL_SHIFT, X86_VIRT_MASK, }; use crate::XenClientError; use libc::c_char; use log::{debug, trace}; use slice_copy::copy; use std::cmp::{max, min}; +use std::mem::size_of; use std::slice; use xencall::domctl::DomainControl; use xencall::memory::MemoryControl; -use xencall::sys::VcpuGuestContext; +use xencall::sys::{VcpuGuestContext, MMUEXT_PIN_L4_TABLE}; use xencall::XenCall; pub trait BootImageLoader { @@ -198,7 +199,12 @@ impl BootSetup<'_> { .as_str(), )); } - copy(p2m.as_mut_slice(), result.as_slice()); + + for (i, item) in result.iter().enumerate() { + let p = (pfn_base + j + i as u64) as usize; + let m = *item; + p2m[p] = m; + } j += allocsz; } } @@ -263,6 +269,22 @@ impl BootSetup<'_> { self.setup_start_info(state, cmdline)?; self.setup_hypercall_page(&state.image_info)?; + self.phys.unmap(state.page_table_segment.pfn)?; + self.phys.unmap(state.p2m_segment.pfn)?; + let pg_pfn = state.page_table_segment.pfn; + let pg_mfn = self.phys.p2m[pg_pfn as usize]; + debug!( + "domain info: {:?}", + self.domctl.get_domain_info(self.domid)? + ); + let page_frame_info = self.domctl.get_page_frame_info(self.domid, &[pg_pfn])?; + debug!("pgtable page frame info: {:#x}", page_frame_info[0]); + debug!("pinning l4 table: mfn={:#x}", pg_mfn); + self.memctl + .mmuext(self.domid, MMUEXT_PIN_L4_TABLE, pg_mfn, 0)?; + debug!("pinned l4 table: {:#x}", state.page_table_segment.pfn); + self.setup_shared_info()?; + let mut vcpu = VcpuGuestContext::default(); vcpu.user_regs.rip = state.image_info.virt_entry; vcpu.user_regs.rsp = @@ -273,7 +295,7 @@ impl BootSetup<'_> { vcpu.debugreg[6] = 0xffff0ff0; vcpu.debugreg[7] = 0x00000400; vcpu.flags = VGCF_IN_KERNEL | VGCF_ONLINE; - let cr3_pfn = self.phys.p2m[state.page_table_segment.pfn as usize]; + let cr3_pfn = pg_mfn; debug!( "cr3: pfn {:#x} mfn {:#x}", state.page_table_segment.pfn, cr3_pfn @@ -358,9 +380,9 @@ impl BootSetup<'_> { const PAGE_PRESENT: u64 = 0x001; const PAGE_RW: u64 = 0x002; + const PAGE_USER: u64 = 0x004; const PAGE_ACCESSED: u64 = 0x020; const PAGE_DIRTY: u64 = 0x040; - const PAGE_USER: u64 = 0x004; fn get_pg_prot(&mut self, l: usize, pfn: u64, table: &PageTable) -> u64 { let prot = [ BootSetup::PAGE_PRESENT | BootSetup::PAGE_RW | BootSetup::PAGE_ACCESSED, @@ -425,6 +447,21 @@ impl BootSetup<'_> { Ok(()) } + fn setup_shared_info(&mut self) -> Result<(), XenClientError> { + let domain_info = self.domctl.get_domain_info(self.domid)?; + let info = self.phys.pfn_to_ptr(domain_info.shared_info_frame, 1)? as *mut SharedInfo; + unsafe { + let size = size_of::(); + let info_as_buff = slice::from_raw_parts_mut(info as *mut u8, size); + info_as_buff.fill(0); + for i in 0..32 { + (*info).vcpu_info[i].evtchn_upcall_mask = 1; + } + trace!("BootSetup setup_shared_info shared_info={:?}", *info); + } + Ok(()) + } + fn load_kernel_segment( &mut self, image_loader: &dyn BootImageLoader, diff --git a/xenclient/src/mem.rs b/xenclient/src/mem.rs index 284dec8..e27a63a 100644 --- a/xenclient/src/mem.rs +++ b/xenclient/src/mem.rs @@ -1,5 +1,7 @@ use crate::sys::XEN_PAGE_SHIFT; use crate::XenClientError; +use libc::munmap; +use std::ffi::c_void; use crate::x86::X86_PAGE_SHIFT; use xencall::sys::MmapEntry; @@ -104,4 +106,28 @@ impl PhysicalPages<'_> { self.pages.push(page); Ok(addr) } + + pub fn unmap(&mut self, pfn: u64) -> Result<(), XenClientError> { + let mut page: Option<&PhysicalPage> = None; + for item in &self.pages { + if pfn >= item.pfn && pfn < (item.pfn + item.count) { + break; + } + page = Some(item); + } + if page.is_none() { + return Err(XenClientError::new("failed to unmap pfn")); + } + let page = page.unwrap(); + unsafe { + let err = munmap( + page.ptr as *mut c_void, + (page.count << X86_PAGE_SHIFT) as usize, + ); + if err != 0 { + return Err(XenClientError::new("failed to munmap pfn")); + } + } + Ok(()) + } } diff --git a/xenclient/src/x86.rs b/xenclient/src/x86.rs index 6268ba5..20d3d07 100644 --- a/xenclient/src/x86.rs +++ b/xenclient/src/x86.rs @@ -62,3 +62,53 @@ pub struct StartInfo { } pub const X86_GUEST_MAGIC: &str = "xen-3.0-x86_64"; + +#[repr(C)] +#[derive(Debug)] +pub struct ArchVcpuInfo { + pub cr2: u64, + pub pad: u64, +} + +#[repr(C)] +#[derive(Debug)] +pub struct VcpuInfoTime { + pub version: u32, + pub pad0: u32, + pub tsc_timestamp: u64, + pub system_time: u64, + pub tsc_to_system_mul: u32, + pub tsc_shift: i8, + pub flags: u8, + pub pad1: [u8; 2], +} + +#[repr(C)] +#[derive(Debug)] +pub struct VcpuInfo { + pub evtchn_upcall_pending: u8, + pub evtchn_upcall_mask: u8, + pub evtchn_pending_sel: u64, + pub arch_vcpu_info: ArchVcpuInfo, + pub vcpu_info_time: VcpuInfoTime, +} + +#[repr(C)] +#[derive(Debug)] +pub struct SharedInfo { + pub vcpu_info: [VcpuInfo; 32], + pub evtchn_pending: [u64; u64::BITS as usize], + pub evtchn_mask: [u64; u64::BITS as usize], + pub wc_version: u32, + pub wc_sec: u32, + pub wc_nsec: u32, + pub wc_sec_hi: u32, + + // arch shared info + pub max_pfn: u64, + pub pfn_to_mfn_frame_list_list: u64, + pub nmi_reason: u64, + pub p2m_cr3: u64, + pub p2m_vaddr: u64, + pub p2m_generation: u64, +}