we've done it, it boots!

xencall/src/domctl.rs

@@ -15,7 +15,7 @@ use std::ptr::addr_of_mut;
 use std::slice;
 
 pub struct DomainControl<'a> {
-    call: &'a XenCall,
+    pub call: &'a XenCall,
 }
 
 impl DomainControl<'_> {

xencall/src/lib.rs

@@ -3,7 +3,7 @@ pub mod memory;
 pub mod sys;
 
 use crate::sys::{
-    Hypercall, MmapBatch, MultiCallEntry, XenCapabilitiesInfo, HYPERVISOR_MULTICALL,
+    Hypercall, MmapBatch, MmapResource, MultiCallEntry, XenCapabilitiesInfo, HYPERVISOR_MULTICALL,
     HYPERVISOR_XEN_VERSION, XENVER_CAPABILITIES,
 };
 use libc::{mmap, MAP_FAILED, MAP_SHARED, PROT_READ, PROT_WRITE};
@@ -168,6 +168,29 @@ impl XenCall {
         Ok(())
     }
 
+    pub fn map_resource(
+        &self,
+        domid: u32,
+        typ: u32,
+        id: u32,
+        idx: u32,
+        num: u64,
+        addr: u64,
+    ) -> Result<(), XenCallError> {
+        let mut resource = MmapResource {
+            dom: domid as u16,
+            typ,
+            id,
+            idx,
+            num,
+            addr,
+        };
+        unsafe {
+            sys::mmap_resource(self.handle.as_raw_fd(), &mut resource)?;
+        }
+        Ok(())
+    }
+
     pub fn mmap_batch(
         &self,
         domid: u32,

xencall/src/sys.rs

@@ -18,6 +18,17 @@ pub struct MmapEntry {
     pub npages: u64,
 }
 
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default)]
+pub struct MmapResource {
+    pub dom: u16,
+    pub typ: u32,
+    pub id: u32,
+    pub idx: u32,
+    pub num: u64,
+    pub addr: u64,
+}
+
 #[repr(C)]
 #[derive(Copy, Clone, Debug)]
 pub struct MmapBatch {
@@ -39,10 +50,12 @@ pub struct Mmap {
 const IOCTL_PRIVCMD_HYPERCALL: u64 = 0x305000;
 const IOCTL_PRIVCMD_MMAP: u64 = 0x105002;
 const IOCTL_PRIVCMD_MMAPBATCH_V2: u64 = 0x205004;
+const IOCTL_PRIVCMD_MMAP_RESOURCE: u64 = 0x205007;
 
 ioctl_readwrite_bad!(hypercall, IOCTL_PRIVCMD_HYPERCALL, Hypercall);
 ioctl_readwrite_bad!(mmap, IOCTL_PRIVCMD_MMAP, Mmap);
 ioctl_readwrite_bad!(mmapbatch, IOCTL_PRIVCMD_MMAPBATCH_V2, MmapBatch);
+ioctl_readwrite_bad!(mmap_resource, IOCTL_PRIVCMD_MMAP_RESOURCE, MmapResource);
 
 pub const HYPERVISOR_SET_TRAP_TABLE: c_ulong = 0;
 pub const HYPERVISOR_MMU_UPDATE: c_ulong = 1;

xenclient/examples/boot.rs

@@ -25,16 +25,13 @@ fn main() -> Result<(), XenClientError> {
         ..Default::default()
     };
     let domid = domctl.create_domain(domain)?;
-    let result = boot(
+    boot(
         domid,
         kernel_image_path.as_str(),
         initrd_path.as_str(),
         &call,
         &domctl,
-    );
+    )?;
-    domctl.destroy_domain(domid)?;
-    result?;
-    println!("domain destroyed: {}", domid);
     Ok(())
 }
 
@@ -50,7 +47,7 @@ fn boot(
     let memctl = MemoryControl::new(call);
     let mut boot = BootSetup::new(call, domctl, &memctl, domid);
     let initrd = read(initrd_path)?;
-    let mut state = boot.initialize(&image_loader, initrd.as_slice(), 512)?;
+    let mut state = boot.initialize(&image_loader, initrd.as_slice(), 1, 512)?;
     boot.boot(&mut state, "debug")?;
     Ok(())
 }

xenclient/src/boot.rs

@@ -1,7 +1,7 @@
 use crate::mem::PhysicalPages;
 use crate::sys::{
-    SUPERPAGE_2MB_NR_PFNS, SUPERPAGE_2MB_SHIFT, SUPERPAGE_BATCH_SIZE, VGCF_IN_KERNEL, VGCF_ONLINE,
+    GrantEntry, SUPERPAGE_2MB_NR_PFNS, SUPERPAGE_2MB_SHIFT, SUPERPAGE_BATCH_SIZE, VGCF_IN_KERNEL,
-    XEN_PAGE_SHIFT,
+    VGCF_ONLINE, XEN_PAGE_SHIFT,
 };
 use crate::x86::{
     PageTable, PageTableMapping, SharedInfo, StartInfo, MAX_GUEST_CMDLINE, X86_GUEST_MAGIC,
@@ -9,10 +9,11 @@ use crate::x86::{
     X86_PGTABLE_LEVEL_SHIFT, X86_VIRT_MASK,
 };
 use crate::XenClientError;
-use libc::c_char;
+use libc::{c_char, munmap};
 use log::{debug, trace};
 use slice_copy::copy;
 use std::cmp::{max, min};
+use std::ffi::c_void;
 use std::mem::size_of;
 use std::slice;
 use xencall::domctl::DomainControl;
@@ -57,7 +58,7 @@ pub struct DomainSegment {
     pfn: u64,
     addr: u64,
     size: u64,
-    _pages: u64,
+    pages: u64,
 }
 
 #[derive(Debug)]
@@ -79,6 +80,8 @@ pub struct BootState {
     pub page_table_segment: DomainSegment,
     pub page_table: PageTable,
     pub image_info: BootImageInfo,
+    pub shared_info_frame: u64,
+    pub initrd_segment: DomainSegment,
 }
 
 impl BootSetup<'_> {
@@ -229,9 +232,14 @@ impl BootSetup<'_> {
         &mut self,
         image_loader: &dyn BootImageLoader,
         initrd: &[u8],
+        max_vcpus: u32,
         mem_mb: u64,
     ) -> Result<BootState, XenClientError> {
-        debug!("BootSetup initialize mem_mb={:?}", mem_mb);
+        debug!(
+            "BootSetup initialize max_vcpus={:?} mem_mb={:?}",
+            max_vcpus, mem_mb
+        );
+        self.domctl.set_max_vcpus(self.domid, max_vcpus)?;
         self.domctl.set_max_mem(self.domid, mem_mb * 1024)?;
 
         let total_pages = mem_mb << (20 - X86_PAGE_SHIFT);
@@ -253,6 +261,11 @@ impl BootSetup<'_> {
         let console_segment = self.alloc_page()?;
         let page_table_segment = self.alloc_page_tables(&mut page_table, &image_info)?;
         let boot_stack_segment = self.alloc_page()?;
+
+        if self.virt_pgtab_end > 0 {
+            self.alloc_padding_pages(self.virt_pgtab_end)?;
+        }
+
         let mut initrd_segment: Option<DomainSegment> = None;
         if !image_info.unmapped_initrd {
             initrd_segment = Some(self.alloc_module(initrd)?);
@@ -264,15 +277,11 @@ impl BootSetup<'_> {
         }
         let p2m_segment = p2m_segment.unwrap();
 
-        if self.virt_pgtab_end > 0 {
-            self.alloc_padding_pages(self.virt_pgtab_end)?;
-        }
-
         if image_info.unmapped_initrd {
             initrd_segment = Some(self.alloc_module(initrd)?);
         }
 
-        let _initrd_segment = initrd_segment.unwrap();
+        let initrd_segment = initrd_segment.unwrap();
 
         let state = BootState {
             kernel_segment,
@@ -284,16 +293,17 @@ impl BootSetup<'_> {
             page_table_segment,
             page_table,
             image_info,
+            initrd_segment,
+            shared_info_frame: 0,
         };
         debug!("BootSetup initialize state={:?}", state);
         Ok(state)
     }
 
     pub fn boot(&mut self, state: &mut BootState, cmdline: &str) -> Result<(), XenClientError> {
-        debug!(
+        let domain_info = self.domctl.get_domain_info(self.domid)?;
-            "domain info: {:?}",
+        let shared_info_frame = domain_info.shared_info_frame;
-            self.domctl.get_domain_info(self.domid)?
+        state.shared_info_frame = shared_info_frame;
-        );
         self.setup_page_tables(state)?;
         self.setup_start_info(state, cmdline)?;
         self.setup_hypercall_page(&state.image_info)?;
@@ -304,7 +314,7 @@ impl BootSetup<'_> {
         let pg_mfn = self.phys.p2m[pg_pfn as usize];
         self.memctl
             .mmuext(self.domid, MMUEXT_PIN_L4_TABLE, pg_mfn, 0)?;
-        // self.setup_shared_info()?;
+        self.setup_shared_info(state.shared_info_frame)?;
 
         let mut vcpu = VcpuGuestContext::default();
         vcpu.user_regs.rip = state.image_info.virt_entry;
@@ -330,13 +340,46 @@ impl BootSetup<'_> {
         vcpu.user_regs.cs = 0xe033;
         vcpu.kernel_ss = vcpu.user_regs.ss as u64;
         vcpu.kernel_sp = vcpu.user_regs.rsp;
+        debug!("vcpu context: {:?}", vcpu);
         self.domctl.set_vcpu_context(self.domid, 0, &vcpu)?;
+        self.phys.unmap_all()?;
+        self.gnttab_seed(state)?;
+        Ok(())
+    }
+
+    fn gnttab_seed(&mut self, state: &mut BootState) -> Result<(), XenClientError> {
+        let console_gfn = self.phys.p2m[state.console_segment.pfn as usize];
+        let xenstore_gfn = self.phys.p2m[state.xenstore_segment.pfn as usize];
+        let addr = self
+            .domctl
+            .call
+            .mmap(0, 1 << XEN_PAGE_SHIFT)
+            .ok_or(XenClientError::new("failed to mmap for resource"))?;
+        self.domctl
+            .call
+            .map_resource(self.domid, 1, 0, 0, 1, addr)?;
+        let entries = unsafe { slice::from_raw_parts_mut(addr as *mut GrantEntry, 2) };
+        entries[0].flags = 1 << 0;
+        entries[0].domid = 0;
+        entries[0].frame = console_gfn as u32;
+        entries[1].flags = 1 << 0;
+        entries[1].domid = 0;
+        entries[1].frame = xenstore_gfn as u32;
+        unsafe {
+            let result = munmap(addr as *mut c_void, 1 << XEN_PAGE_SHIFT);
+            if result != 0 {
+                return Err(XenClientError::new("failed to unmap resource"));
+            }
+        }
         Ok(())
     }
 
     fn setup_page_tables(&mut self, state: &mut BootState) -> Result<(), XenClientError> {
         let p2m_guest = unsafe {
-            slice::from_raw_parts_mut(state.p2m_segment.addr as *mut u64, self.phys.p2m.len())
+            slice::from_raw_parts_mut(
+                state.p2m_segment.addr as *mut u64,
+                self.phys.p2m_size() as usize,
+            )
         };
         copy(p2m_guest, &self.phys.p2m);
 
@@ -430,37 +473,44 @@ impl BootSetup<'_> {
     }
 
     fn setup_start_info(&mut self, state: &BootState, cmdline: &str) -> Result<(), XenClientError> {
-        let info = self.phys.pfn_to_ptr(state.start_info_segment.pfn, 1)? as *mut StartInfo;
+        let ptr = self.phys.pfn_to_ptr(state.start_info_segment.pfn, 1)?;
+        let byte_slice =
+            unsafe { slice::from_raw_parts_mut(ptr as *mut u8, X86_PAGE_SIZE as usize) };
+        byte_slice.fill(0);
+        let info = ptr as *mut StartInfo;
         unsafe {
             for (i, c) in X86_GUEST_MAGIC.chars().enumerate() {
                 (*info).magic[i] = c as c_char;
             }
+            (*info).magic[X86_GUEST_MAGIC.len()] = 0 as c_char;
             (*info).nr_pages = self.total_pages;
-            (*info).shared_info = 0;
+            (*info).shared_info = state.shared_info_frame << X86_PAGE_SHIFT;
             (*info).pt_base = state.page_table_segment.vstart;
             (*info).nr_pt_frames = state.page_table.mappings[0].area.pgtables as u64;
-            (*info).mfn_list = 0;
+            (*info).mfn_list = state.p2m_segment.vstart;
-            (*info).first_p2m_pfn = 0;
+            (*info).first_p2m_pfn = state.p2m_segment.pfn;
-            (*info).nr_p2m_frames = 0;
+            (*info).nr_p2m_frames = state.p2m_segment.pages;
             (*info).flags = 0;
             (*info).store_evtchn = 0;
             (*info).store_mfn = self.phys.p2m[state.xenstore_segment.pfn as usize];
             (*info).console.mfn = self.phys.p2m[state.console_segment.pfn as usize];
             (*info).console.evtchn = 0;
-            (*info).mod_start = 0;
+            (*info).mod_start = state.initrd_segment.vstart;
-            (*info).mod_len = 0;
+            (*info).mod_len = state.initrd_segment.size;
             for (i, c) in cmdline.chars().enumerate() {
                 (*info).cmdline[i] = c as c_char;
-                (*info).cmdline[MAX_GUEST_CMDLINE - 1] = 0;
             }
+            (*info).cmdline[MAX_GUEST_CMDLINE - 1] = 0;
             trace!("BootSetup setup_start_info start_info={:?}", *info);
         }
         Ok(())
     }
 
-    fn _setup_shared_info(&mut self) -> Result<(), XenClientError> {
+    fn setup_shared_info(&mut self, shared_info_frame: u64) -> Result<(), XenClientError> {
-        let domain_info = self.domctl.get_domain_info(self.domid)?;
+        let info = self
-        let info = self.phys.pfn_to_ptr(domain_info.shared_info_frame, 1)? as *mut SharedInfo;
+            .phys
+            .map_foreign_pages(shared_info_frame, X86_PAGE_SIZE)?
+            as *mut SharedInfo;
         unsafe {
             let size = size_of::<SharedInfo>();
             let info_as_buff = slice::from_raw_parts_mut(info as *mut u8, size);
@@ -656,7 +706,7 @@ impl BootSetup<'_> {
             pfn: self.pfn_alloc_end,
             addr: 0,
             size,
-            _pages: pages,
+            pages,
         };
 
         self.chk_alloc_pages(pages)?;
@@ -687,7 +737,7 @@ impl BootSetup<'_> {
             pfn,
             addr: 0,
             size: 0,
-            _pages: 1,
+            pages: 1,
         })
     }
 

xenclient/src/mem.rs

@@ -1,4 +1,4 @@
-use crate::sys::XEN_PAGE_SHIFT;
+use crate::sys::{XEN_PAGE_SHIFT, XEN_PAGE_SIZE};
 use crate::XenClientError;
 use libc::munmap;
 use log::debug;
@@ -114,6 +114,36 @@ impl PhysicalPages<'_> {
         Ok(addr)
     }
 
+    pub fn map_foreign_pages(&mut self, mfn: u64, size: u64) -> Result<u64, XenClientError> {
+        let num = ((size + XEN_PAGE_SIZE - 1) >> XEN_PAGE_SHIFT) as usize;
+        let mut pfns = vec![u64::MAX; num];
+        for (i, item) in pfns.iter_mut().enumerate().take(num) {
+            *item = mfn + i as u64;
+        }
+
+        let actual_mmap_len = (num as u64) << XEN_PAGE_SHIFT;
+        let addr = self
+            .call
+            .mmap(0, actual_mmap_len)
+            .ok_or(XenClientError::new("failed to mmap address"))?;
+        debug!("mapped {:#x} foreign bytes at {:#x}", actual_mmap_len, addr);
+        let result = self.call.mmap_batch(self.domid, num as u64, addr, pfns)?;
+        if result != 0 {
+            return Err(XenClientError::new("mmap_batch call failed"));
+        }
+        let page = PhysicalPage {
+            pfn: u64::MAX,
+            ptr: addr,
+            count: num as u64,
+        };
+        debug!(
+            "alloc_mfn {:#x}+{:#x} at {:#x}",
+            page.pfn, page.count, page.ptr
+        );
+        self.pages.push(page);
+        Ok(addr)
+    }
+
     pub fn unmap_all(&mut self) -> Result<(), XenClientError> {
         for page in &self.pages {
             unsafe {
@@ -131,11 +161,11 @@ impl PhysicalPages<'_> {
     }
 
     pub fn unmap(&mut self, pfn: u64) -> Result<(), XenClientError> {
-        let page = self.pages.iter().find(|x| x.pfn == pfn);
+        let page = self.pages.iter().enumerate().find(|(_, x)| x.pfn == pfn);
         if page.is_none() {
             return Err(XenClientError::new("unable to find page to unmap"));
         }
-        let page = page.unwrap();
+        let (i, page) = page.unwrap();
 
         unsafe {
             let err = munmap(
@@ -150,6 +180,7 @@ impl PhysicalPages<'_> {
             if err != 0 {
                 return Err(XenClientError::new("failed to munmap page"));
             }
+            self.pages.remove(i);
         }
         Ok(())
     }

xenclient/src/sys.rs

@@ -121,3 +121,10 @@ pub const SUPERPAGE_2MB_SHIFT: u64 = 9;
 pub const SUPERPAGE_2MB_NR_PFNS: u64 = 1u64 << SUPERPAGE_2MB_SHIFT;
 pub const VGCF_IN_KERNEL: u64 = 1 << 2;
 pub const VGCF_ONLINE: u64 = 1 << 5;
+
+#[repr(C)]
+pub struct GrantEntry {
+    pub flags: u16,
+    pub domid: u16,
+    pub frame: u32,
+}