feat: add configurable scratch region base GPA

src/hyperlight_common/src/arch/i686/layout.rs

@@ -22,6 +22,6 @@ pub const SNAPSHOT_PT_GVA_MIN: usize = 0xef00_0000;
 pub const SNAPSHOT_PT_GVA_MAX: usize = 0xefff_efff;
 pub const MAX_GPA: usize = 0xffff_ffff;
 
-pub fn min_scratch_size() -> usize {
-    1 * crate::vmem::PAGE_SIZE
+pub fn min_scratch_size(_input_data_size: usize, _output_data_size: usize) -> usize {
+    crate::vmem::PAGE_SIZE
 }

src/hyperlight_common/src/layout.rs

@@ -14,8 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
  */
 
-#[cfg_attr(target_arch = "x86_64", path = "arch/amd64/layout.rs")]
 #[cfg_attr(target_arch = "x86", path = "arch/i686/layout.rs")]
+#[cfg_attr(
+    all(target_arch = "x86_64", feature = "init-paging"),
+    path = "arch/amd64/layout.rs"
+)]
+#[cfg_attr(
+    all(target_arch = "x86_64", not(feature = "init-paging")),
+    path = "arch/i686/layout.rs"
+)]
 mod arch;
 
 pub use arch::{MAX_GPA, MAX_GVA, SNAPSHOT_PT_GVA_MAX, SNAPSHOT_PT_GVA_MIN};

src/hyperlight_host/Cargo.toml

@@ -40,7 +40,7 @@ tracing = { version = "0.1.44", features = ["log"] }
 tracing-log = "0.2.0"
 tracing-core = "0.1.36"
 tracing-opentelemetry = { version = "0.32.1", optional = true }
-hyperlight-common = { workspace = true, default-features = true, features = [ "std", "init-paging" ] }
+hyperlight-common = { workspace = true, default-features = true, features = [ "std" ] }
 hyperlight-guest-tracing = { workspace = true, default-features = true, optional = true }
 vmm-sys-util = "0.15.0"
 crossbeam-channel = "0.5.15"
@@ -137,7 +137,7 @@ mshv3 = ["dep:mshv-bindings", "dep:mshv-ioctls"]
 gdb = ["dep:gdbstub", "dep:gdbstub_arch"]
 fuzzing = ["hyperlight-common/fuzzing"]
 build-metadata = ["dep:built"]
-init-paging = []
+init-paging = ["hyperlight-common/init-paging"]
 
 [[bench]]
 name = "benchmarks"

src/hyperlight_host/src/hypervisor/gdb/mod.rs

@@ -39,7 +39,7 @@ use crate::hypervisor::virtual_machine::{HypervisorError, RegisterError, Virtual
 use crate::mem::layout::SandboxMemoryLayout;
 use crate::mem::memory_region::MemoryRegion;
 use crate::mem::mgr::SandboxMemoryManager;
-use crate::mem::shared_mem::{HostSharedMemory, SharedMemory};
+use crate::mem::shared_mem::HostSharedMemory;
 
 #[derive(Debug, Error)]
 pub enum GdbTargetError {
@@ -166,8 +166,7 @@ impl DebugMemoryAccess {
                 .dbg_mem_access_fn
                 .try_lock()
                 .map_err(|e| DebugMemoryAccessError::LockFailed(file!(), line!(), e.to_string()))?;
-            let scratch_base =
-                hyperlight_common::layout::scratch_base_gpa(mgr.scratch_mem.mem_size());
+            let scratch_base = mgr.layout.get_scratch_base_gpa();
             let (mem, offset, name): (&mut HostSharedMemory, _, _) = if gpa >= scratch_base {
                 (
                     &mut mgr.scratch_mem,
@@ -249,8 +248,7 @@ impl DebugMemoryAccess {
                 .dbg_mem_access_fn
                 .try_lock()
                 .map_err(|e| DebugMemoryAccessError::LockFailed(file!(), line!(), e.to_string()))?;
-            let scratch_base =
-                hyperlight_common::layout::scratch_base_gpa(mgr.scratch_mem.mem_size());
+            let scratch_base = mgr.layout.get_scratch_base_gpa();
             let (mem, offset, name): (&mut HostSharedMemory, _, _) = if gpa >= scratch_base {
                 (
                     &mut mgr.scratch_mem,

src/hyperlight_host/src/hypervisor/hyperlight_vm.rs

@@ -166,6 +166,8 @@ pub(crate) struct HyperlightVm {
     // The current scratch region, used to keep it alive as long as it
     // is used & when unmapping
     scratch_memory: Option<GuestSharedMemory>,
+    // The resolved base GPA of the scratch region
+    scratch_base_gpa: u64,
 
     mmap_regions: Vec<(u32, MemoryRegion)>, // Later mapped regions (slot number, region)
 
@@ -472,6 +474,8 @@ impl HyperlightVm {
 
         let snapshot_slot = 0u32;
         let scratch_slot = 1u32;
+        let scratch_base_gpa =
+            hyperlight_common::layout::scratch_base_gpa(scratch_mem.mem_size());
         #[cfg_attr(not(gdb), allow(unused_mut))]
         let mut ret = Self {
             vm,
@@ -487,6 +491,7 @@ impl HyperlightVm {
             snapshot_memory: None,
             scratch_slot,
             scratch_memory: None,
+            scratch_base_gpa,
 
             mmap_regions: Vec::new(),
 
@@ -665,11 +670,11 @@ impl HyperlightVm {
         &mut self,
         scratch: GuestSharedMemory,
     ) -> Result<(), UpdateRegionError> {
-        let guest_base = hyperlight_common::layout::scratch_base_gpa(scratch.mem_size());
+        let guest_base = self.scratch_base_gpa;
         let rgn = scratch.mapping_at(guest_base, MemoryRegionType::Scratch);
 
         if let Some(old_scratch) = self.scratch_memory.replace(scratch) {
-            let old_base = hyperlight_common::layout::scratch_base_gpa(old_scratch.mem_size());
+            let old_base = self.scratch_base_gpa;
             let old_rgn = old_scratch.mapping_at(old_base, MemoryRegionType::Scratch);
             self.vm.unmap_memory((self.scratch_slot, &old_rgn))?;
         }
@@ -2189,9 +2194,10 @@ mod tests {
         let (mut hshm, gshm) = mem_mgr.build().unwrap();
 
         let peb_address = gshm.layout.peb_address;
-        let stack_top_gva = hyperlight_common::layout::MAX_GVA as u64
-            - hyperlight_common::layout::SCRATCH_TOP_EXN_STACK_OFFSET
-            + 1;
+        let scratch_base_gva =
+            hyperlight_common::layout::scratch_base_gva(config.get_scratch_size());
+        let stack_top_gva = scratch_base_gva + config.get_scratch_size() as u64
+            - hyperlight_common::layout::SCRATCH_TOP_EXN_STACK_OFFSET;
         let mut vm = set_up_hypervisor_partition(
             gshm,
             &config,
@@ -2795,9 +2801,10 @@ mod tests {
 
             /// Get the stack top GVA, same as the regular codepath.
             fn stack_top_gva(&self) -> u64 {
-                hyperlight_common::layout::MAX_GVA as u64
+                let scratch_size = crate::sandbox::SandboxConfiguration::DEFAULT_SCRATCH_SIZE;
+                let scratch_base_gva = hyperlight_common::layout::scratch_base_gva(scratch_size);
+                scratch_base_gva + scratch_size as u64
                     - hyperlight_common::layout::SCRATCH_TOP_EXN_STACK_OFFSET
-                    + 1
             }
         }
 

src/hyperlight_host/src/mem/layout.rs

@@ -108,6 +108,15 @@ pub(crate) struct SandboxMemoryLayout {
     // The size of the scratch region in physical memory; note that
     // this will appear under the top of physical memory.
     scratch_size: usize,
+    // The resolved base GPA of the scratch region, computed as
+    // MAX_GPA - scratch_size + 1, where MAX_GPA depends on
+    // whether init-paging is enabled (36-bit) or not (32-bit).
+    scratch_base_gpa: u64,
+    // The resolved base GVA of the scratch region. For non-init-paging
+    // (no page tables), GVA = GPA (identity mapped). For init-paging
+    // (64-bit page tables), GVA = MAX_GVA - scratch_size + 1 (high
+    // canonical address).
+    scratch_base_gva: u64,
 }
 
 impl Debug for SandboxMemoryLayout {
@@ -158,6 +167,14 @@ impl Debug for SandboxMemoryLayout {
                 "Scratch region size",
                 &format_args!("{:#x}", self.scratch_size),
             )
+            .field(
+                "Scratch base GPA",
+                &format_args!("{:#x}", self.scratch_base_gpa),
+            )
+            .field(
+                "Scratch base GVA",
+                &format_args!("{:#x}", self.scratch_base_gva),
+            )
             .finish()
     }
 }
@@ -204,6 +221,16 @@ impl SandboxMemoryLayout {
             return Err(MemoryRequestTooSmall(scratch_size, min_scratch_size));
         }
 
+        let scratch_base_gpa = hyperlight_common::layout::scratch_base_gpa(scratch_size);
+
+        // For init-paging guests, GVA comes from the high canonical address
+        // (MAX_GVA - scratch_size + 1). For non-init-paging guests, GVA = GPA
+        // (identity mapped, no hypervisor-managed page tables).
+        #[cfg(feature = "init-paging")]
+        let scratch_base_gva = hyperlight_common::layout::scratch_base_gva(scratch_size);
+        #[cfg(not(feature = "init-paging"))]
+        let scratch_base_gva = scratch_base_gpa;
+
         let guest_code_offset = 0;
         // The following offsets are to the fields of the PEB struct itself!
         let peb_offset = code_size.next_multiple_of(PAGE_SIZE_USIZE);
@@ -240,6 +267,8 @@ impl SandboxMemoryLayout {
             init_data_permissions,
             pt_size: None,
             scratch_size,
+            scratch_base_gpa,
+            scratch_base_gva,
         })
     }
 
@@ -262,6 +291,20 @@ impl SandboxMemoryLayout {
         self.scratch_size
     }
 
+    /// Get the resolved base GPA of the scratch region.
+    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
+    pub(crate) fn get_scratch_base_gpa(&self) -> u64 {
+        self.scratch_base_gpa
+    }
+
+    /// Get the base GVA of the scratch region.
+    /// For init-paging guests (feature = "init-paging"), this is the high canonical address
+    /// (MAX_GVA - scratch_size + 1). For non-init-paging guests, GVA = GPA (identity mapped).
+    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
+    pub(crate) fn get_scratch_base_gva(&self) -> u64 {
+        self.scratch_base_gva
+    }
+
     /// Get the offset in guest memory to the output data pointer.
     #[instrument(skip_all, parent = Span::current(), level= "Trace")]
     fn get_output_data_pointer_offset(&self) -> usize {
@@ -281,8 +324,7 @@ impl SandboxMemoryLayout {
     /// Get the guest virtual address of the start of output data.
     #[instrument(skip_all, parent = Span::current(), level= "Trace")]
     pub(crate) fn get_output_data_buffer_gva(&self) -> u64 {
-        hyperlight_common::layout::scratch_base_gva(self.scratch_size)
-            + self.sandbox_memory_config.get_input_data_size() as u64
+        self.get_scratch_base_gva() + self.sandbox_memory_config.get_input_data_size() as u64
     }
 
     /// Get the offset into the host scratch buffer of the start of
@@ -310,7 +352,7 @@ impl SandboxMemoryLayout {
     /// Get the guest virtual address of the start of input data
     #[instrument(skip_all, parent = Span::current(), level= "Trace")]
     fn get_input_data_buffer_gva(&self) -> u64 {
-        hyperlight_common::layout::scratch_base_gva(self.scratch_size)
+        self.get_scratch_base_gva()
     }
 
     /// Get the offset into the host scratch buffer of the start of
@@ -333,8 +375,7 @@ impl SandboxMemoryLayout {
     /// copied on restore
     #[instrument(skip_all, parent = Span::current(), level= "Trace")]
     pub(crate) fn get_pt_base_gpa(&self) -> u64 {
-        hyperlight_common::layout::scratch_base_gpa(self.scratch_size)
-            + self.get_pt_base_scratch_offset() as u64
+        self.scratch_base_gpa + self.get_pt_base_scratch_offset() as u64
     }
 
     /// Get the first GPA of the scratch region that the host hasn't

src/hyperlight_host/src/mem/shared_mem.rs

@@ -679,7 +679,22 @@ impl GuestSharedMemory {
             MemoryRegionType::Scratch => {
                 MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
             }
-            MemoryRegionType::Snapshot => MemoryRegionFlags::READ | MemoryRegionFlags::EXECUTE,
+            // For init-paging, the snapshot is read-only because guest page
+            // tables provide CoW semantics for writable pages.  For
+            // non-init-paging there are no guest page tables, so the snapshot
+            // must be writable — otherwise writes (including the CPU setting
+            // the "Accessed" bit in GDT descriptors during segment loads)
+            // cause EPT violations that KVM retries forever.
+            MemoryRegionType::Snapshot => {
+                #[cfg(feature = "init-paging")]
+                {
+                    MemoryRegionFlags::READ | MemoryRegionFlags::EXECUTE
+                }
+                #[cfg(not(feature = "init-paging"))]
+                {
+                    MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
+                }
+            }
             #[allow(clippy::panic)]
             // In the future, all the host side knowledge about memory
             // region types should collapse down to Snapshot vs