From 76bcca935c0736241bc15ef078fc325b0e2bde4d Mon Sep 17 00:00:00 2001 From: Steven Malis Date: Wed, 20 May 2026 16:47:40 -0400 Subject: [PATCH 1/7] virt_whp: Reset VTL2-enable state for VPs during scrub --- vmm_core/virt_whp/src/lib.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index 7611128055..8c0daa8e36 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -490,6 +490,21 @@ impl virt::ScrubVtl for WhpPartition { self.inner.vtl2_emulation.as_ref().unwrap().reset(false); self.validate_is_reset(Vtl::Vtl2); + // Reset per-VP VTL2-enable state on non-BSP VPs. The new VTL2 firmware + // will re-issue `HvCallEnableVpVtl` on each AP to program its startup + // context (RIP/RSP/CR3/GDT/IDT). Without clearing this flag, the + // hypercall handler would short-circuit with `VtlAlreadyEnabled` and + // skip programming the AP context, leaving APs to resume at stale + // pre-scrub register values and potentially fault. Note that + // `set_initial_regs` only programs the BSP, so APs rely entirely on + // this enable-VP-VTL path after scrub. The BSP is left alone because + // it boots directly via `set_initial_regs` and is always in VTL2. + for vp in &self.inner.vps { + if !vp.vp_info.base.vp_index.is_bsp() { + vp.vtl2_enable.store(false, Ordering::SeqCst); + } + } + #[cfg(guest_arch = "x86_64")] { self.access_state(Vtl::Vtl2).set_reftime(&reference_time)?; From d192195171d4db79289c6f3fb02198d555ae417f Mon Sep 17 00:00:00 2001 From: Steven Malis Date: Wed, 20 May 2026 17:07:55 -0400 Subject: [PATCH 2/7] move to per-vp code --- vmm_core/virt_whp/src/lib.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index 8c0daa8e36..52f76ab38c 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -490,21 +490,6 @@ impl virt::ScrubVtl for WhpPartition { self.inner.vtl2_emulation.as_ref().unwrap().reset(false); self.validate_is_reset(Vtl::Vtl2); - // Reset per-VP VTL2-enable state on non-BSP VPs. The new VTL2 firmware - // will re-issue `HvCallEnableVpVtl` on each AP to program its startup - // context (RIP/RSP/CR3/GDT/IDT). Without clearing this flag, the - // hypercall handler would short-circuit with `VtlAlreadyEnabled` and - // skip programming the AP context, leaving APs to resume at stale - // pre-scrub register values and potentially fault. Note that - // `set_initial_regs` only programs the BSP, so APs rely entirely on - // this enable-VP-VTL path after scrub. The BSP is left alone because - // it boots directly via `set_initial_regs` and is always in VTL2. - for vp in &self.inner.vps { - if !vp.vp_info.base.vp_index.is_bsp() { - vp.vtl2_enable.store(false, Ordering::SeqCst); - } - } - #[cfg(guest_arch = "x86_64")] { self.access_state(Vtl::Vtl2).set_reftime(&reference_time)?; @@ -1684,6 +1669,19 @@ impl<'p> virt::Processor for WhpProcessor<'p> { self.finish_reset(Vtl::Vtl2); self.vplc(Vtl::Vtl2).message_queues.clear(); + // Reset per-VP VTL2-enable state on non-BSP VPs. The new VTL2 + // will re-issue `HvCallEnableVpVtl` on each AP to program its startup + // context (RIP/RSP/CR3/GDT/IDT). Without clearing this flag, the + // hypercall handler would short-circuit with `VtlAlreadyEnabled` and + // skip programming the AP context, leaving APs to resume at stale + // pre-scrub register values and potentially fault. Note that + // `set_initial_regs` only programs the BSP, so APs rely entirely on + // this enable-VP-VTL path after scrub. The BSP is left alone because + // it boots directly via `set_initial_regs` and is always in VTL2. + if !is_bsp { + self.inner.vtl2_enable.store(false, Ordering::SeqCst); + } + if cfg!(debug_assertions) { let vp_info = &self.inner.vp_info; self.access_state(Vtl::Vtl2).check_reset_all(vp_info); From fc5793e1f007ece12e7051b64a8cc5ef6eef3c61 Mon Sep 17 00:00:00 2001 From: Steven Malis Date: Thu, 21 May 2026 17:57:32 -0400 Subject: [PATCH 3/7] do more --- .../src/processor/hardware_cvm/mod.rs | 2 +- vmm_core/virt_whp/src/lib.rs | 50 +++++++++++++++---- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/openhcl/virt_mshv_vtl/src/processor/hardware_cvm/mod.rs b/openhcl/virt_mshv_vtl/src/processor/hardware_cvm/mod.rs index c4a969571d..9680d8aa4d 100644 --- a/openhcl/virt_mshv_vtl/src/processor/hardware_cvm/mod.rs +++ b/openhcl/virt_mshv_vtl/src/processor/hardware_cvm/mod.rs @@ -1448,7 +1448,7 @@ impl hv1_hypercall::EnablePartitionVtl GuestVsmState::NotGuestEnabled => (), GuestVsmState::Enabled { vtl1: _ } => { // VTL 1 cannot be already enabled - return Err(HvError::VtlAlreadyEnabled); + return Err(HvError::InvalidVtlState); } } diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index 52f76ab38c..edacd3d47e 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -388,6 +388,28 @@ impl Vplc { start_vp_context: Default::default(), } } + + /// Resets the pending per-VTL VP signal flags (`message_queues`, + /// `check_queues`, `extint_pending`, and `start_vp`) to the initial + /// state from `Vplc::new`. + /// + /// This is used when resetting the partition or scrubbing a VTL, so that + /// the freshly-reinitialized VTL does not observe stale events queued + /// before the reset. + fn reset(&self) { + let Self { + message_queues, + check_queues, + extint_pending, + start_vp_context, + start_vp, + } = self; + message_queues.clear(); + check_queues.store(false, Ordering::SeqCst); + extint_pending.store(false, Ordering::SeqCst); + start_vp_context.lock().take(); + start_vp.store(false, Ordering::SeqCst); + } } impl<'a> WhpVpRef<'a> { @@ -1642,13 +1664,18 @@ impl<'p> virt::Processor for WhpProcessor<'p> { let is_bsp = self.inner.vp_info.base.is_bsp(); self.state.reset(false, is_bsp); + // For each enabled VTL: apply arch fixups that WHP doesn't handle + // and clear any pending `start_vp_context` (via `finish_reset`), + // then clear stale pending per-VTL VP signal flags (via + // `Vplc::reset`). // VTL0 is always present. self.finish_reset(Vtl::Vtl0); - self.vplc(Vtl::Vtl0).message_queues.clear(); + self.vplc(Vtl::Vtl0).reset(); if self.state.vtls.vtl2.is_some() { self.finish_reset(Vtl::Vtl2); - self.vplc(Vtl::Vtl2).message_queues.clear(); + self.vplc(Vtl::Vtl2).reset(); } + self.inner.vtl2_wake.store(false, Ordering::SeqCst); if cfg!(debug_assertions) { let vp_info = &self.inner.vp_info; @@ -1665,19 +1692,20 @@ impl<'p> virt::Processor for WhpProcessor<'p> { let is_bsp = self.inner.vp_info.base.is_bsp(); self.state.reset(true, is_bsp); - // Scrub only resets VTL2. + // Scrub only resets VTL2. Reset the Vplc to clear any stale pending + // signals (message queue notifications, external interrupts, start-VP + // requests, etc.) -- the hypervisor zeroes the equivalent per-VTL + // activity flags during a VTL scrub. self.finish_reset(Vtl::Vtl2); - self.vplc(Vtl::Vtl2).message_queues.clear(); + self.vplc(Vtl::Vtl2).reset(); + + // Clear any pending VTL2 wake signal, since VTL2 is now back in + // startup suspend and any prior wake request is stale. + self.inner.vtl2_wake.store(false, Ordering::SeqCst); // Reset per-VP VTL2-enable state on non-BSP VPs. The new VTL2 // will re-issue `HvCallEnableVpVtl` on each AP to program its startup - // context (RIP/RSP/CR3/GDT/IDT). Without clearing this flag, the - // hypercall handler would short-circuit with `VtlAlreadyEnabled` and - // skip programming the AP context, leaving APs to resume at stale - // pre-scrub register values and potentially fault. Note that - // `set_initial_regs` only programs the BSP, so APs rely entirely on - // this enable-VP-VTL path after scrub. The BSP is left alone because - // it boots directly via `set_initial_regs` and is always in VTL2. + // context (RIP/RSP/CR3/GDT/IDT). if !is_bsp { self.inner.vtl2_enable.store(false, Ordering::SeqCst); } From 62ae3fcd5cf5e937f817d6d808d9f8924899e8d7 Mon Sep 17 00:00:00 2001 From: Steven Malis Date: Thu, 21 May 2026 18:04:35 -0400 Subject: [PATCH 4/7] feedback --- vmm_core/virt_whp/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index edacd3d47e..3f7a8015ad 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -389,9 +389,7 @@ impl Vplc { } } - /// Resets the pending per-VTL VP signal flags (`message_queues`, - /// `check_queues`, `extint_pending`, and `start_vp`) to the initial - /// state from `Vplc::new`. + /// Resets the pending per-VTL VP signals to the initial state from `Vplc::new`. /// /// This is used when resetting the partition or scrubbing a VTL, so that /// the freshly-reinitialized VTL does not observe stale events queued @@ -407,7 +405,7 @@ impl Vplc { message_queues.clear(); check_queues.store(false, Ordering::SeqCst); extint_pending.store(false, Ordering::SeqCst); - start_vp_context.lock().take(); + *start_vp_context.lock() = None; start_vp.store(false, Ordering::SeqCst); } } From 6beb1715ba112111c65476d1374bf256cb856f68 Mon Sep 17 00:00:00 2001 From: Steven Malis Date: Thu, 21 May 2026 18:06:29 -0400 Subject: [PATCH 5/7] fix opentmk --- opentmk/src/platform/hyperv/arch/hypercall.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentmk/src/platform/hyperv/arch/hypercall.rs b/opentmk/src/platform/hyperv/arch/hypercall.rs index d3a3193ecf..d1a63fc395 100644 --- a/opentmk/src/platform/hyperv/arch/hypercall.rs +++ b/opentmk/src/platform/hyperv/arch/hypercall.rs @@ -142,7 +142,7 @@ impl HvCall { let output = self.dispatch_hvcall(hvdef::HypercallCode::HvCallEnablePartitionVtl, None); match output.result() { - Ok(()) | Err(hvdef::HvError::VtlAlreadyEnabled) => Ok(()), + Ok(()) | Err(hvdef::HvError::InvalidVtlState) => Ok(()), err => err, } } From 6b898ee4ee6df9ab28291bef16bb178c70486017 Mon Sep 17 00:00:00 2001 From: Steven Malis Date: Thu, 21 May 2026 19:12:17 -0400 Subject: [PATCH 6/7] feedback --- vmm_core/virt_whp/src/lib.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index 3f7a8015ad..b5c2fc8006 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -1688,6 +1688,17 @@ impl<'p> virt::Processor for WhpProcessor<'p> { fn scrub(&mut self, vtl: Vtl) -> Result<(), impl std::error::Error + Send + Sync + 'static> { assert_eq!(vtl, Vtl::Vtl2); let is_bsp = self.inner.vp_info.base.is_bsp(); + + // Reset per-VP VTL2-enable state on non-BSP VPs. The new VTL2 + // will re-issue `HvCallEnableVpVtl` on each AP to program its startup + // context (RIP/RSP/CR3/GDT/IDT). Clear VTL2 from `enabled_vtls` + // before `state.reset` so that `runnable_vtls` and `active_vtl` are + // derived from the post-scrub set rather than the pre-scrub one. + if !is_bsp { + self.inner.vtl2_enable.store(false, Ordering::SeqCst); + self.state.enabled_vtls.clear(Vtl::Vtl2); + } + self.state.reset(true, is_bsp); // Scrub only resets VTL2. Reset the Vplc to clear any stale pending @@ -1701,13 +1712,6 @@ impl<'p> virt::Processor for WhpProcessor<'p> { // startup suspend and any prior wake request is stale. self.inner.vtl2_wake.store(false, Ordering::SeqCst); - // Reset per-VP VTL2-enable state on non-BSP VPs. The new VTL2 - // will re-issue `HvCallEnableVpVtl` on each AP to program its startup - // context (RIP/RSP/CR3/GDT/IDT). - if !is_bsp { - self.inner.vtl2_enable.store(false, Ordering::SeqCst); - } - if cfg!(debug_assertions) { let vp_info = &self.inner.vp_info; self.access_state(Vtl::Vtl2).check_reset_all(vp_info); From 33d6fc7f4c8917ad58a2664ada8053a88bf4aa9d Mon Sep 17 00:00:00 2001 From: Steven Malis <137308034+smalis-msft@users.noreply.github.com> Date: Thu, 21 May 2026 20:13:27 -0400 Subject: [PATCH 7/7] relax em all --- vmm_core/virt_whp/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vmm_core/virt_whp/src/lib.rs b/vmm_core/virt_whp/src/lib.rs index b5c2fc8006..6657b6d17f 100644 --- a/vmm_core/virt_whp/src/lib.rs +++ b/vmm_core/virt_whp/src/lib.rs @@ -403,10 +403,10 @@ impl Vplc { start_vp, } = self; message_queues.clear(); - check_queues.store(false, Ordering::SeqCst); - extint_pending.store(false, Ordering::SeqCst); + check_queues.store(false, Ordering::Relaxed); + extint_pending.store(false, Ordering::Relaxed); *start_vp_context.lock() = None; - start_vp.store(false, Ordering::SeqCst); + start_vp.store(false, Ordering::Relaxed); } } @@ -1673,7 +1673,7 @@ impl<'p> virt::Processor for WhpProcessor<'p> { self.finish_reset(Vtl::Vtl2); self.vplc(Vtl::Vtl2).reset(); } - self.inner.vtl2_wake.store(false, Ordering::SeqCst); + self.inner.vtl2_wake.store(false, Ordering::Relaxed); if cfg!(debug_assertions) { let vp_info = &self.inner.vp_info; @@ -1695,7 +1695,7 @@ impl<'p> virt::Processor for WhpProcessor<'p> { // before `state.reset` so that `runnable_vtls` and `active_vtl` are // derived from the post-scrub set rather than the pre-scrub one. if !is_bsp { - self.inner.vtl2_enable.store(false, Ordering::SeqCst); + self.inner.vtl2_enable.store(false, Ordering::Relaxed); self.state.enabled_vtls.clear(Vtl::Vtl2); } @@ -1710,7 +1710,7 @@ impl<'p> virt::Processor for WhpProcessor<'p> { // Clear any pending VTL2 wake signal, since VTL2 is now back in // startup suspend and any prior wake request is stale. - self.inner.vtl2_wake.store(false, Ordering::SeqCst); + self.inner.vtl2_wake.store(false, Ordering::Relaxed); if cfg!(debug_assertions) { let vp_info = &self.inner.vp_info;