Skip to content

vmm_core: decouple chipset MMIO ranges from MemoryLayout #3550

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jstarks wants to merge 4 commits into
base: main
Choose a base branch
from
Open

vmm_core: decouple chipset MMIO ranges from MemoryLayout #3550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
33a5119
vmm_core: decouple chipset MMIO ranges from MemoryLayout
jstarks May 21, 2026
928797d
tweaks
jstarks May 22, 2026
505c089
feedback
jstarks May 23, 2026
6da9cdb
cleanup
jstarks May 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 10 additions & 11 deletions openhcl/underhill_core/src/loader/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

use self::vtl2_config::RuntimeParameters;
use crate::loader::vtl0_config::LinuxInfo;
use crate::worker::ChipsetMmioRanges;
use crate::worker::FirmwareType;
use cvm_tracing::CVM_ALLOWED;
use guest_emulation_transport::api::platform_settings::DevicePlatformSettings;
Expand Down Expand Up @@ -78,9 +79,6 @@ pub enum Error {
InvalidAcpiTableLength,
#[error("invalid acpi table: unknown header signature {0:?}")]
InvalidAcpiTableSignature([u8; 4]),
#[cfg(guest_arch = "x86_64")]
#[error("acpi tables require at least two mmio ranges")]
UnsupportedMmio,
#[cfg(guest_arch = "aarch64")]
#[error("expected GICv3 topology")]
ExpectedGicV3,
Expand Down Expand Up @@ -115,6 +113,7 @@ pub fn load(
config: Config,
caps: &virt::PartitionCapabilities,
isolated: bool,
chipset_mmio: &ChipsetMmioRanges,
) -> Result<VpContext, Error> {
let context = match load_kind {
LoadKind::None => {
Expand All @@ -137,6 +136,7 @@ pub fn load(
platform_config,
caps,
isolated,
chipset_mmio,
)?;
uefi_info.vp_context.clone()
}
Expand Down Expand Up @@ -188,6 +188,7 @@ pub fn load(
processor_topology,
platform_config,
chipset_capabilities,
chipset_mmio,
kernel_range: *kernel_range,
kernel_entrypoint: *kernel_entrypoint,
initrd: *initrd,
Expand Down Expand Up @@ -236,6 +237,7 @@ struct LoadLinuxParams<'a> {
processor_topology: &'a ProcessorTopology,
platform_config: &'a DevicePlatformSettings,
chipset_capabilities: VmChipsetCapabilities,
chipset_mmio: &'a ChipsetMmioRanges,
/// The region of memory used by the kernel.
kernel_range: MemoryRange,
/// The entrypoint of the kernel.
Expand All @@ -261,6 +263,7 @@ fn load_linux(params: LoadLinuxParams<'_>) -> Result<VpContext, Error> {
processor_topology,
platform_config,
chipset_capabilities,
chipset_mmio,
kernel_range,
kernel_entrypoint,
initrd,
Expand All @@ -287,11 +290,7 @@ fn load_linux(params: LoadLinuxParams<'_>) -> Result<VpContext, Error> {
},
};

if mem_layout.mmio().len() < 2 {
return Err(Error::UnsupportedMmio);
}

let acpi_tables = acpi_builder.build_acpi_tables(ACPI_BASE, |mem_layout, dsdt| {
let acpi_tables = acpi_builder.build_acpi_tables(ACPI_BASE, |dsdt| {
dsdt.add_apic();

// Add serial ports if enabled.
Expand All @@ -315,7 +314,7 @@ fn load_linux(params: LoadLinuxParams<'_>) -> Result<VpContext, Error> {
);
}

dsdt.add_mmio_module(mem_layout.mmio()[0], mem_layout.mmio()[1]);
dsdt.add_mmio_module(chipset_mmio.low, chipset_mmio.high);
// TODO: change this once PCI is running in underhill
dsdt.add_vmbus(false, None);
dsdt.add_rtc();
Expand Down Expand Up @@ -425,6 +424,7 @@ pub fn write_uefi_config(
platform_config: &DevicePlatformSettings,
caps: &virt::PartitionCapabilities,
isolated: bool,
chipset_mmio: &ChipsetMmioRanges,
) -> Result<(), Error> {
use guest_emulation_transport::api::platform_settings::UefiConsoleMode;

Expand Down Expand Up @@ -524,8 +524,7 @@ pub fn write_uefi_config(
)
.add_raw(
config::BlobStructureType::MmioRanges,
mem_layout
.mmio()
[chipset_mmio.low, chipset_mmio.high]
.iter()
.map(|range| config::Mmio {
mmio_page_number_start: range.start() / HV_PAGE_SIZE,
Expand Down
82 changes: 59 additions & 23 deletions openhcl/underhill_core/src/worker.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1158,13 +1158,28 @@ struct BuiltVtl0MemoryLayout {
complete_memory_layout: MemoryLayout,
}

/// Chipset MMIO ranges.
#[derive(Debug, Copy, Clone)]
pub(crate) struct ChipsetMmioRanges {
/// Chipset low MMIO range (below 4 GB) for VMOD/PCI0 _CRS. Always at
/// least the architectural reserved zone (LAPIC, IOAPIC, TPM, ...).
pub low: MemoryRange,
/// Chipset high MMIO range (above RAM) for VMOD/PCI0 _CRS. `EMPTY` when
/// no chipset high MMIO is configured.
pub high: MemoryRange,
}

/// Build the VTL0 memory map after carving out any memory requested for shared
/// visibility memory to be used by VTL2.
fn build_vtl0_memory_layout(
vtl0_memory_map: Vec<(MemoryRangeWithNode, MemoryMapEntryType)>,
mmio: &[MemoryRange],
chipset_mmio: &ChipsetMmioRanges,
mut shared_pool_size: u64,
) -> anyhow::Result<BuiltVtl0MemoryLayout> {
let mmio: Vec<MemoryRange> = [chipset_mmio.low, chipset_mmio.high]
.into_iter()
.filter(|r| !r.is_empty())
.collect();
// Allocate shared_pool memory starting from the last (top of memory)
// continuing downward until the size is covered.
//
Expand Down Expand Up @@ -1220,13 +1235,13 @@ fn build_vtl0_memory_layout(
.collect::<Vec<_>>();

let vtl0_memory_layout =
MemoryLayout::new_from_ranges(&memory, mmio).context("invalid memory layout")?;
MemoryLayout::new_from_ranges(&memory, &mmio).context("invalid memory layout")?;

let complete_memory = vtl0_memory_map
.iter()
.map(|(entry, _typ)| entry.clone())
.collect::<Vec<_>>();
let complete_memory_layout = MemoryLayout::new_from_ranges(&complete_memory, mmio)
let complete_memory_layout = MemoryLayout::new_from_ranges(&complete_memory, &mmio)
.context("invalid complete memory layout")?;

tracing::info!(
Expand Down Expand Up @@ -1694,28 +1709,42 @@ async fn new_underhill_vm(
anyhow::bail!("cannot run the VPCI relay without the VMBus relay");
}

let mut vtl0_mmio;
let (vtl0_mmio, vpci_relay_mmio) = if enable_vpci_relay {
// Carve out enough VTL0 MMIO space for 64 devices.
// Construct chipset MMIO ranges from the positional convention in the
// device tree: [0] = low (below 4 GiB), [1] = high (above RAM).
let mut chipset_mmio = ChipsetMmioRanges {
low: boot_info
.vtl0_mmio
.first()
.copied()
.unwrap_or(MemoryRange::EMPTY),
high: boot_info
.vtl0_mmio
.get(1)
.copied()
.unwrap_or(MemoryRange::EMPTY),
};
Comment on lines +1712 to +1725

let vpci_relay_mmio = if enable_vpci_relay {
// Carve out enough VTL0 MMIO space from the high range for 64 devices.
let required_len = 64 * vpci_relay::VPCI_RELAY_MMIO_PER_DEVICE;
vtl0_mmio = boot_info.vtl0_mmio.clone();
if vtl0_mmio.last().is_none_or(|r| r.len() < required_len) {
if chipset_mmio.high.is_empty() || chipset_mmio.high.len() < required_len {
anyhow::bail!("too little VTL0 MMIO space to take for the VPCI relay");
}
let r = vtl0_mmio.last().unwrap();
let (rest, vpci) = r.split_at_offset(r.len() - required_len);
*vtl0_mmio.last_mut().unwrap() = rest;
(&vtl0_mmio, vpci)
let (rest, vpci) = chipset_mmio
.high
.split_at_offset(chipset_mmio.high.len() - required_len);
chipset_mmio.high = rest;
vpci
} else {
(&boot_info.vtl0_mmio, MemoryRange::EMPTY)
MemoryRange::EMPTY
};

let BuiltVtl0MemoryLayout {
vtl0_memory_map,
vtl0_memory_layout: mem_layout,
shared_pool,
complete_memory_layout,
} = build_vtl0_memory_layout(vtl0_memory_map, vtl0_mmio, shared_pool_size)?;
} = build_vtl0_memory_layout(vtl0_memory_map, &chipset_mmio, shared_pool_size)?;

// Determine if x2apic is supported so that the topology matches
// reality.
Expand Down Expand Up @@ -2446,6 +2475,8 @@ async fn new_underhill_vm(
let config = firmware_pcat::config::PcatBiosConfig {
processor_topology: processor_topology.clone(),
mem_layout: mem_layout.clone(),
chipset_low_mmio: chipset_mmio.low,
chipset_high_mmio: chipset_mmio.high,
srat: acpi_builder.build_srat(),
hibernation_enabled: dps.general.hibernation_enabled,
initial_generation_id,
Expand Down Expand Up @@ -3002,9 +3033,12 @@ async fn new_underhill_vm(
// By default on aarch64 send all MMIO accesses to the host.
None
} else {
let mut untrusted_mmio_ranges: Vec<_> = mem_layout.mmio().to_vec();
let chipset_mmio_ranges = [chipset_mmio.low, chipset_mmio.high]
.into_iter()
.filter(|r| !r.is_empty());
let mut untrusted_mmio_ranges: Vec<_> = chipset_mmio_ranges.clone().collect();
if vtom > 0 {
untrusted_mmio_ranges.extend(mem_layout.mmio().iter().map(|range| {
untrusted_mmio_ranges.extend(chipset_mmio_ranges.map(|range| {
MemoryRange::new((range.start() + vtom)..(range.end() + vtom))
}));
}
Expand Down Expand Up @@ -3592,6 +3626,7 @@ async fn new_underhill_vm(
load_kind,
&dps,
isolation.is_isolated(),
&chipset_mmio,
)
.instrument(tracing::info_span!("load_firmware", CVM_ALLOWED))
.await?;
Expand Down Expand Up @@ -3871,6 +3906,7 @@ async fn load_firmware(
load_kind: LoadKind,
dps: &DevicePlatformSettings,
isolated: bool,
chipset_mmio: &ChipsetMmioRanges,
) -> Result<(), anyhow::Error> {
let cmdline_append = match cmdline_append {
Some(cmdline) => CString::new(cmdline.as_bytes()).context("bad command line")?,
Expand All @@ -3891,19 +3927,19 @@ async fn load_firmware(
loader_config,
caps,
isolated,
chipset_mmio,
)
.context("failed to load firmware")?;

#[cfg(guest_arch = "x86_64")]
let registers = {
let crate::loader::VpContext::Vbs(mut registers) = vtl0_vp_context;
registers.extend(
loader::common::compute_variable_mtrrs(
mem_layout,
partition.caps().physical_address_width,
)
.context("Failed to compute variable mtrrs")?,
);
registers.extend(loader::common::compute_variable_mtrrs(
mem_layout,
partition.caps().physical_address_width,
chipset_mmio.low,
chipset_mmio.high,
));
registers
};
#[cfg(guest_arch = "aarch64")]
Expand Down
Loading
Loading