diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 40494c026767..6e68e8c3835d 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -11,6 +11,10 @@ component_files = { # hostos components Label("hostos/guestos/guestos.service"): "/etc/systemd/system/guestos.service", Label("hostos/guestos/upgrade-guestos.service"): "/etc/systemd/system/upgrade-guestos.service", + Label("multi-hostos/start-guestos.sh"): "/opt/ic/bin/start-guestos.sh", + Label("multi-hostos/start-guestos.service"): "/etc/systemd/system/start-guestos.service", + Label("multi-hostos/guestos@.service"): "/etc/systemd/system/guestos@.service", + Label("multi-hostos/guestos.target"): "/etc/systemd/system/guestos.target", Label("hostos/libvirt/setup-libvirt.sh"): "/opt/ic/bin/setup-libvirt.sh", Label("hostos/libvirt/setup-libvirt.service"): "/etc/systemd/system/setup-libvirt.service", Label("hostos/misc/setup-var.sh"): "/opt/ic/bin/setup-var.sh", @@ -42,6 +46,7 @@ component_files = { Label("misc/output-wrapper.sh"): "/opt/ic/bin/output-wrapper.sh", Label("misc/vsock/vsock-agent.service"): "/etc/systemd/system/vsock-agent.service", Label("misc/vsock/10-vhost-vsock.rules"): "/etc/udev/rules.d/10-vhost-vsock.rules", + Label("multi-hostos/skip-vsock.sh"): "/opt/ic/bin/skip-vsock.sh", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", Label("misc/chrony/chrony-var.service"): "/etc/systemd/system/chrony-var.service", Label("hostos/misc/sudoers"): "/etc/sudoers", diff --git a/ic-os/components/hostos/guestos/guestos.service b/ic-os/components/hostos/guestos/guestos.service index fc9c73f5a7a2..ec2949da7cf2 100644 --- a/ic-os/components/hostos/guestos/guestos.service +++ b/ic-os/components/hostos/guestos/guestos.service @@ -15,6 +15,3 @@ ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm Restart=always RestartSec=60 TimeoutStopSec=130s - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/hostos/update-config/update-config.service b/ic-os/components/hostos/update-config/update-config.service index b17f43638763..c322c4e96ca7 100644 --- a/ic-os/components/hostos/update-config/update-config.service +++ b/ic-os/components/hostos/update-config/update-config.service @@ -2,7 +2,7 @@ Description=Update HostOS Configuration RequiresMountsFor=/boot/config Before=node_exporter.service -Before=guestos.service +Before=guestos.target Before=log-config.service [Service] diff --git a/ic-os/components/hostos/verbose-logging/verbose-logging.service b/ic-os/components/hostos/verbose-logging/verbose-logging.service index 09bee517f73a..fdcc193474a9 100644 --- a/ic-os/components/hostos/verbose-logging/verbose-logging.service +++ b/ic-os/components/hostos/verbose-logging/verbose-logging.service @@ -1,7 +1,7 @@ [Unit] Description=If verbose flag enabled, pipe GuestOS console to the Host terminal -Wants=guestos.service -After=guestos.service +Wants=guestos.target +After=guestos.target [Service] ExecStart=/opt/ic/bin/verbose-logging.sh diff --git a/ic-os/components/misc/vsock/vsock-agent.service b/ic-os/components/misc/vsock/vsock-agent.service index 6d1bd8c9c4a5..6c98068d8f06 100644 --- a/ic-os/components/misc/vsock/vsock-agent.service +++ b/ic-os/components/misc/vsock/vsock-agent.service @@ -2,8 +2,9 @@ Description=VSOCK agent daemon [Service] +ExecCondition=/opt/ic/bin/skip-vsock.sh ExecStart=/opt/ic/bin/vsock_host -Restart=always +Restart=on-failure [Install] WantedBy=multi-user.target diff --git a/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh b/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh index b64a79578017..8a0b0c740fcd 100644 --- a/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh +++ b/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh @@ -1,10 +1,34 @@ #!/bin/bash +set -euo pipefail # Export GuestOS serial logs to journald +# Strip ANSI color/escape codes from serial console output before forwarding to journald -set -e +source /opt/ic/bin/config.sh -# Strip ANSI color/escape codes from serial console output before forwarding to journald -tail -F /var/log/libvirt/qemu/guestos-serial.log | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t guestos-serial -p info & +node_reward_type=$(get_config_value '.icos_settings.node_reward_type') + +case "${node_reward_type}" in + type4.1) COUNT=32 ;; + type4.2) COUNT=8 ;; + type4.3) COUNT=4 ;; + type4.4) COUNT=2 ;; + *) COUNT=1 ;; +esac + +# Forward all the GuestOS logs +for i in $(seq 0 "$((COUNT - 1))"); do + # A single GuestOS keeps the guestos-serial.log name + if [ "$COUNT" -eq 1 ]; then + s="" + else + s=$i + fi + + tail -F "/var/log/libvirt/qemu/guestos-serial$s.log" | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t "guestos-serial$s" -p info & +done + +# And the upgrade VM tail -F /var/log/libvirt/qemu/upgrade-guestos-serial.log | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t upgrade-guestos-serial -p info & + wait diff --git a/ic-os/components/monitoring/hostos/monitor-guestos.service b/ic-os/components/monitoring/hostos/monitor-guestos.service index 358b7201f0c6..bc52951fc203 100644 --- a/ic-os/components/monitoring/hostos/monitor-guestos.service +++ b/ic-os/components/monitoring/hostos/monitor-guestos.service @@ -1,6 +1,7 @@ [Unit] Description=Monitor GuestOS virtual machine -After=guestos.service +Wants=guestos.target +After=guestos.target [Service] Type=oneshot diff --git a/ic-os/components/multi-hostos/guestos.target b/ic-os/components/multi-hostos/guestos.target new file mode 100644 index 000000000000..0bef6956c108 --- /dev/null +++ b/ic-os/components/multi-hostos/guestos.target @@ -0,0 +1,2 @@ +[Unit] +Description=All GuestOS VMs diff --git a/ic-os/components/multi-hostos/guestos@.service b/ic-os/components/multi-hostos/guestos@.service new file mode 100644 index 000000000000..752bf816d3b2 --- /dev/null +++ b/ic-os/components/multi-hostos/guestos@.service @@ -0,0 +1,17 @@ +[Unit] +Description=Manage GuestOS %i virtual machine +# Not Requires, otherwise guestos will restart when libvirtd restarts +Wants=libvirtd.service +After=libvirtd.service +RequiresMountsFor=/var +PartOf=guestos.target + +[Service] +Type=notify +ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS %i service...\n\n" | tee /dev/tty1 > /dev/ttyS0' +ExecStartPre=/opt/ic/bin/detect-first-boot.sh +ExecStart=/opt/ic/bin/guest_vm_runner run --slot %i +ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm +Restart=always +RestartSec=60 +TimeoutStopSec=130s diff --git a/ic-os/components/multi-hostos/skip-vsock.sh b/ic-os/components/multi-hostos/skip-vsock.sh new file mode 100644 index 000000000000..307c357305de --- /dev/null +++ b/ic-os/components/multi-hostos/skip-vsock.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +source /opt/ic/bin/config.sh + +node_reward_type="$(get_config_value '.icos_settings.node_reward_type')" + +if [[ "$node_reward_type" =~ ^type4(\.[0-9]+)?$ ]]; then + echo "node_reward_type=${node_reward_type}; skipping vsock-agent" + exit 1 +fi + +exit 0 diff --git a/ic-os/components/multi-hostos/start-guestos.service b/ic-os/components/multi-hostos/start-guestos.service new file mode 100644 index 000000000000..dfec1d7937ab --- /dev/null +++ b/ic-os/components/multi-hostos/start-guestos.service @@ -0,0 +1,13 @@ +[Unit] +Description=Startup GuestOS VMs +RequiresMountsFor=/boot/config +Wants=guestos.target +Before=guestos.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/opt/ic/bin/start-guestos.sh + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/start-guestos.sh b/ic-os/components/multi-hostos/start-guestos.sh new file mode 100644 index 000000000000..f72a4ba8c210 --- /dev/null +++ b/ic-os/components/multi-hostos/start-guestos.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +# This script dynamically starts guestos services to create the right number of VMs per node type + +source /opt/ic/bin/config.sh + +node_reward_type=$(get_config_value '.icos_settings.node_reward_type') + +case "${node_reward_type}" in + type4.1) COUNT=32 ;; + type4.2) COUNT=8 ;; + type4.3) COUNT=4 ;; + type4.4) COUNT=2 ;; + *) COUNT=1 ;; +esac + +# If not a type4 node, fall to the default GuestOS flow +if ((COUNT == 1)); then + systemctl start guestos.service || true + exit 0 +fi + +eval systemctl start guestos@{0..$((COUNT - 1))}.service || true diff --git a/ic-os/components/setupos/install-guestos.sh b/ic-os/components/setupos/install-guestos.sh index f2b4b5b450ed..3a2f542c32f9 100755 --- a/ic-os/components/setupos/install-guestos.sh +++ b/ic-os/components/setupos/install-guestos.sh @@ -11,7 +11,7 @@ source /opt/ic/bin/functions.sh LV="/dev/mapper/hostlvm-guestos" function install_guestos() { - echo "* Installing GuestOS disk-image..." + echo "* Installing GuestOS disk-images..." vgchange -ay hostlvm log_and_halt_installation_on_error "${?}" "Unable to activate HostOS volume group." @@ -21,9 +21,11 @@ function install_guestos() { tar xaf /data/guest-os.img.tar.zst -C "${TMPDIR}" disk.img log_and_halt_installation_on_error "${?}" "Unable to extract GuestOS disk-image." - echo "* Writing the GuestOS image to ${LV}..." - dd if="${TMPDIR}/disk.img" of=${LV} bs=10M conv=sparse status=progress - log_and_halt_installation_on_error "${?}" "Unable to install GuestOS disk-image." + for lv in "${LV}"*; do + echo "* Writing the GuestOS image to ${lv}..." + dd if="${TMPDIR}/disk.img" of="${lv}" bs=10M conv=sparse status=progress + log_and_halt_installation_on_error "${?}" "Unable to install GuestOS disk-image to ${lv}." + done rm -rf "${TMPDIR}" diff --git a/ic-os/components/setupos/install-hostos.sh b/ic-os/components/setupos/install-hostos.sh index 6fea50a15fd2..c0ee5dd6530d 100755 --- a/ic-os/components/setupos/install-hostos.sh +++ b/ic-os/components/setupos/install-hostos.sh @@ -6,6 +6,7 @@ set -o pipefail SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/config.sh source /opt/ic/bin/functions.sh function install_hostos() { @@ -96,9 +97,44 @@ function resize_partition() { log_and_halt_installation_on_error "${?}" "Unable to include PV '/dev/${drive}' in VG." done - # Extend GuestOS LV to fill VG space - lvextend -i "${count}" --type striped -l +100%FREE /dev/hostlvm/guestos >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to extend logical volume: /dev/hostlvm/guestos" + local node_reward_type=$(get_config_value '.icos_settings.node_reward_type') + + # Configure multiple GuestOS if type4.X + if [[ $node_reward_type =~ ^type4(\.[0-9]+)?$ ]]; then + # Cleanup the initial GuestOS + lvremove -f hostlvm/guestos >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to cleanup initial GuestOS volume" + + # And set up new split volumes + free=$(vgs --noheadings -o vg_free_count hostlvm | tr -d ' ') + + create_guestos_lvs() { + local total="$1" + local each=$((free / total)) + local last=$((total - 1)) + local i + + for ((i = 0; i < last; i++)); do + lvcreate -i "${count}" --type striped -l "${each}" -n "guestos${i}" hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + done + + lvcreate -i "${count}" --type striped -l 100%FREE -n "guestos${last}" hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + } + + case "${node_reward_type}" in + type4.1) create_guestos_lvs 32 ;; + type4.2) create_guestos_lvs 8 ;; + type4.3) create_guestos_lvs 4 ;; + type4.4) create_guestos_lvs 2 ;; + esac + # "Normal" behavior + else + # Extend GuestOS LV to fill VG space + lvextend -i "${count}" --type striped -l +100%FREE /dev/hostlvm/guestos >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to extend logical volume: /dev/hostlvm/guestos" + fi } # Establish run order diff --git a/rs/ic_os/config/tool/src/hostos/guestos_config.rs b/rs/ic_os/config/tool/src/hostos/guestos_config.rs index b392bf83b9e1..0736d10283b7 100644 --- a/rs/ic_os/config/tool/src/hostos/guestos_config.rs +++ b/rs/ic_os/config/tool/src/hostos/guestos_config.rs @@ -4,7 +4,7 @@ use config_types::{ GuestVMType, HostOSConfig, Ipv6Config, RecoveryConfig, TrustedExecutionEnvironmentConfig, }; use deterministic_ips::node_type::NodeType; -use deterministic_ips::{MacAddr6Ext, calculate_deterministic_mac}; +use deterministic_ips::{MacAddr6Ext, calculate_deterministic_mac_w_slot}; use std::net::Ipv6Addr; use std::path::Path; use utils::to_cidr; @@ -18,6 +18,18 @@ pub fn generate_guestos_config( hostos_config: &HostOSConfig, guest_vm_type: GuestVMType, sev_certificate_chain_pem: Option, +) -> Result { + generate_guestos_config_w_slot(hostos_config, 0, guest_vm_type, sev_certificate_chain_pem) +} + +/// Generate the GuestOS configuration based on the provided HostOS configuration. +/// If hostos_config.icos_settings.enable_trusted_execution_environment is true, +/// sev_certificate_chain_pem must be provided. +pub fn generate_guestos_config_w_slot( + hostos_config: &HostOSConfig, + guest_vm_slot: usize, + guest_vm_type: GuestVMType, + sev_certificate_chain_pem: Option, ) -> Result { ensure!( !hostos_config @@ -47,9 +59,14 @@ pub fn generate_guestos_config( } }; - let guestos_ipv6_address = - node_ipv6_address(node_type, hostos_config, deterministic_ipv6_config)?; + let guestos_ipv6_address = node_ipv6_address( + guest_vm_slot, + node_type, + hostos_config, + deterministic_ipv6_config, + )?; let peer_ipv6_address = node_ipv6_address( + 0, upgrade_peer_node_type, hostos_config, deterministic_ipv6_config, @@ -90,13 +107,15 @@ pub fn generate_guestos_config( } fn node_ipv6_address( + slot: usize, node_type: NodeType, hostos_config: &HostOSConfig, deterministic_config: &DeterministicIpv6Config, ) -> Result { - let mac = calculate_deterministic_mac( + let mac = calculate_deterministic_mac_w_slot( &hostos_config.icos_settings.mgmt_mac, hostos_config.icos_settings.deployment_environment, + slot, node_type, ); diff --git a/rs/ic_os/networking/deterministic_ips/src/lib.rs b/rs/ic_os/networking/deterministic_ips/src/lib.rs index 67d4a22529ba..f011f5401254 100644 --- a/rs/ic_os/networking/deterministic_ips/src/lib.rs +++ b/rs/ic_os/networking/deterministic_ips/src/lib.rs @@ -65,20 +65,47 @@ pub fn calculate_deterministic_mac( deployment_environment: DeploymentEnvironment, node_type: NodeType, ) -> MacAddr6 { - let index = node_type.to_index(); + calculate_deterministic_mac_w_slot(mgmt_mac, deployment_environment, 0, node_type) +} - // NOTE: In order to be backwards compatible with existing scripts, this - // **MUST** have a newline. - let seed = format!( - "{}{}\n", - mgmt_mac.to_string().to_lowercase(), - deployment_environment - ); +pub fn calculate_deterministic_mac_w_slot( + mgmt_mac: &MacAddr6, + deployment_environment: DeploymentEnvironment, + slot: usize, + node_type: NodeType, +) -> MacAddr6 { + if slot < 16 { + let index = node_type.to_index() | (slot as u8) << 4; // Mix in the slot index + + // NOTE: In order to be backwards compatible with existing scripts, this + // **MUST** have a newline. + let seed = format!( + "{}{}\n", + mgmt_mac.to_string().to_lowercase(), + deployment_environment + ); - let hash = Sha256::hash(seed.as_bytes()); + let hash = Sha256::hash(seed.as_bytes()); - // 0x6a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing. - [0x6a, index, hash[0], hash[1], hash[2], hash[3]].into() + // 0x6a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing. + [0x6a, index, hash[0], hash[1], hash[2], hash[3]].into() + } else { + let index = node_type.to_index(); + + // NOTE: In order to be backwards compatible with existing scripts, this + // **MUST** have a newline. + let seed = format!( + "{}{}\n", + mgmt_mac.to_string().to_lowercase(), + deployment_environment + ); + + let hash = Sha256::hash(seed.as_bytes()); + + // NOTE: We extend to 7a to shrink the hash, so we can fit more slot info into the address. + // 0x7a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing. + [0x7a, index, slot as u8, hash[0], hash[1], hash[2]].into() + } } #[cfg(test)] diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs index d770eb2cbfc5..a9cd83e6c845 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs @@ -3,10 +3,12 @@ use crate::metrics::GuestVmMetrics; use anyhow::{Context, Result, ensure}; use askama::Template; use config_tool::hostos::guestos_bootstrap_image::BootstrapOptions; -use config_tool::hostos::guestos_config::generate_guestos_config; +use config_tool::hostos::guestos_config::{ + generate_guestos_config, generate_guestos_config_w_slot, +}; use config_types::{GuestOSConfig, HostOSConfig}; -use deterministic_ips::calculate_deterministic_mac; use deterministic_ips::node_type::NodeType; +use deterministic_ips::{calculate_deterministic_mac, calculate_deterministic_mac_w_slot}; use std::path::{Path, PathBuf}; use tracing::info; @@ -31,6 +33,7 @@ pub struct GuestOSTemplateProps { pub console_log_path: String, pub vm_memory: u32, pub nr_of_vcpus: u32, + pub topology: Topology, pub mac_address: macaddr::MacAddr6, pub disk_device: PathBuf, pub config_media_path: PathBuf, @@ -51,17 +54,33 @@ pub struct DirectBootConfig { pub kernel_cmdline: String, } +pub struct Topology { + pub nr_of_sockets: u32, + pub nr_of_cores: u32, + pub nr_of_threads: u32, +} + pub fn assemble_config_media( hostos_config: &HostOSConfig, + guest_vm_slot: Option, guest_vm_type: GuestVMType, sev_certificate_chain_pem: Option, media_path: &Path, ) -> Result<()> { - let guestos_config = generate_guestos_config( - hostos_config, - guest_vm_type.to_config_type(), - sev_certificate_chain_pem, - ) + let guestos_config = if let Some(slot) = guest_vm_slot { + generate_guestos_config_w_slot( + hostos_config, + slot, + guest_vm_type.to_config_type(), + sev_certificate_chain_pem, + ) + } else { + generate_guestos_config( + hostos_config, + guest_vm_type.to_config_type(), + sev_certificate_chain_pem, + ) + } .context("Failed to generate GuestOS config")?; let bootstrap_options = make_bootstrap_options(hostos_config, guestos_config)?; @@ -105,6 +124,7 @@ pub fn generate_vm_config( direct_boot: Option, disk_device: &Path, serial_log_path: &Path, + guest_vm_slot: Option, guest_vm_type: GuestVMType, available_hugepages_gib: u64, metrics: &GuestVmMetrics, @@ -113,25 +133,37 @@ pub fn generate_vm_config( GuestVMType::Default => NodeType::GuestOS, GuestVMType::Upgrade => NodeType::UpgradeGuestOS, }; - let mac_address = calculate_deterministic_mac( - &config.icos_settings.mgmt_mac, - config.icos_settings.deployment_environment, - node_type, - ); + let mac_address = if let Some(slot) = guest_vm_slot { + calculate_deterministic_mac_w_slot( + &config.icos_settings.mgmt_mac, + config.icos_settings.deployment_environment, + slot, + node_type, + ) + } else { + calculate_deterministic_mac( + &config.icos_settings.mgmt_mac, + config.icos_settings.deployment_environment, + node_type, + ) + }; let (cpu_domain, total_vm_memory, nr_of_vcpus) = vm_resources(config); + let (vm_memory, nr_of_vcpus, topology) = + split_resources_for_type_4(config, total_vm_memory, nr_of_vcpus); // We need 4GB for the upgrade VM. We subtract that from the total memory. This is not // necessary when SEV is disabled (since no upgrade VM is needed) but mixed subnets that // contain nodes with and without SEV should have the same memory settings for consistency // across nodes. ensure!( - total_vm_memory >= UPGRADE_VM_MEMORY_GIB, + vm_memory >= UPGRADE_VM_MEMORY_GIB, "GuestOS VM memory must be at least {UPGRADE_VM_MEMORY_GIB}GiB but is {total_vm_memory}GiB." ); - let vm_memory_gib = match guest_vm_type { - GuestVMType::Default => total_vm_memory - UPGRADE_VM_MEMORY_GIB, - GuestVMType::Upgrade => UPGRADE_VM_MEMORY_GIB, + let vm_memory_gib = match (guest_vm_type, &config.icos_settings.node_reward_type) { + (GuestVMType::Default, Some(val)) if val.starts_with("type4") => vm_memory, + (GuestVMType::Default, _) => vm_memory - UPGRADE_VM_MEMORY_GIB, + (GuestVMType::Upgrade, _) => UPGRADE_VM_MEMORY_GIB, }; // Enable hugepages if enough are available for this VM and TEE is disabled (hugepages are not @@ -142,13 +174,14 @@ pub fn generate_vm_config( metrics.set_hugepages_enabled(guest_vm_type, use_hugepages); GuestOSTemplateProps { - domain_name: vm_domain_name(guest_vm_type).to_string(), - domain_uuid: vm_domain_uuid(guest_vm_type).to_string(), + domain_name: vm_domain_name(guest_vm_type, guest_vm_slot), + domain_uuid: vm_domain_uuid(guest_vm_type, guest_vm_slot), disk_device: disk_device.to_path_buf(), cpu_domain, console_log_path: serial_log_path.display().to_string(), vm_memory: vm_memory_gib, nr_of_vcpus, + topology, mac_address, config_media_path: media_path.to_path_buf(), direct_boot, @@ -178,30 +211,63 @@ pub(crate) fn vm_resources(_config: &HostOSConfig) -> (String, u32, u32) { ("kvm".to_string(), DEFAULT_VM_MEMORY_GIB, DEFAULT_VM_VCPUS) } -pub fn vm_domain_name(guest_vm_type: GuestVMType) -> &'static str { +fn split_resources_for_type_4( + config: &HostOSConfig, + memory: u32, + vcpus: u32, +) -> (u32, u32, Topology) { + let (memory, vcpus) = match &config.icos_settings.node_reward_type { + Some(val) if val == "type4.1" => (memory / 32, vcpus / 32), + Some(val) if val == "type4.2" => (memory / 8, vcpus / 8), + Some(val) if val == "type4.3" => (memory / 4, vcpus / 4), + Some(val) if val == "type4.4" => (memory / 2, vcpus / 2), + _ => (memory, vcpus), + }; + + let topology = match &config.icos_settings.node_reward_type { + Some(val) if val == "type4.1" => Topology { + nr_of_sockets: 1, + nr_of_cores: vcpus, + nr_of_threads: 1, + }, + _ => Topology { + nr_of_sockets: 2, + nr_of_cores: vcpus / 4, + nr_of_threads: 2, + }, + }; + + (memory, vcpus, topology) +} + +pub fn vm_domain_name(guest_vm_type: GuestVMType, slot: Option) -> String { + let slot_suffix = slot.map(|v| v.to_string()).unwrap_or_default(); match guest_vm_type { - GuestVMType::Default => DEFAULT_GUEST_VM_DOMAIN_NAME, - GuestVMType::Upgrade => UPGRADE_GUEST_VM_DOMAIN_NAME, + GuestVMType::Default => format!("{DEFAULT_GUEST_VM_DOMAIN_NAME}{slot_suffix}"), + GuestVMType::Upgrade => UPGRADE_GUEST_VM_DOMAIN_NAME.to_string(), } } -pub fn vm_domain_uuid(guest_vm_type: GuestVMType) -> &'static str { +pub fn vm_domain_uuid(guest_vm_type: GuestVMType, slot: Option) -> String { + let slot = slot.unwrap_or(0x66); match guest_vm_type { - GuestVMType::Default => "fd897da5-8017-41c8-8575-a706dba30766", - GuestVMType::Upgrade => "1ea49839-7f46-4560-a4c7-fce677bbfbbd", + GuestVMType::Default => format!("fd897da5-8017-41c8-8575-a706dba307{slot:02x}"), + GuestVMType::Upgrade => "1ea49839-7f46-4560-a4c7-fce677bbfbbd".to_string(), } } -pub fn serial_log_path(guest_vm_type: GuestVMType) -> &'static Path { +pub fn serial_log_path(guest_vm_type: GuestVMType, slot: Option) -> PathBuf { + let slot_suffix = slot.map(|v| v.to_string()).unwrap_or_default(); match guest_vm_type { - GuestVMType::Default => Path::new(DEFAULT_SERIAL_LOG_PATH), - GuestVMType::Upgrade => Path::new(UPGRADE_SERIAL_LOG_PATH), + GuestVMType::Default => PathBuf::from(format!("{DEFAULT_SERIAL_LOG_PATH}{slot_suffix}")), + GuestVMType::Upgrade => PathBuf::from(UPGRADE_SERIAL_LOG_PATH.to_string()), } } #[cfg(all(test, not(feature = "skip_default_tests")))] mod tests { use super::*; + use config_tool::hostos::guestos_config::generate_guestos_config; use config_types::{ DeterministicIpv6Config, HostOSConfig, HostOSDevSettings, HostOSSettings, ICOSSettings, Ipv6Config, NetworkSettings, @@ -316,6 +382,7 @@ mod tests { direct_boot, Path::new("/dev/guest_disk"), Path::new("/var/serial/console.txt"), + None, guest_vm_type, available_hugepages_gib, &metrics, @@ -406,7 +473,7 @@ mod tests { let media_path = temp_dir.path().join("config.img"); let config = create_test_hostos_config(); - let result = assemble_config_media(&config, GuestVMType::Upgrade, None, &media_path); + let result = assemble_config_media(&config, None, GuestVMType::Upgrade, None, &media_path); assert!( result.is_ok(), diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/main.rs b/rs/ic_os/os_tools/guest_vm_runner/src/main.rs index 34f0c1ffc7ac..e5e02ddfafc4 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/main.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/main.rs @@ -93,6 +93,9 @@ impl GuestVMType { enum Command { /// Run the GuestOS virtual machine Run { + #[arg(long = "slot")] + slot: Option, + #[arg(long = "type", default_value = "default", value_enum)] vm_type: GuestVMType, }, @@ -121,11 +124,11 @@ pub async fn main() -> Result<()> { match args.command { Command::ReserveHugepages => reserve_hugepages(), - Command::Run { vm_type } => run(vm_type).await, + Command::Run { slot, vm_type } => run(slot, vm_type).await, } } -async fn run(vm_type: GuestVMType) -> Result<()> { +async fn run(slot: Option, vm_type: GuestVMType) -> Result<()> { let startup_message = match vm_type { GuestVMType::Default => "Launching GuestOS Virtual Machine...", GuestVMType::Upgrade => "Launching Upgrade GuestOS Virtual Machine...", @@ -141,7 +144,7 @@ async fn run(vm_type: GuestVMType) -> Result<()> { let termination_token = CancellationToken::new(); setup_signal_handler(termination_token.clone()).context("Failed to setup signal handler")?; - GuestVmService::create_and_run(vm_type, termination_token).await + GuestVmService::create_and_run(slot, vm_type, termination_token).await } fn setup_signal_handler(termination_token: CancellationToken) -> Result<()> { @@ -414,6 +417,7 @@ pub struct GuestVmService { hostos_config: HostOSConfig, systemd_notifier: Arc, console_ttys: Vec>>, + guest_vm_slot: Option, guest_vm_type: GuestVMType, sev_certificate_provider: HostSevCertificateProvider, disk_device: PathBuf, @@ -432,7 +436,7 @@ impl GuestVmService { } #[cfg(target_os = "linux")] - pub fn new(guest_vm_type: GuestVMType) -> Result { + pub fn new(guest_vm_slot: Option, guest_vm_type: GuestVMType) -> Result { let metrics = GuestVmMetrics::new(PathBuf::from(Self::metrics_path(guest_vm_type))) .context("Failed to create metrics")?; let libvirt_connection = LibvirtConnectionWithReconnect::new(Arc::new(|| { @@ -469,15 +473,18 @@ impl GuestVmService { }) .transpose()?; + let slot_suffix = guest_vm_slot.map(|v| v.to_string()).unwrap_or_default(); + let device_string = format!("{GUESTOS_DEVICE}{slot_suffix}"); let disk_device = upgrade_mapped_device .as_ref() .map(|x| x.path()) - .unwrap_or(Path::new(GUESTOS_DEVICE)); + .unwrap_or(Path::new(&device_string)); Ok(Self { metrics, libvirt_connection: Arc::new(libvirt_connection), hostos_config, + guest_vm_slot, guest_vm_type, systemd_notifier: Arc::new(systemd_notifier::DefaultSystemdNotifier), console_ttys: vec![ @@ -492,17 +499,18 @@ impl GuestVmService { disk_device: disk_device.to_path_buf(), _upgrade_mapped_device: upgrade_mapped_device, guestos_boot_timeout: GUESTOS_BOOT_TIMEOUT, - vm_serial_log_path: serial_log_path(guest_vm_type).to_path_buf(), + vm_serial_log_path: serial_log_path(guest_vm_type, guest_vm_slot).to_path_buf(), command_runner: Arc::new(RealAsyncCommandRunner), }) } #[cfg(target_os = "linux")] pub async fn create_and_run( + slot: Option, guest_vm_type: GuestVMType, termination_token: CancellationToken, ) -> Result<()> { - let mut guest_vm_service = Self::new(guest_vm_type)?; + let mut guest_vm_service = Self::new(slot, guest_vm_type)?; guest_vm_service.run(termination_token).await } @@ -567,7 +575,7 @@ impl GuestVmService { async fn start_virtual_machine(&mut self) -> Result { VirtualMachine::try_destroy_existing_vm( self.libvirt_connection.as_ref(), - vm_domain_name(self.guest_vm_type), + &vm_domain_name(self.guest_vm_type, self.guest_vm_slot), self.command_runner.as_ref(), ) .await?; @@ -609,6 +617,7 @@ impl GuestVmService { assemble_config_media( &self.hostos_config, + self.guest_vm_slot, self.guest_vm_type, sev_certificate_chain_pem, config_media.path(), @@ -624,6 +633,7 @@ impl GuestVmService { direct_boot.as_ref().map(DirectBoot::to_config), &self.disk_device, &self.vm_serial_log_path, + self.guest_vm_slot, self.guest_vm_type, available_hugepages_gib, &self.metrics, @@ -637,7 +647,7 @@ impl GuestVmService { &vm_config, config_media, direct_boot, - vm_domain_name(self.guest_vm_type), + &vm_domain_name(self.guest_vm_type, self.guest_vm_slot), ) .await?; @@ -1053,6 +1063,7 @@ mod tests { .unwrap(), ), guest_vm_type, + guest_vm_slot: None, sev_certificate_provider, disk_device: GUESTOS_DEVICE.into(), _upgrade_mapped_device: None, @@ -1072,7 +1083,7 @@ mod tests { systemd_notifier, termination_token, libvirt_connection: self.libvirt_connection.clone(), - vm_domain_name: vm_domain_name(guest_vm_type).to_string(), + vm_domain_name: vm_domain_name(guest_vm_type, None).to_string(), _sev_certificate_cache_dir: sev_certificate_cache_dir, } } diff --git a/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml b/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml index 4d01b5670545..cea127e09348 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml +++ b/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml @@ -20,7 +20,7 @@ {% else %} - + {%- if enable_sev %}