From 9415ee246f68b2c3f96b3745e99a111a71799839 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 12 Jun 2026 22:58:51 +0000 Subject: [PATCH 1/8] Create new variants in bazel --- ic-os/components/BUILD.bazel | 3 ++- ic-os/components/multi-hostos.bzl | 17 ++++++++++++++++ ic-os/components/multi-hostos/guestos.target | 3 +++ .../components/multi-hostos/guestos1.service | 20 +++++++++++++++++++ .../components/multi-hostos/guestos2.service | 20 +++++++++++++++++++ .../components/multi-hostos/guestos3.service | 20 +++++++++++++++++++ .../components/multi-hostos/guestos4.service | 20 +++++++++++++++++++ .../components/multi-hostos/guestos5.service | 20 +++++++++++++++++++ .../multi-hostos/monitor-guestos.service | 15 ++++++++++++++ .../multi-hostos/update-config.service | 14 +++++++++++++ .../multi-hostos/verbose-logging.service | 11 ++++++++++ ic-os/hostos/defs.bzl | 10 +++++++--- ic-os/hostos/envs/multi/BUILD.bazel | 12 +++++++++++ ic-os/setupos/defs.bzl | 8 +++++++- ic-os/setupos/envs/multi/BUILD.bazel | 15 ++++++++++++++ 15 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 ic-os/components/multi-hostos.bzl create mode 100644 ic-os/components/multi-hostos/guestos.target create mode 100644 ic-os/components/multi-hostos/guestos1.service create mode 100644 ic-os/components/multi-hostos/guestos2.service create mode 100644 ic-os/components/multi-hostos/guestos3.service create mode 100644 ic-os/components/multi-hostos/guestos4.service create mode 100644 ic-os/components/multi-hostos/guestos5.service create mode 100644 ic-os/components/multi-hostos/monitor-guestos.service create mode 100644 ic-os/components/multi-hostos/update-config.service create mode 100644 ic-os/components/multi-hostos/verbose-logging.service create mode 100644 ic-os/hostos/envs/multi/BUILD.bazel create mode 100644 ic-os/setupos/envs/multi/BUILD.bazel diff --git a/ic-os/components/BUILD.bazel b/ic-os/components/BUILD.bazel index ef7c64c8f6b6..aea74a342ed6 100644 --- a/ic-os/components/BUILD.bazel +++ b/ic-os/components/BUILD.bazel @@ -2,6 +2,7 @@ load("//bazel:defs.bzl", "write_stable_status_file_var") load("//ic-os/components/conformance_tests:defs.bzl", "check_unused_components_test") load("guestos.bzl", guestos_component_files = "component_files") load("hostos.bzl", hostos_component_files = "component_files") +load("multi-hostos.bzl", multi_hostos_component_files = "component_files") load("setupos.bzl", setupos_component_files = "component_files") package(default_visibility = ["//ic-os:__subpackages__"]) @@ -32,7 +33,7 @@ write_stable_status_file_var( varname = "STABLE_COMMIT_TIMESTAMP", ) -used_components = guestos_component_files("dev") | guestos_component_files("prod") | hostos_component_files | setupos_component_files +used_components = guestos_component_files("dev") | guestos_component_files("prod") | hostos_component_files | multi_hostos_component_files | setupos_component_files REPO_COMPONENTS = glob( ["**/*"], diff --git a/ic-os/components/multi-hostos.bzl b/ic-os/components/multi-hostos.bzl new file mode 100644 index 000000000000..d7dc6a8ecb18 --- /dev/null +++ b/ic-os/components/multi-hostos.bzl @@ -0,0 +1,17 @@ +""" +Override a few components for multi-hostos +""" + +load("hostos.bzl", hostos_component_files = "component_files") + +component_files = hostos_component_files | { + Label("multi-hostos/guestos1.service"): "/etc/systemd/system/guestos1.service", + Label("multi-hostos/guestos2.service"): "/etc/systemd/system/guestos2.service", + Label("multi-hostos/guestos3.service"): "/etc/systemd/system/guestos3.service", + Label("multi-hostos/guestos4.service"): "/etc/systemd/system/guestos4.service", + Label("multi-hostos/guestos5.service"): "/etc/systemd/system/guestos5.service", + Label("multi-hostos/guestos.target"): "/etc/systemd/system/guestos.target", + Label("multi-hostos/verbose-logging.service"): "/etc/systemd/system/verbose-logging.service", + Label("multi-hostos/monitor-guestos.service"): "/etc/systemd/system/monitor-guestos.service", + Label("multi-hostos/update-config.service"): "/etc/systemd/system/update-config.service", +} diff --git a/ic-os/components/multi-hostos/guestos.target b/ic-os/components/multi-hostos/guestos.target new file mode 100644 index 000000000000..178701f4fb04 --- /dev/null +++ b/ic-os/components/multi-hostos/guestos.target @@ -0,0 +1,3 @@ +[Unit] +Description=All GuestOS VMs +Wants=guestos1.service guestos2.service guestos3.service guestos4.service guestos5.service diff --git a/ic-os/components/multi-hostos/guestos1.service b/ic-os/components/multi-hostos/guestos1.service new file mode 100644 index 000000000000..50bc25cfeab8 --- /dev/null +++ b/ic-os/components/multi-hostos/guestos1.service @@ -0,0 +1,20 @@ +[Unit] +Description=Manage GuestOS 1 virtual machine +# Not Requires, otherwise guestos will restart when libvirtd restarts +Wants=libvirtd.service +After=libvirtd.service + +RequiresMountsFor=/var + +[Service] +Type=notify +ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' +ExecStartPre=/opt/ic/bin/detect-first-boot.sh +ExecStart=/opt/ic/bin/guest_vm_runner run --slot 0 +ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm +Restart=always +RestartSec=60 +TimeoutStopSec=130s + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos2.service b/ic-os/components/multi-hostos/guestos2.service new file mode 100644 index 000000000000..937398f66c54 --- /dev/null +++ b/ic-os/components/multi-hostos/guestos2.service @@ -0,0 +1,20 @@ +[Unit] +Description=Manage GuestOS virtual machine +# Not Requires, otherwise guestos will restart when libvirtd restarts +Wants=libvirtd.service +After=libvirtd.service + +RequiresMountsFor=/var + +[Service] +Type=notify +ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' +ExecStartPre=/opt/ic/bin/detect-first-boot.sh +ExecStart=/opt/ic/bin/guest_vm_runner run --slot 1 +ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm +Restart=always +RestartSec=60 +TimeoutStopSec=130s + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos3.service b/ic-os/components/multi-hostos/guestos3.service new file mode 100644 index 000000000000..563de1a2197c --- /dev/null +++ b/ic-os/components/multi-hostos/guestos3.service @@ -0,0 +1,20 @@ +[Unit] +Description=Manage GuestOS virtual machine +# Not Requires, otherwise guestos will restart when libvirtd restarts +Wants=libvirtd.service +After=libvirtd.service + +RequiresMountsFor=/var + +[Service] +Type=notify +ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' +ExecStartPre=/opt/ic/bin/detect-first-boot.sh +ExecStart=/opt/ic/bin/guest_vm_runner run --slot 2 +ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm +Restart=always +RestartSec=60 +TimeoutStopSec=130s + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos4.service b/ic-os/components/multi-hostos/guestos4.service new file mode 100644 index 000000000000..3b22e87f33c3 --- /dev/null +++ b/ic-os/components/multi-hostos/guestos4.service @@ -0,0 +1,20 @@ +[Unit] +Description=Manage GuestOS virtual machine +# Not Requires, otherwise guestos will restart when libvirtd restarts +Wants=libvirtd.service +After=libvirtd.service + +RequiresMountsFor=/var + +[Service] +Type=notify +ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' +ExecStartPre=/opt/ic/bin/detect-first-boot.sh +ExecStart=/opt/ic/bin/guest_vm_runner run --slot 3 +ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm +Restart=always +RestartSec=60 +TimeoutStopSec=130s + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos5.service b/ic-os/components/multi-hostos/guestos5.service new file mode 100644 index 000000000000..01f11b3fa33d --- /dev/null +++ b/ic-os/components/multi-hostos/guestos5.service @@ -0,0 +1,20 @@ +[Unit] +Description=Manage GuestOS virtual machine +# Not Requires, otherwise guestos will restart when libvirtd restarts +Wants=libvirtd.service +After=libvirtd.service + +RequiresMountsFor=/var + +[Service] +Type=notify +ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' +ExecStartPre=/opt/ic/bin/detect-first-boot.sh +ExecStart=/opt/ic/bin/guest_vm_runner run --slot 4 +ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm +Restart=always +RestartSec=60 +TimeoutStopSec=130s + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/monitor-guestos.service b/ic-os/components/multi-hostos/monitor-guestos.service new file mode 100644 index 000000000000..bc52951fc203 --- /dev/null +++ b/ic-os/components/multi-hostos/monitor-guestos.service @@ -0,0 +1,15 @@ +[Unit] +Description=Monitor GuestOS virtual machine +Wants=guestos.target +After=guestos.target + +[Service] +Type=oneshot +ExecStart=/opt/ic/bin/monitor-guestos.sh + +# Disable systemd start and stop logs +LogLevelMax=1 +SyslogLevel=2 + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/update-config.service b/ic-os/components/multi-hostos/update-config.service new file mode 100644 index 000000000000..c322c4e96ca7 --- /dev/null +++ b/ic-os/components/multi-hostos/update-config.service @@ -0,0 +1,14 @@ +[Unit] +Description=Update HostOS Configuration +RequiresMountsFor=/boot/config +Before=node_exporter.service +Before=guestos.target +Before=log-config.service + +[Service] +Type=oneshot +ExecStart=/opt/ic/bin/config_tool update-config +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/verbose-logging.service b/ic-os/components/multi-hostos/verbose-logging.service new file mode 100644 index 000000000000..fdcc193474a9 --- /dev/null +++ b/ic-os/components/multi-hostos/verbose-logging.service @@ -0,0 +1,11 @@ +[Unit] +Description=If verbose flag enabled, pipe GuestOS console to the Host terminal +Wants=guestos.target +After=guestos.target + +[Service] +ExecStart=/opt/ic/bin/verbose-logging.sh +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/hostos/defs.bzl b/ic-os/hostos/defs.bzl index 5de2eedd3570..e1a919c76d09 100644 --- a/ic-os/hostos/defs.bzl +++ b/ic-os/hostos/defs.bzl @@ -2,7 +2,8 @@ Hold manifest common to all HostOS variants. """ -load("//ic-os/components:hostos.bzl", "component_files") +load("//ic-os/components:hostos.bzl", default_component_files = "component_files") +load("//ic-os/components:multi-hostos.bzl", multi_component_files = "component_files") load("//toolchains/sysimage:toolchain.bzl", "lvm_image") # Declare the dependencies that we will have for the built filesystem images. @@ -15,7 +16,7 @@ def image_deps(mode, _malicious = False): Define all HostOS inputs. Args: - mode: Variant to be built, dev or prod. + mode: Variant to be built, dev or prod (or multi). _malicious: Unused, but currently needed to fit generic build structure. Returns: A dict containing inputs to build this image. @@ -42,7 +43,7 @@ def image_deps(mode, _malicious = False): # Set various configuration values "container_context_files": Label("//ic-os/hostos/context:context-files"), - "component_files": dict(component_files), # Make a copy because we might update it later + "component_files": dict(default_component_files if mode != "multi" else multi_component_files), # Make a copy because we might update it later "partition_table": Label("//ic-os/hostos:partitions.csv"), "volume_table": Label("//ic-os/hostos:volumes.csv"), "rootfs_size": "3G", @@ -86,6 +87,9 @@ def image_deps(mode, _malicious = False): # Allow root console access on dev console_override_label_1 = Label("//ic-os/hostos:console-override-dev") console_override_label_2 = Label("//ic-os/hostos:console-override-dev-alias") + elif "multi" in mode: + console_override_label_1 = Label("//ic-os/hostos:console-override-dev") + console_override_label_2 = Label("//ic-os/hostos:console-override-dev-alias") else: # Allow limited-console access on prod console_override_label_1 = Label("//ic-os/hostos:console-override-prod") diff --git a/ic-os/hostos/envs/multi/BUILD.bazel b/ic-os/hostos/envs/multi/BUILD.bazel new file mode 100644 index 000000000000..9c77673349ec --- /dev/null +++ b/ic-os/hostos/envs/multi/BUILD.bazel @@ -0,0 +1,12 @@ +load("//ic-os:defs.bzl", "icos_build") +load("//ic-os/hostos:defs.bzl", "image_deps") + +icos_images = icos_build( + name = "multi", + image_deps_func = image_deps, + visibility = [ + "//rs:ic-os-pkg", + "//rs:system-tests-pkg", + ], + vuln_scan = False, +) diff --git a/ic-os/setupos/defs.bzl b/ic-os/setupos/defs.bzl index 3ca1dcf86a37..e75932fa4d7e 100644 --- a/ic-os/setupos/defs.bzl +++ b/ic-os/setupos/defs.bzl @@ -17,7 +17,7 @@ def image_deps(mode, _malicious = False): Define all SetupOS inputs. Args: - mode: Variant to be built, dev or prod. + mode: Variant to be built, dev or prod (or multi). _malicious: Unused, but currently needed to fit generic build structure. Returns: A dict containing inputs to build this image. @@ -99,6 +99,12 @@ def _custom_partitions(mode): nns_urls = '["https://icp-api.io", "https://icp0.io", "https://ic0.app"]' include_nns_public_key_override = False deployment_environment = "mainnet" + elif mode == "multi": + guest_image = Label("//ic-os/guestos/envs/prod:disk-img.tar.zst") + host_image = Label("//ic-os/hostos/envs/multi:disk-img.tar.zst") + nns_urls = '["https://cloudflare.com/cdn-cgi/trace"]' + include_nns_public_key_override = True + deployment_environment = "testnet" else: fail("Unkown mode detected: " + mode) diff --git a/ic-os/setupos/envs/multi/BUILD.bazel b/ic-os/setupos/envs/multi/BUILD.bazel new file mode 100644 index 000000000000..5ba2c04301d2 --- /dev/null +++ b/ic-os/setupos/envs/multi/BUILD.bazel @@ -0,0 +1,15 @@ +load("//ic-os:defs.bzl", "icos_build") +load("//ic-os/dev-tools/bare_metal_deployment:tools.bzl", "launch_bare_metal") +load("//ic-os/setupos:defs.bzl", "image_deps") + +icos_images = icos_build( + name = "multi", + image_deps_func = image_deps, + upgrades = False, + vuln_scan = False, +) + +launch_bare_metal( + name = "launch_bare_metal", + image_zst_file = ":disk-img.tar.zst", +) From 8de88ca4058a0a52f6937b08a3ebaa9bca8a8881 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 12 Jun 2026 22:58:55 +0000 Subject: [PATCH 2/8] Changes to SetupOS install process --- ic-os/components/setupos/install-guestos.sh | 10 +++++--- ic-os/components/setupos/install-hostos.sh | 28 ++++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/ic-os/components/setupos/install-guestos.sh b/ic-os/components/setupos/install-guestos.sh index f2b4b5b450ed..3a2f542c32f9 100755 --- a/ic-os/components/setupos/install-guestos.sh +++ b/ic-os/components/setupos/install-guestos.sh @@ -11,7 +11,7 @@ source /opt/ic/bin/functions.sh LV="/dev/mapper/hostlvm-guestos" function install_guestos() { - echo "* Installing GuestOS disk-image..." + echo "* Installing GuestOS disk-images..." vgchange -ay hostlvm log_and_halt_installation_on_error "${?}" "Unable to activate HostOS volume group." @@ -21,9 +21,11 @@ function install_guestos() { tar xaf /data/guest-os.img.tar.zst -C "${TMPDIR}" disk.img log_and_halt_installation_on_error "${?}" "Unable to extract GuestOS disk-image." - echo "* Writing the GuestOS image to ${LV}..." - dd if="${TMPDIR}/disk.img" of=${LV} bs=10M conv=sparse status=progress - log_and_halt_installation_on_error "${?}" "Unable to install GuestOS disk-image." + for lv in "${LV}"*; do + echo "* Writing the GuestOS image to ${lv}..." + dd if="${TMPDIR}/disk.img" of="${lv}" bs=10M conv=sparse status=progress + log_and_halt_installation_on_error "${?}" "Unable to install GuestOS disk-image to ${lv}." + done rm -rf "${TMPDIR}" diff --git a/ic-os/components/setupos/install-hostos.sh b/ic-os/components/setupos/install-hostos.sh index 6fea50a15fd2..dece8e5a47ac 100755 --- a/ic-os/components/setupos/install-hostos.sh +++ b/ic-os/components/setupos/install-hostos.sh @@ -6,6 +6,7 @@ set -o pipefail SHELL="/bin/bash" PATH="/sbin:/bin:/usr/sbin:/usr/bin" +source /opt/ic/bin/config.sh source /opt/ic/bin/functions.sh function install_hostos() { @@ -96,9 +97,30 @@ function resize_partition() { log_and_halt_installation_on_error "${?}" "Unable to include PV '/dev/${drive}' in VG." done - # Extend GuestOS LV to fill VG space - lvextend -i "${count}" --type striped -l +100%FREE /dev/hostlvm/guestos >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to extend logical volume: /dev/hostlvm/guestos" + local node_reward_type=$(get_config_value '.icos_settings.node_reward_type') + + # Configure multiple GuestOS if type4.X + # TODO: Configure differently for each variant + if [[ $node_reward_type =~ ^type4(\.[0-9]+)?$ ]]; then + # Cleanup the initial GuestOS + lvremove -f hostlvm/guestos >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to cleanup initial GuestOS volume" + + # And set up new split volumes + free=$(vgs --noheadings -o vg_free_count hostlvm | tr -d ' ') + each=$((free / 5)) + for i in 1 2 3 4; do + lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + done + lvcreate -i "${count}" --type striped -l 100%FREE -n guestos5 hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + # "Normal" behavior + else + # Extend GuestOS LV to fill VG space + lvextend -i "${count}" --type striped -l +100%FREE /dev/hostlvm/guestos >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to extend logical volume: /dev/hostlvm/guestos" + fi } # Establish run order From fab400980d29a793abc1f30d2929cef4d37a2754 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 12 Jun 2026 22:58:59 +0000 Subject: [PATCH 3/8] Changes to VM runner --- .../config/tool/src/hostos/guestos_config.rs | 16 +++++++--- .../networking/deterministic_ips/src/lib.rs | 12 ++++++- .../guest_vm_runner/src/guest_vm_config.rs | 31 +++++++++++-------- rs/ic_os/os_tools/guest_vm_runner/src/main.rs | 27 ++++++++++------ 4 files changed, 59 insertions(+), 27 deletions(-) diff --git a/rs/ic_os/config/tool/src/hostos/guestos_config.rs b/rs/ic_os/config/tool/src/hostos/guestos_config.rs index b392bf83b9e1..00de3c0787a5 100644 --- a/rs/ic_os/config/tool/src/hostos/guestos_config.rs +++ b/rs/ic_os/config/tool/src/hostos/guestos_config.rs @@ -4,7 +4,7 @@ use config_types::{ GuestVMType, HostOSConfig, Ipv6Config, RecoveryConfig, TrustedExecutionEnvironmentConfig, }; use deterministic_ips::node_type::NodeType; -use deterministic_ips::{MacAddr6Ext, calculate_deterministic_mac}; +use deterministic_ips::{MacAddr6Ext, calculate_deterministic_mac_w_slot}; use std::net::Ipv6Addr; use std::path::Path; use utils::to_cidr; @@ -16,6 +16,7 @@ const DEFAULT_GUESTOS_RECOVERY_FILE_PATH: &str = "/run/config/guestos_recovery_h /// sev_certificate_chain_pem must be provided. pub fn generate_guestos_config( hostos_config: &HostOSConfig, + guest_vm_slot: usize, guest_vm_type: GuestVMType, sev_certificate_chain_pem: Option, ) -> Result { @@ -47,9 +48,14 @@ pub fn generate_guestos_config( } }; - let guestos_ipv6_address = - node_ipv6_address(node_type, hostos_config, deterministic_ipv6_config)?; + let guestos_ipv6_address = node_ipv6_address( + guest_vm_slot, + node_type, + hostos_config, + deterministic_ipv6_config, + )?; let peer_ipv6_address = node_ipv6_address( + 0, upgrade_peer_node_type, hostos_config, deterministic_ipv6_config, @@ -90,13 +96,15 @@ pub fn generate_guestos_config( } fn node_ipv6_address( + slot: usize, node_type: NodeType, hostos_config: &HostOSConfig, deterministic_config: &DeterministicIpv6Config, ) -> Result { - let mac = calculate_deterministic_mac( + let mac = calculate_deterministic_mac_w_slot( &hostos_config.icos_settings.mgmt_mac, hostos_config.icos_settings.deployment_environment, + slot, node_type, ); diff --git a/rs/ic_os/networking/deterministic_ips/src/lib.rs b/rs/ic_os/networking/deterministic_ips/src/lib.rs index 67d4a22529ba..02e3691bd4d7 100644 --- a/rs/ic_os/networking/deterministic_ips/src/lib.rs +++ b/rs/ic_os/networking/deterministic_ips/src/lib.rs @@ -65,7 +65,16 @@ pub fn calculate_deterministic_mac( deployment_environment: DeploymentEnvironment, node_type: NodeType, ) -> MacAddr6 { - let index = node_type.to_index(); + calculate_deterministic_mac_w_slot(mgmt_mac, deployment_environment, 0, node_type) +} + +pub fn calculate_deterministic_mac_w_slot( + mgmt_mac: &MacAddr6, + deployment_environment: DeploymentEnvironment, + slot: usize, + node_type: NodeType, +) -> MacAddr6 { + let index = node_type.to_index() | (slot as u8) << 4; // Mix in the slot index // NOTE: In order to be backwards compatible with existing scripts, this // **MUST** have a newline. @@ -91,6 +100,7 @@ mod test { let mgmt_mac: MacAddr6 = "70:B5:E8:E8:25:DE".parse().unwrap(); let expected_mac: MacAddr6 = "6a:00:f8:87:a4:8a".parse().unwrap(); let mac = calculate_deterministic_mac( + 0, &mgmt_mac, DeploymentEnvironment::Testnet, NodeType::HostOS, diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs index d770eb2cbfc5..ff9af2d5adb2 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs @@ -5,7 +5,7 @@ use askama::Template; use config_tool::hostos::guestos_bootstrap_image::BootstrapOptions; use config_tool::hostos::guestos_config::generate_guestos_config; use config_types::{GuestOSConfig, HostOSConfig}; -use deterministic_ips::calculate_deterministic_mac; +use deterministic_ips::calculate_deterministic_mac_w_slot; use deterministic_ips::node_type::NodeType; use std::path::{Path, PathBuf}; use tracing::info; @@ -53,12 +53,14 @@ pub struct DirectBootConfig { pub fn assemble_config_media( hostos_config: &HostOSConfig, + guest_vm_slot: usize, guest_vm_type: GuestVMType, sev_certificate_chain_pem: Option, media_path: &Path, ) -> Result<()> { let guestos_config = generate_guestos_config( hostos_config, + guest_vm_slot, guest_vm_type.to_config_type(), sev_certificate_chain_pem, ) @@ -105,6 +107,7 @@ pub fn generate_vm_config( direct_boot: Option, disk_device: &Path, serial_log_path: &Path, + guest_vm_slot: usize, guest_vm_type: GuestVMType, available_hugepages_gib: u64, metrics: &GuestVmMetrics, @@ -113,9 +116,10 @@ pub fn generate_vm_config( GuestVMType::Default => NodeType::GuestOS, GuestVMType::Upgrade => NodeType::UpgradeGuestOS, }; - let mac_address = calculate_deterministic_mac( + let mac_address = calculate_deterministic_mac_w_slot( &config.icos_settings.mgmt_mac, config.icos_settings.deployment_environment, + guest_vm_slot, node_type, ); @@ -142,8 +146,8 @@ pub fn generate_vm_config( metrics.set_hugepages_enabled(guest_vm_type, use_hugepages); GuestOSTemplateProps { - domain_name: vm_domain_name(guest_vm_type).to_string(), - domain_uuid: vm_domain_uuid(guest_vm_type).to_string(), + domain_name: vm_domain_name(guest_vm_type, guest_vm_slot), + domain_uuid: vm_domain_uuid(guest_vm_type, guest_vm_slot), disk_device: disk_device.to_path_buf(), cpu_domain, console_log_path: serial_log_path.display().to_string(), @@ -178,24 +182,24 @@ pub(crate) fn vm_resources(_config: &HostOSConfig) -> (String, u32, u32) { ("kvm".to_string(), DEFAULT_VM_MEMORY_GIB, DEFAULT_VM_VCPUS) } -pub fn vm_domain_name(guest_vm_type: GuestVMType) -> &'static str { +pub fn vm_domain_name(guest_vm_type: GuestVMType, slot: usize) -> String { match guest_vm_type { - GuestVMType::Default => DEFAULT_GUEST_VM_DOMAIN_NAME, - GuestVMType::Upgrade => UPGRADE_GUEST_VM_DOMAIN_NAME, + GuestVMType::Default => format!("{DEFAULT_GUEST_VM_DOMAIN_NAME}{slot}"), + GuestVMType::Upgrade => UPGRADE_GUEST_VM_DOMAIN_NAME.to_string(), } } -pub fn vm_domain_uuid(guest_vm_type: GuestVMType) -> &'static str { +pub fn vm_domain_uuid(guest_vm_type: GuestVMType, slot: usize) -> String { match guest_vm_type { - GuestVMType::Default => "fd897da5-8017-41c8-8575-a706dba30766", - GuestVMType::Upgrade => "1ea49839-7f46-4560-a4c7-fce677bbfbbd", + GuestVMType::Default => format!("fd897da5-8017-41c8-8575-a706dba3076{slot:x}"), + GuestVMType::Upgrade => "1ea49839-7f46-4560-a4c7-fce677bbfbbd".to_string(), } } -pub fn serial_log_path(guest_vm_type: GuestVMType) -> &'static Path { +pub fn serial_log_path(guest_vm_type: GuestVMType, slot: usize) -> PathBuf { match guest_vm_type { - GuestVMType::Default => Path::new(DEFAULT_SERIAL_LOG_PATH), - GuestVMType::Upgrade => Path::new(UPGRADE_SERIAL_LOG_PATH), + GuestVMType::Default => PathBuf::from(format!("{DEFAULT_SERIAL_LOG_PATH}{slot}")), + GuestVMType::Upgrade => PathBuf::from(UPGRADE_SERIAL_LOG_PATH.to_string()), } } @@ -316,6 +320,7 @@ mod tests { direct_boot, Path::new("/dev/guest_disk"), Path::new("/var/serial/console.txt"), + 0, guest_vm_type, available_hugepages_gib, &metrics, diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/main.rs b/rs/ic_os/os_tools/guest_vm_runner/src/main.rs index 34f0c1ffc7ac..c2e36ea49fc0 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/main.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/main.rs @@ -93,6 +93,9 @@ impl GuestVMType { enum Command { /// Run the GuestOS virtual machine Run { + #[arg(long = "slot", default_value_t = 0)] + slot: usize, + #[arg(long = "type", default_value = "default", value_enum)] vm_type: GuestVMType, }, @@ -121,11 +124,11 @@ pub async fn main() -> Result<()> { match args.command { Command::ReserveHugepages => reserve_hugepages(), - Command::Run { vm_type } => run(vm_type).await, + Command::Run { slot, vm_type } => run(slot, vm_type).await, } } -async fn run(vm_type: GuestVMType) -> Result<()> { +async fn run(slot: usize, vm_type: GuestVMType) -> Result<()> { let startup_message = match vm_type { GuestVMType::Default => "Launching GuestOS Virtual Machine...", GuestVMType::Upgrade => "Launching Upgrade GuestOS Virtual Machine...", @@ -141,7 +144,7 @@ async fn run(vm_type: GuestVMType) -> Result<()> { let termination_token = CancellationToken::new(); setup_signal_handler(termination_token.clone()).context("Failed to setup signal handler")?; - GuestVmService::create_and_run(vm_type, termination_token).await + GuestVmService::create_and_run(slot, vm_type, termination_token).await } fn setup_signal_handler(termination_token: CancellationToken) -> Result<()> { @@ -414,6 +417,7 @@ pub struct GuestVmService { hostos_config: HostOSConfig, systemd_notifier: Arc, console_ttys: Vec>>, + guest_vm_slot: usize, guest_vm_type: GuestVMType, sev_certificate_provider: HostSevCertificateProvider, disk_device: PathBuf, @@ -432,7 +436,7 @@ impl GuestVmService { } #[cfg(target_os = "linux")] - pub fn new(guest_vm_type: GuestVMType) -> Result { + pub fn new(guest_vm_slot: usize, guest_vm_type: GuestVMType) -> Result { let metrics = GuestVmMetrics::new(PathBuf::from(Self::metrics_path(guest_vm_type))) .context("Failed to create metrics")?; let libvirt_connection = LibvirtConnectionWithReconnect::new(Arc::new(|| { @@ -469,15 +473,17 @@ impl GuestVmService { }) .transpose()?; + let device_string = format!("{GUESTOS_DEVICE}{guest_vm_slot}"); let disk_device = upgrade_mapped_device .as_ref() .map(|x| x.path()) - .unwrap_or(Path::new(GUESTOS_DEVICE)); + .unwrap_or(Path::new(&device_string)); Ok(Self { metrics, libvirt_connection: Arc::new(libvirt_connection), hostos_config, + guest_vm_slot, guest_vm_type, systemd_notifier: Arc::new(systemd_notifier::DefaultSystemdNotifier), console_ttys: vec![ @@ -492,17 +498,18 @@ impl GuestVmService { disk_device: disk_device.to_path_buf(), _upgrade_mapped_device: upgrade_mapped_device, guestos_boot_timeout: GUESTOS_BOOT_TIMEOUT, - vm_serial_log_path: serial_log_path(guest_vm_type).to_path_buf(), + vm_serial_log_path: serial_log_path(guest_vm_type, guest_vm_slot).to_path_buf(), command_runner: Arc::new(RealAsyncCommandRunner), }) } #[cfg(target_os = "linux")] pub async fn create_and_run( + slot: usize, guest_vm_type: GuestVMType, termination_token: CancellationToken, ) -> Result<()> { - let mut guest_vm_service = Self::new(guest_vm_type)?; + let mut guest_vm_service = Self::new(slot, guest_vm_type)?; guest_vm_service.run(termination_token).await } @@ -567,7 +574,7 @@ impl GuestVmService { async fn start_virtual_machine(&mut self) -> Result { VirtualMachine::try_destroy_existing_vm( self.libvirt_connection.as_ref(), - vm_domain_name(self.guest_vm_type), + &vm_domain_name(self.guest_vm_type, self.guest_vm_slot), self.command_runner.as_ref(), ) .await?; @@ -609,6 +616,7 @@ impl GuestVmService { assemble_config_media( &self.hostos_config, + self.guest_vm_slot, self.guest_vm_type, sev_certificate_chain_pem, config_media.path(), @@ -624,6 +632,7 @@ impl GuestVmService { direct_boot.as_ref().map(DirectBoot::to_config), &self.disk_device, &self.vm_serial_log_path, + self.guest_vm_slot, self.guest_vm_type, available_hugepages_gib, &self.metrics, @@ -637,7 +646,7 @@ impl GuestVmService { &vm_config, config_media, direct_boot, - vm_domain_name(self.guest_vm_type), + &vm_domain_name(self.guest_vm_type, self.guest_vm_slot), ) .await?; From 4594fdcdbf8c6bcf29e378acae49008cd3f59659 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 17 Jun 2026 00:27:01 +0000 Subject: [PATCH 4/8] Try to do more resources dynamically --- ic-os/components/multi-hostos.bzl | 13 +++--- ic-os/components/multi-hostos/guestos.target | 3 -- .../components/multi-hostos/guestos1.service | 20 ---------- .../components/multi-hostos/guestos2.service | 20 ---------- .../components/multi-hostos/guestos3.service | 20 ---------- .../components/multi-hostos/guestos5.service | 20 ---------- .../{guestos4.service => guestos@.service} | 6 +-- .../multi-hostos/start-guestos.service | 13 ++++++ .../components/multi-hostos/start-guestos.sh | 17 ++++++++ ic-os/components/setupos/install-hostos.sh | 40 +++++++++++++++---- .../guest_vm_runner/src/guest_vm_config.rs | 30 ++++++++++++-- 11 files changed, 100 insertions(+), 102 deletions(-) delete mode 100644 ic-os/components/multi-hostos/guestos.target delete mode 100644 ic-os/components/multi-hostos/guestos1.service delete mode 100644 ic-os/components/multi-hostos/guestos2.service delete mode 100644 ic-os/components/multi-hostos/guestos3.service delete mode 100644 ic-os/components/multi-hostos/guestos5.service rename ic-os/components/multi-hostos/{guestos4.service => guestos@.service} (64%) create mode 100644 ic-os/components/multi-hostos/start-guestos.service create mode 100644 ic-os/components/multi-hostos/start-guestos.sh diff --git a/ic-os/components/multi-hostos.bzl b/ic-os/components/multi-hostos.bzl index d7dc6a8ecb18..6307da1c9988 100644 --- a/ic-os/components/multi-hostos.bzl +++ b/ic-os/components/multi-hostos.bzl @@ -5,13 +5,14 @@ Override a few components for multi-hostos load("hostos.bzl", hostos_component_files = "component_files") component_files = hostos_component_files | { - Label("multi-hostos/guestos1.service"): "/etc/systemd/system/guestos1.service", - Label("multi-hostos/guestos2.service"): "/etc/systemd/system/guestos2.service", - Label("multi-hostos/guestos3.service"): "/etc/systemd/system/guestos3.service", - Label("multi-hostos/guestos4.service"): "/etc/systemd/system/guestos4.service", - Label("multi-hostos/guestos5.service"): "/etc/systemd/system/guestos5.service", - Label("multi-hostos/guestos.target"): "/etc/systemd/system/guestos.target", + Label("multi-hostos/start-guestos.sh"): "/opt/ic/bin/start-guestos.sh", + Label("multi-hostos/start-guestos.service"): "/etc/systemd/system/start-guestos.service", + Label("multi-hostos/guestos@.service"): "/etc/systemd/system/guestos@.service", Label("multi-hostos/verbose-logging.service"): "/etc/systemd/system/verbose-logging.service", Label("multi-hostos/monitor-guestos.service"): "/etc/systemd/system/monitor-guestos.service", Label("multi-hostos/update-config.service"): "/etc/systemd/system/update-config.service", } +component_files.pop(Label("hostos/guestos/guestos.service")) +component_files.pop(Label("hostos/verbose-logging/verbose-logging.service")) +component_files.pop(Label("monitoring/hostos/monitor-guestos.service")) +component_files.pop(Label("hostos/update-config/update-config.service")) diff --git a/ic-os/components/multi-hostos/guestos.target b/ic-os/components/multi-hostos/guestos.target deleted file mode 100644 index 178701f4fb04..000000000000 --- a/ic-os/components/multi-hostos/guestos.target +++ /dev/null @@ -1,3 +0,0 @@ -[Unit] -Description=All GuestOS VMs -Wants=guestos1.service guestos2.service guestos3.service guestos4.service guestos5.service diff --git a/ic-os/components/multi-hostos/guestos1.service b/ic-os/components/multi-hostos/guestos1.service deleted file mode 100644 index 50bc25cfeab8..000000000000 --- a/ic-os/components/multi-hostos/guestos1.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Manage GuestOS 1 virtual machine -# Not Requires, otherwise guestos will restart when libvirtd restarts -Wants=libvirtd.service -After=libvirtd.service - -RequiresMountsFor=/var - -[Service] -Type=notify -ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' -ExecStartPre=/opt/ic/bin/detect-first-boot.sh -ExecStart=/opt/ic/bin/guest_vm_runner run --slot 0 -ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm -Restart=always -RestartSec=60 -TimeoutStopSec=130s - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos2.service b/ic-os/components/multi-hostos/guestos2.service deleted file mode 100644 index 937398f66c54..000000000000 --- a/ic-os/components/multi-hostos/guestos2.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Manage GuestOS virtual machine -# Not Requires, otherwise guestos will restart when libvirtd restarts -Wants=libvirtd.service -After=libvirtd.service - -RequiresMountsFor=/var - -[Service] -Type=notify -ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' -ExecStartPre=/opt/ic/bin/detect-first-boot.sh -ExecStart=/opt/ic/bin/guest_vm_runner run --slot 1 -ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm -Restart=always -RestartSec=60 -TimeoutStopSec=130s - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos3.service b/ic-os/components/multi-hostos/guestos3.service deleted file mode 100644 index 563de1a2197c..000000000000 --- a/ic-os/components/multi-hostos/guestos3.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Manage GuestOS virtual machine -# Not Requires, otherwise guestos will restart when libvirtd restarts -Wants=libvirtd.service -After=libvirtd.service - -RequiresMountsFor=/var - -[Service] -Type=notify -ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' -ExecStartPre=/opt/ic/bin/detect-first-boot.sh -ExecStart=/opt/ic/bin/guest_vm_runner run --slot 2 -ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm -Restart=always -RestartSec=60 -TimeoutStopSec=130s - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos5.service b/ic-os/components/multi-hostos/guestos5.service deleted file mode 100644 index 01f11b3fa33d..000000000000 --- a/ic-os/components/multi-hostos/guestos5.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Manage GuestOS virtual machine -# Not Requires, otherwise guestos will restart when libvirtd restarts -Wants=libvirtd.service -After=libvirtd.service - -RequiresMountsFor=/var - -[Service] -Type=notify -ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' -ExecStartPre=/opt/ic/bin/detect-first-boot.sh -ExecStart=/opt/ic/bin/guest_vm_runner run --slot 4 -ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm -Restart=always -RestartSec=60 -TimeoutStopSec=130s - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/guestos4.service b/ic-os/components/multi-hostos/guestos@.service similarity index 64% rename from ic-os/components/multi-hostos/guestos4.service rename to ic-os/components/multi-hostos/guestos@.service index 3b22e87f33c3..44c72d627ead 100644 --- a/ic-os/components/multi-hostos/guestos4.service +++ b/ic-os/components/multi-hostos/guestos@.service @@ -1,5 +1,5 @@ [Unit] -Description=Manage GuestOS virtual machine +Description=Manage GuestOS %i virtual machine # Not Requires, otherwise guestos will restart when libvirtd restarts Wants=libvirtd.service After=libvirtd.service @@ -8,9 +8,9 @@ RequiresMountsFor=/var [Service] Type=notify -ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS service...\n\n" | tee /dev/tty1 > /dev/ttyS0' +ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS %i service...\n\n" | tee /dev/tty1 > /dev/ttyS0' ExecStartPre=/opt/ic/bin/detect-first-boot.sh -ExecStart=/opt/ic/bin/guest_vm_runner run --slot 3 +ExecStart=/opt/ic/bin/guest_vm_runner run --slot %i ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm Restart=always RestartSec=60 diff --git a/ic-os/components/multi-hostos/start-guestos.service b/ic-os/components/multi-hostos/start-guestos.service new file mode 100644 index 000000000000..dfec1d7937ab --- /dev/null +++ b/ic-os/components/multi-hostos/start-guestos.service @@ -0,0 +1,13 @@ +[Unit] +Description=Startup GuestOS VMs +RequiresMountsFor=/boot/config +Wants=guestos.target +Before=guestos.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/opt/ic/bin/start-guestos.sh + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/start-guestos.sh b/ic-os/components/multi-hostos/start-guestos.sh new file mode 100644 index 000000000000..7e83623551b7 --- /dev/null +++ b/ic-os/components/multi-hostos/start-guestos.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# This script dynamically starts "guestos@" services to create the right number of VMs per node type + +source /opt/ic/bin/config.sh + +node_reward_type=$(get_config_value '.icos_settings.node_reward_type') + +if [[ $node_reward_type =~ ^type4.1$ ]]; then + systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service guestos@4.service guestos@5.service guestos@6.service guestos@7.service guestos@8.service guestos@9.service guestos@10.service guestos@11.service guestos@12.service guestos@13.service guestos@14.service guestos@15.service +elif [[ $node_reward_type =~ ^type4.2$ ]]; then + systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service guestos@4.service guestos@5.service guestos@6.service guestos@7.service +elif [[ $node_reward_type =~ ^type4.3$ ]]; then + systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service +elif [[ $node_reward_type =~ ^type4.4$ ]]; then + systemctl start guestos@0.service guestos@1.service +fi diff --git a/ic-os/components/setupos/install-hostos.sh b/ic-os/components/setupos/install-hostos.sh index dece8e5a47ac..e9725d42efe2 100755 --- a/ic-os/components/setupos/install-hostos.sh +++ b/ic-os/components/setupos/install-hostos.sh @@ -100,7 +100,6 @@ function resize_partition() { local node_reward_type=$(get_config_value '.icos_settings.node_reward_type') # Configure multiple GuestOS if type4.X - # TODO: Configure differently for each variant if [[ $node_reward_type =~ ^type4(\.[0-9]+)?$ ]]; then # Cleanup the initial GuestOS lvremove -f hostlvm/guestos >/dev/null 2>&1 @@ -108,13 +107,40 @@ function resize_partition() { # And set up new split volumes free=$(vgs --noheadings -o vg_free_count hostlvm | tr -d ' ') - each=$((free / 5)) - for i in 1 2 3 4; do - lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 + + + if [[ $node_reward_type =~ ^type4.1$ ]]; then + each=$((free / 16)) + for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14; do + lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + done + lvcreate -i "${count}" --type striped -l 100%FREE -n guestos15 hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + elif [[ $node_reward_type =~ ^type4.2$ ]]; then + each=$((free / 8)) + for i in 0 1 2 3 4 5 6; do + lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + done + lvcreate -i "${count}" --type striped -l 100%FREE -n guestos7 hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + elif [[ $node_reward_type =~ ^type4.3$ ]]; then + each=$((free / 4)) + for i in 0 1 2; do + lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + done + lvcreate -i "${count}" --type striped -l 100%FREE -n guestos3 hostlvm >/dev/null 2>&1 log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - done - lvcreate -i "${count}" --type striped -l 100%FREE -n guestos5 hostlvm >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + elif [[ $node_reward_type =~ ^type4.4$ ]]; then + each=$((free / 2)) + lvcreate -i "${count}" --type striped -l $each -n guestos0 hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + + lvcreate -i "${count}" --type striped -l 100%FREE -n guestos1 hostlvm >/dev/null 2>&1 + log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" + fi # "Normal" behavior else # Extend GuestOS LV to fill VG space diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs index ff9af2d5adb2..29c96dc59369 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs @@ -134,7 +134,9 @@ pub fn generate_vm_config( "GuestOS VM memory must be at least {UPGRADE_VM_MEMORY_GIB}GiB but is {total_vm_memory}GiB." ); let vm_memory_gib = match guest_vm_type { - GuestVMType::Default => total_vm_memory - UPGRADE_VM_MEMORY_GIB, + // XXX TODO + // GuestVMType::Default => total_vm_memory - UPGRADE_VM_MEMORY_GIB, + GuestVMType::Default => total_vm_memory, GuestVMType::Upgrade => UPGRADE_VM_MEMORY_GIB, }; @@ -178,8 +180,30 @@ pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32) { } #[cfg(not(feature = "dev"))] -pub(crate) fn vm_resources(_config: &HostOSConfig) -> (String, u32, u32) { - ("kvm".to_string(), DEFAULT_VM_MEMORY_GIB, DEFAULT_VM_VCPUS) +pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32) { + match &config.icos_settings.node_reward_type { + Some(val) if val == "type4.1" => ( + "kvm".to_string(), + DEFAULT_VM_MEMORY_GIB / 16, + DEFAULT_VM_VCPUS / 16, + ), + Some(val) if val == "type4.2" => ( + "kvm".to_string(), + DEFAULT_VM_MEMORY_GIB / 8, + DEFAULT_VM_VCPUS / 8, + ), + Some(val) if val == "type4.3" => ( + "kvm".to_string(), + DEFAULT_VM_MEMORY_GIB / 4, + DEFAULT_VM_VCPUS / 4, + ), + Some(val) if val == "type4.4" => ( + "kvm".to_string(), + DEFAULT_VM_MEMORY_GIB / 2, + DEFAULT_VM_VCPUS / 2, + ), + _ => ("kvm".to_string(), DEFAULT_VM_MEMORY_GIB, DEFAULT_VM_VCPUS), + } } pub fn vm_domain_name(guest_vm_type: GuestVMType, slot: usize) -> String { From 71f47b27f0a5fbbb6592b6df111cd651a082a0c4 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Thu, 18 Jun 2026 18:30:28 +0000 Subject: [PATCH 5/8] Misc fixes --- ic-os/components/multi-hostos.bzl | 2 ++ ic-os/hostos/defs.bzl | 3 --- ic-os/setupos/defs.bzl | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ic-os/components/multi-hostos.bzl b/ic-os/components/multi-hostos.bzl index 6307da1c9988..e85c5c530296 100644 --- a/ic-os/components/multi-hostos.bzl +++ b/ic-os/components/multi-hostos.bzl @@ -16,3 +16,5 @@ component_files.pop(Label("hostos/guestos/guestos.service")) component_files.pop(Label("hostos/verbose-logging/verbose-logging.service")) component_files.pop(Label("monitoring/hostos/monitor-guestos.service")) component_files.pop(Label("hostos/update-config/update-config.service")) +component_files.pop(Label("misc/vsock/vsock-agent.service")) +component_files.pop(Label("misc/vsock/10-vhost-vsock.rules")) diff --git a/ic-os/hostos/defs.bzl b/ic-os/hostos/defs.bzl index e1a919c76d09..30ad54274a0e 100644 --- a/ic-os/hostos/defs.bzl +++ b/ic-os/hostos/defs.bzl @@ -87,9 +87,6 @@ def image_deps(mode, _malicious = False): # Allow root console access on dev console_override_label_1 = Label("//ic-os/hostos:console-override-dev") console_override_label_2 = Label("//ic-os/hostos:console-override-dev-alias") - elif "multi" in mode: - console_override_label_1 = Label("//ic-os/hostos:console-override-dev") - console_override_label_2 = Label("//ic-os/hostos:console-override-dev-alias") else: # Allow limited-console access on prod console_override_label_1 = Label("//ic-os/hostos:console-override-prod") diff --git a/ic-os/setupos/defs.bzl b/ic-os/setupos/defs.bzl index e75932fa4d7e..1feb0e51a32c 100644 --- a/ic-os/setupos/defs.bzl +++ b/ic-os/setupos/defs.bzl @@ -102,9 +102,9 @@ def _custom_partitions(mode): elif mode == "multi": guest_image = Label("//ic-os/guestos/envs/prod:disk-img.tar.zst") host_image = Label("//ic-os/hostos/envs/multi:disk-img.tar.zst") - nns_urls = '["https://cloudflare.com/cdn-cgi/trace"]' - include_nns_public_key_override = True - deployment_environment = "testnet" + nns_urls = '["https://icp-api.io", "https://icp0.io", "https://ic0.app"]' + include_nns_public_key_override = False + deployment_environment = "mainnet" else: fail("Unkown mode detected: " + mode) From 10cf6d02d56c751344d64afe34634be4fb21d55e Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 19 Jun 2026 00:45:56 +0000 Subject: [PATCH 6/8] Fix type4.1 --- .../components/multi-hostos/start-guestos.sh | 2 +- ic-os/components/setupos/install-hostos.sh | 6 +-- .../networking/deterministic_ips/src/lib.rs | 40 ++++++++++++++----- .../guest_vm_runner/src/guest_vm_config.rs | 38 +++++++++++++++--- .../templates/guestos_vm_template.xml | 2 +- 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/ic-os/components/multi-hostos/start-guestos.sh b/ic-os/components/multi-hostos/start-guestos.sh index 7e83623551b7..bd9dd256b6d4 100644 --- a/ic-os/components/multi-hostos/start-guestos.sh +++ b/ic-os/components/multi-hostos/start-guestos.sh @@ -7,7 +7,7 @@ source /opt/ic/bin/config.sh node_reward_type=$(get_config_value '.icos_settings.node_reward_type') if [[ $node_reward_type =~ ^type4.1$ ]]; then - systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service guestos@4.service guestos@5.service guestos@6.service guestos@7.service guestos@8.service guestos@9.service guestos@10.service guestos@11.service guestos@12.service guestos@13.service guestos@14.service guestos@15.service + systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service guestos@4.service guestos@5.service guestos@6.service guestos@7.service guestos@8.service guestos@9.service guestos@10.service guestos@11.service guestos@12.service guestos@13.service guestos@14.service guestos@15.service guestos@16.service guestos@17.service guestos@18.service guestos@19.service guestos@20.service guestos@21.service guestos@22.service guestos@23.service guestos@24.service guestos@25.service guestos@26.service guestos@27.service guestos@28.service guestos@29.service guestos@30.service guestos@31.service elif [[ $node_reward_type =~ ^type4.2$ ]]; then systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service guestos@4.service guestos@5.service guestos@6.service guestos@7.service elif [[ $node_reward_type =~ ^type4.3$ ]]; then diff --git a/ic-os/components/setupos/install-hostos.sh b/ic-os/components/setupos/install-hostos.sh index e9725d42efe2..e13c28aa74f6 100755 --- a/ic-os/components/setupos/install-hostos.sh +++ b/ic-os/components/setupos/install-hostos.sh @@ -110,12 +110,12 @@ function resize_partition() { if [[ $node_reward_type =~ ^type4.1$ ]]; then - each=$((free / 16)) - for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14; do + each=$((free / 32)) + for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" done - lvcreate -i "${count}" --type striped -l 100%FREE -n guestos15 hostlvm >/dev/null 2>&1 + lvcreate -i "${count}" --type striped -l 100%FREE -n guestos31 hostlvm >/dev/null 2>&1 log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" elif [[ $node_reward_type =~ ^type4.2$ ]]; then each=$((free / 8)) diff --git a/rs/ic_os/networking/deterministic_ips/src/lib.rs b/rs/ic_os/networking/deterministic_ips/src/lib.rs index 02e3691bd4d7..e8aca007964f 100644 --- a/rs/ic_os/networking/deterministic_ips/src/lib.rs +++ b/rs/ic_os/networking/deterministic_ips/src/lib.rs @@ -74,20 +74,38 @@ pub fn calculate_deterministic_mac_w_slot( slot: usize, node_type: NodeType, ) -> MacAddr6 { - let index = node_type.to_index() | (slot as u8) << 4; // Mix in the slot index + if slot < 16 { + let index = node_type.to_index() | (slot as u8) << 4; // Mix in the slot index + + // NOTE: In order to be backwards compatible with existing scripts, this + // **MUST** have a newline. + let seed = format!( + "{}{}\n", + mgmt_mac.to_string().to_lowercase(), + deployment_environment + ); - // NOTE: In order to be backwards compatible with existing scripts, this - // **MUST** have a newline. - let seed = format!( - "{}{}\n", - mgmt_mac.to_string().to_lowercase(), - deployment_environment - ); + let hash = Sha256::hash(seed.as_bytes()); - let hash = Sha256::hash(seed.as_bytes()); + // 0x6a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing. + [0x6a, index, hash[0], hash[1], hash[2], hash[3]].into() + } else { + let index = node_type.to_index(); + + // NOTE: In order to be backwards compatible with existing scripts, this + // **MUST** have a newline. + let seed = format!( + "{}{}\n", + mgmt_mac.to_string().to_lowercase(), + deployment_environment + ); - // 0x6a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing. - [0x6a, index, hash[0], hash[1], hash[2], hash[3]].into() + let hash = Sha256::hash(seed.as_bytes()); + + // NOTE: We extend to 7a to shrink the hash, so we can fit more slot info into the address. + // 0x7a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing. + [0x7a, index, slot as u8, hash[0], hash[1], hash[2]].into() + } } #[cfg(test)] diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs index 29c96dc59369..a230cfe0a5a6 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs @@ -31,6 +31,9 @@ pub struct GuestOSTemplateProps { pub console_log_path: String, pub vm_memory: u32, pub nr_of_vcpus: u32, + pub nr_of_sockets: u32, + pub nr_of_cores: u32, + pub nr_of_threads: u32, pub mac_address: macaddr::MacAddr6, pub disk_device: PathBuf, pub config_media_path: PathBuf, @@ -123,7 +126,8 @@ pub fn generate_vm_config( node_type, ); - let (cpu_domain, total_vm_memory, nr_of_vcpus) = vm_resources(config); + let (cpu_domain, total_vm_memory, nr_of_vcpus, nr_of_sockets, nr_of_cores, nr_of_threads) = + vm_resources(config); // We need 4GB for the upgrade VM. We subtract that from the total memory. This is not // necessary when SEV is disabled (since no upgrade VM is needed) but mixed subnets that @@ -155,6 +159,9 @@ pub fn generate_vm_config( console_log_path: serial_log_path.display().to_string(), vm_memory: vm_memory_gib, nr_of_vcpus, + nr_of_sockets, + nr_of_cores, + nr_of_threads, mac_address, config_media_path: media_path.to_path_buf(), direct_boot, @@ -180,29 +187,48 @@ pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32) { } #[cfg(not(feature = "dev"))] -pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32) { +pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32, u32, u32, u32) { match &config.icos_settings.node_reward_type { Some(val) if val == "type4.1" => ( "kvm".to_string(), - DEFAULT_VM_MEMORY_GIB / 16, - DEFAULT_VM_VCPUS / 16, + DEFAULT_VM_MEMORY_GIB / 32, + DEFAULT_VM_VCPUS / 32, + 1, + DEFAULT_VM_VCPUS / 32, + 1, ), Some(val) if val == "type4.2" => ( "kvm".to_string(), DEFAULT_VM_MEMORY_GIB / 8, DEFAULT_VM_VCPUS / 8, + 2, + DEFAULT_VM_VCPUS / 8 / 4, + 2, ), Some(val) if val == "type4.3" => ( "kvm".to_string(), DEFAULT_VM_MEMORY_GIB / 4, DEFAULT_VM_VCPUS / 4, + 2, + DEFAULT_VM_VCPUS / 4 / 4, + 2, ), Some(val) if val == "type4.4" => ( "kvm".to_string(), DEFAULT_VM_MEMORY_GIB / 2, DEFAULT_VM_VCPUS / 2, + 2, + DEFAULT_VM_VCPUS / 2 / 4, + 2, + ), + _ => ( + "kvm".to_string(), + DEFAULT_VM_MEMORY_GIB, + DEFAULT_VM_VCPUS, + 2, + DEFAULT_VM_VCPUS / 4, + 2, ), - _ => ("kvm".to_string(), DEFAULT_VM_MEMORY_GIB, DEFAULT_VM_VCPUS), } } @@ -215,7 +241,7 @@ pub fn vm_domain_name(guest_vm_type: GuestVMType, slot: usize) -> String { pub fn vm_domain_uuid(guest_vm_type: GuestVMType, slot: usize) -> String { match guest_vm_type { - GuestVMType::Default => format!("fd897da5-8017-41c8-8575-a706dba3076{slot:x}"), + GuestVMType::Default => format!("fd897da5-8017-41c8-8575-a706dba307{slot:02x}"), GuestVMType::Upgrade => "1ea49839-7f46-4560-a4c7-fce677bbfbbd".to_string(), } } diff --git a/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml b/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml index 4d01b5670545..6bdf6805b950 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml +++ b/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml @@ -20,7 +20,7 @@ {% else %} - + {%- if enable_sev %} From 57dac25cec223b1488eee46a5cbd7f9690293b7d Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 19 Jun 2026 20:58:50 +0000 Subject: [PATCH 7/8] Fixup CI --- ic-os/components/setupos/install-hostos.sh | 1 - ic-os/hostos/envs/multi/BUILD.bazel | 2 +- ic-os/setupos/envs/multi/BUILD.bazel | 2 +- .../config/tool/src/hostos/guestos_config.rs | 11 +++ .../networking/deterministic_ips/src/lib.rs | 1 - .../guest_vm_runner/src/guest_vm_config.rs | 88 ++++++++++++------- rs/ic_os/os_tools/guest_vm_runner/src/main.rs | 18 ++-- 7 files changed, 81 insertions(+), 42 deletions(-) diff --git a/ic-os/components/setupos/install-hostos.sh b/ic-os/components/setupos/install-hostos.sh index e13c28aa74f6..15f85e0bfd59 100755 --- a/ic-os/components/setupos/install-hostos.sh +++ b/ic-os/components/setupos/install-hostos.sh @@ -108,7 +108,6 @@ function resize_partition() { # And set up new split volumes free=$(vgs --noheadings -o vg_free_count hostlvm | tr -d ' ') - if [[ $node_reward_type =~ ^type4.1$ ]]; then each=$((free / 32)) for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do diff --git a/ic-os/hostos/envs/multi/BUILD.bazel b/ic-os/hostos/envs/multi/BUILD.bazel index 9c77673349ec..dd48a8613e08 100644 --- a/ic-os/hostos/envs/multi/BUILD.bazel +++ b/ic-os/hostos/envs/multi/BUILD.bazel @@ -1,7 +1,7 @@ load("//ic-os:defs.bzl", "icos_build") load("//ic-os/hostos:defs.bzl", "image_deps") -icos_images = icos_build( +icos_build( name = "multi", image_deps_func = image_deps, visibility = [ diff --git a/ic-os/setupos/envs/multi/BUILD.bazel b/ic-os/setupos/envs/multi/BUILD.bazel index 5ba2c04301d2..b7cbbb021fc5 100644 --- a/ic-os/setupos/envs/multi/BUILD.bazel +++ b/ic-os/setupos/envs/multi/BUILD.bazel @@ -2,7 +2,7 @@ load("//ic-os:defs.bzl", "icos_build") load("//ic-os/dev-tools/bare_metal_deployment:tools.bzl", "launch_bare_metal") load("//ic-os/setupos:defs.bzl", "image_deps") -icos_images = icos_build( +icos_build( name = "multi", image_deps_func = image_deps, upgrades = False, diff --git a/rs/ic_os/config/tool/src/hostos/guestos_config.rs b/rs/ic_os/config/tool/src/hostos/guestos_config.rs index 00de3c0787a5..0736d10283b7 100644 --- a/rs/ic_os/config/tool/src/hostos/guestos_config.rs +++ b/rs/ic_os/config/tool/src/hostos/guestos_config.rs @@ -15,6 +15,17 @@ const DEFAULT_GUESTOS_RECOVERY_FILE_PATH: &str = "/run/config/guestos_recovery_h /// If hostos_config.icos_settings.enable_trusted_execution_environment is true, /// sev_certificate_chain_pem must be provided. pub fn generate_guestos_config( + hostos_config: &HostOSConfig, + guest_vm_type: GuestVMType, + sev_certificate_chain_pem: Option, +) -> Result { + generate_guestos_config_w_slot(hostos_config, 0, guest_vm_type, sev_certificate_chain_pem) +} + +/// Generate the GuestOS configuration based on the provided HostOS configuration. +/// If hostos_config.icos_settings.enable_trusted_execution_environment is true, +/// sev_certificate_chain_pem must be provided. +pub fn generate_guestos_config_w_slot( hostos_config: &HostOSConfig, guest_vm_slot: usize, guest_vm_type: GuestVMType, diff --git a/rs/ic_os/networking/deterministic_ips/src/lib.rs b/rs/ic_os/networking/deterministic_ips/src/lib.rs index e8aca007964f..f011f5401254 100644 --- a/rs/ic_os/networking/deterministic_ips/src/lib.rs +++ b/rs/ic_os/networking/deterministic_ips/src/lib.rs @@ -118,7 +118,6 @@ mod test { let mgmt_mac: MacAddr6 = "70:B5:E8:E8:25:DE".parse().unwrap(); let expected_mac: MacAddr6 = "6a:00:f8:87:a4:8a".parse().unwrap(); let mac = calculate_deterministic_mac( - 0, &mgmt_mac, DeploymentEnvironment::Testnet, NodeType::HostOS, diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs index a230cfe0a5a6..fdf92c28e8fb 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs @@ -3,10 +3,12 @@ use crate::metrics::GuestVmMetrics; use anyhow::{Context, Result, ensure}; use askama::Template; use config_tool::hostos::guestos_bootstrap_image::BootstrapOptions; -use config_tool::hostos::guestos_config::generate_guestos_config; +use config_tool::hostos::guestos_config::{ + generate_guestos_config, generate_guestos_config_w_slot, +}; use config_types::{GuestOSConfig, HostOSConfig}; -use deterministic_ips::calculate_deterministic_mac_w_slot; use deterministic_ips::node_type::NodeType; +use deterministic_ips::{calculate_deterministic_mac, calculate_deterministic_mac_w_slot}; use std::path::{Path, PathBuf}; use tracing::info; @@ -56,17 +58,25 @@ pub struct DirectBootConfig { pub fn assemble_config_media( hostos_config: &HostOSConfig, - guest_vm_slot: usize, + guest_vm_slot: Option, guest_vm_type: GuestVMType, sev_certificate_chain_pem: Option, media_path: &Path, ) -> Result<()> { - let guestos_config = generate_guestos_config( - hostos_config, - guest_vm_slot, - guest_vm_type.to_config_type(), - sev_certificate_chain_pem, - ) + let guestos_config = if let Some(slot) = guest_vm_slot { + generate_guestos_config_w_slot( + hostos_config, + slot, + guest_vm_type.to_config_type(), + sev_certificate_chain_pem, + ) + } else { + generate_guestos_config( + hostos_config, + guest_vm_type.to_config_type(), + sev_certificate_chain_pem, + ) + } .context("Failed to generate GuestOS config")?; let bootstrap_options = make_bootstrap_options(hostos_config, guestos_config)?; @@ -110,7 +120,7 @@ pub fn generate_vm_config( direct_boot: Option, disk_device: &Path, serial_log_path: &Path, - guest_vm_slot: usize, + guest_vm_slot: Option, guest_vm_type: GuestVMType, available_hugepages_gib: u64, metrics: &GuestVmMetrics, @@ -119,12 +129,20 @@ pub fn generate_vm_config( GuestVMType::Default => NodeType::GuestOS, GuestVMType::Upgrade => NodeType::UpgradeGuestOS, }; - let mac_address = calculate_deterministic_mac_w_slot( - &config.icos_settings.mgmt_mac, - config.icos_settings.deployment_environment, - guest_vm_slot, - node_type, - ); + let mac_address = if let Some(slot) = guest_vm_slot { + calculate_deterministic_mac_w_slot( + &config.icos_settings.mgmt_mac, + config.icos_settings.deployment_environment, + slot, + node_type, + ) + } else { + calculate_deterministic_mac( + &config.icos_settings.mgmt_mac, + config.icos_settings.deployment_environment, + node_type, + ) + }; let (cpu_domain, total_vm_memory, nr_of_vcpus, nr_of_sockets, nr_of_cores, nr_of_threads) = vm_resources(config); @@ -137,11 +155,10 @@ pub fn generate_vm_config( total_vm_memory >= UPGRADE_VM_MEMORY_GIB, "GuestOS VM memory must be at least {UPGRADE_VM_MEMORY_GIB}GiB but is {total_vm_memory}GiB." ); - let vm_memory_gib = match guest_vm_type { - // XXX TODO - // GuestVMType::Default => total_vm_memory - UPGRADE_VM_MEMORY_GIB, - GuestVMType::Default => total_vm_memory, - GuestVMType::Upgrade => UPGRADE_VM_MEMORY_GIB, + let vm_memory_gib = match (guest_vm_type, &config.icos_settings.node_reward_type) { + (GuestVMType::Default, Some(val)) if val.starts_with("type4") => total_vm_memory, + (GuestVMType::Default, _) => total_vm_memory - UPGRADE_VM_MEMORY_GIB, + (GuestVMType::Upgrade, _) => UPGRADE_VM_MEMORY_GIB, }; // Enable hugepages if enough are available for this VM and TEE is disabled (hugepages are not @@ -173,7 +190,7 @@ pub fn generate_vm_config( } #[cfg(feature = "dev")] -pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32) { +pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32, u32, u32, u32) { let cpu_domain = if config.hostos_settings.hostos_dev_settings.vm_cpu == "qemu" { "qemu".to_string() } else { @@ -183,7 +200,14 @@ pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32) { let total_vm_memory = config.hostos_settings.hostos_dev_settings.vm_memory; let vm_nr_of_vcpus = config.hostos_settings.hostos_dev_settings.vm_nr_of_vcpus; - (cpu_domain, total_vm_memory, vm_nr_of_vcpus) + ( + cpu_domain, + total_vm_memory, + vm_nr_of_vcpus, + 2, + vm_nr_of_vcpus / 4, + 2, + ) } #[cfg(not(feature = "dev"))] @@ -232,23 +256,26 @@ pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32, u32, u32 } } -pub fn vm_domain_name(guest_vm_type: GuestVMType, slot: usize) -> String { +pub fn vm_domain_name(guest_vm_type: GuestVMType, slot: Option) -> String { + let slot_suffix = slot.map(|v| v.to_string()).unwrap_or_default(); match guest_vm_type { - GuestVMType::Default => format!("{DEFAULT_GUEST_VM_DOMAIN_NAME}{slot}"), + GuestVMType::Default => format!("{DEFAULT_GUEST_VM_DOMAIN_NAME}{slot_suffix}"), GuestVMType::Upgrade => UPGRADE_GUEST_VM_DOMAIN_NAME.to_string(), } } -pub fn vm_domain_uuid(guest_vm_type: GuestVMType, slot: usize) -> String { +pub fn vm_domain_uuid(guest_vm_type: GuestVMType, slot: Option) -> String { + let slot = slot.unwrap_or(0x66); match guest_vm_type { GuestVMType::Default => format!("fd897da5-8017-41c8-8575-a706dba307{slot:02x}"), GuestVMType::Upgrade => "1ea49839-7f46-4560-a4c7-fce677bbfbbd".to_string(), } } -pub fn serial_log_path(guest_vm_type: GuestVMType, slot: usize) -> PathBuf { +pub fn serial_log_path(guest_vm_type: GuestVMType, slot: Option) -> PathBuf { + let slot_suffix = slot.map(|v| v.to_string()).unwrap_or_default(); match guest_vm_type { - GuestVMType::Default => PathBuf::from(format!("{DEFAULT_SERIAL_LOG_PATH}{slot}")), + GuestVMType::Default => PathBuf::from(format!("{DEFAULT_SERIAL_LOG_PATH}{slot_suffix}")), GuestVMType::Upgrade => PathBuf::from(UPGRADE_SERIAL_LOG_PATH.to_string()), } } @@ -256,6 +283,7 @@ pub fn serial_log_path(guest_vm_type: GuestVMType, slot: usize) -> PathBuf { #[cfg(all(test, not(feature = "skip_default_tests")))] mod tests { use super::*; + use config_tool::hostos::guestos_config::generate_guestos_config; use config_types::{ DeterministicIpv6Config, HostOSConfig, HostOSDevSettings, HostOSSettings, ICOSSettings, Ipv6Config, NetworkSettings, @@ -370,7 +398,7 @@ mod tests { direct_boot, Path::new("/dev/guest_disk"), Path::new("/var/serial/console.txt"), - 0, + None, guest_vm_type, available_hugepages_gib, &metrics, @@ -461,7 +489,7 @@ mod tests { let media_path = temp_dir.path().join("config.img"); let config = create_test_hostos_config(); - let result = assemble_config_media(&config, GuestVMType::Upgrade, None, &media_path); + let result = assemble_config_media(&config, None, GuestVMType::Upgrade, None, &media_path); assert!( result.is_ok(), diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/main.rs b/rs/ic_os/os_tools/guest_vm_runner/src/main.rs index c2e36ea49fc0..e5e02ddfafc4 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/main.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/main.rs @@ -93,8 +93,8 @@ impl GuestVMType { enum Command { /// Run the GuestOS virtual machine Run { - #[arg(long = "slot", default_value_t = 0)] - slot: usize, + #[arg(long = "slot")] + slot: Option, #[arg(long = "type", default_value = "default", value_enum)] vm_type: GuestVMType, @@ -128,7 +128,7 @@ pub async fn main() -> Result<()> { } } -async fn run(slot: usize, vm_type: GuestVMType) -> Result<()> { +async fn run(slot: Option, vm_type: GuestVMType) -> Result<()> { let startup_message = match vm_type { GuestVMType::Default => "Launching GuestOS Virtual Machine...", GuestVMType::Upgrade => "Launching Upgrade GuestOS Virtual Machine...", @@ -417,7 +417,7 @@ pub struct GuestVmService { hostos_config: HostOSConfig, systemd_notifier: Arc, console_ttys: Vec>>, - guest_vm_slot: usize, + guest_vm_slot: Option, guest_vm_type: GuestVMType, sev_certificate_provider: HostSevCertificateProvider, disk_device: PathBuf, @@ -436,7 +436,7 @@ impl GuestVmService { } #[cfg(target_os = "linux")] - pub fn new(guest_vm_slot: usize, guest_vm_type: GuestVMType) -> Result { + pub fn new(guest_vm_slot: Option, guest_vm_type: GuestVMType) -> Result { let metrics = GuestVmMetrics::new(PathBuf::from(Self::metrics_path(guest_vm_type))) .context("Failed to create metrics")?; let libvirt_connection = LibvirtConnectionWithReconnect::new(Arc::new(|| { @@ -473,7 +473,8 @@ impl GuestVmService { }) .transpose()?; - let device_string = format!("{GUESTOS_DEVICE}{guest_vm_slot}"); + let slot_suffix = guest_vm_slot.map(|v| v.to_string()).unwrap_or_default(); + let device_string = format!("{GUESTOS_DEVICE}{slot_suffix}"); let disk_device = upgrade_mapped_device .as_ref() .map(|x| x.path()) @@ -505,7 +506,7 @@ impl GuestVmService { #[cfg(target_os = "linux")] pub async fn create_and_run( - slot: usize, + slot: Option, guest_vm_type: GuestVMType, termination_token: CancellationToken, ) -> Result<()> { @@ -1062,6 +1063,7 @@ mod tests { .unwrap(), ), guest_vm_type, + guest_vm_slot: None, sev_certificate_provider, disk_device: GUESTOS_DEVICE.into(), _upgrade_mapped_device: None, @@ -1081,7 +1083,7 @@ mod tests { systemd_notifier, termination_token, libvirt_connection: self.libvirt_connection.clone(), - vm_domain_name: vm_domain_name(guest_vm_type).to_string(), + vm_domain_name: vm_domain_name(guest_vm_type, None).to_string(), _sev_certificate_cache_dir: sev_certificate_cache_dir, } } From f25bdbe11dd44809dcce12e46bf5c4e8579ae034 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Mon, 22 Jun 2026 18:57:31 +0000 Subject: [PATCH 8/8] Cleanup code --- ic-os/components/BUILD.bazel | 3 +- ic-os/components/hostos.bzl | 5 + .../components/hostos/guestos/guestos.service | 3 - .../update-config/update-config.service | 2 +- .../verbose-logging/verbose-logging.service | 4 +- .../components/misc/vsock/vsock-agent.service | 3 +- .../hostos/export-guestos-serial-logs.sh | 30 ++++- .../monitoring/hostos/monitor-guestos.service | 3 +- ic-os/components/multi-hostos.bzl | 20 ---- ic-os/components/multi-hostos/guestos.target | 2 + .../components/multi-hostos/guestos@.service | 5 +- .../multi-hostos/monitor-guestos.service | 15 --- ic-os/components/multi-hostos/skip-vsock.sh | 12 ++ .../components/multi-hostos/start-guestos.sh | 25 ++-- .../multi-hostos/update-config.service | 14 --- .../multi-hostos/verbose-logging.service | 11 -- ic-os/components/setupos/install-hostos.sh | 45 +++---- ic-os/hostos/defs.bzl | 7 +- ic-os/hostos/envs/multi/BUILD.bazel | 12 -- ic-os/setupos/defs.bzl | 8 +- ic-os/setupos/envs/multi/BUILD.bazel | 15 --- .../guest_vm_runner/src/guest_vm_config.rs | 110 ++++++++---------- .../templates/guestos_vm_template.xml | 2 +- 23 files changed, 140 insertions(+), 216 deletions(-) delete mode 100644 ic-os/components/multi-hostos.bzl create mode 100644 ic-os/components/multi-hostos/guestos.target delete mode 100644 ic-os/components/multi-hostos/monitor-guestos.service create mode 100644 ic-os/components/multi-hostos/skip-vsock.sh delete mode 100644 ic-os/components/multi-hostos/update-config.service delete mode 100644 ic-os/components/multi-hostos/verbose-logging.service delete mode 100644 ic-os/hostos/envs/multi/BUILD.bazel delete mode 100644 ic-os/setupos/envs/multi/BUILD.bazel diff --git a/ic-os/components/BUILD.bazel b/ic-os/components/BUILD.bazel index aea74a342ed6..ef7c64c8f6b6 100644 --- a/ic-os/components/BUILD.bazel +++ b/ic-os/components/BUILD.bazel @@ -2,7 +2,6 @@ load("//bazel:defs.bzl", "write_stable_status_file_var") load("//ic-os/components/conformance_tests:defs.bzl", "check_unused_components_test") load("guestos.bzl", guestos_component_files = "component_files") load("hostos.bzl", hostos_component_files = "component_files") -load("multi-hostos.bzl", multi_hostos_component_files = "component_files") load("setupos.bzl", setupos_component_files = "component_files") package(default_visibility = ["//ic-os:__subpackages__"]) @@ -33,7 +32,7 @@ write_stable_status_file_var( varname = "STABLE_COMMIT_TIMESTAMP", ) -used_components = guestos_component_files("dev") | guestos_component_files("prod") | hostos_component_files | multi_hostos_component_files | setupos_component_files +used_components = guestos_component_files("dev") | guestos_component_files("prod") | hostos_component_files | setupos_component_files REPO_COMPONENTS = glob( ["**/*"], diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 40494c026767..6e68e8c3835d 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -11,6 +11,10 @@ component_files = { # hostos components Label("hostos/guestos/guestos.service"): "/etc/systemd/system/guestos.service", Label("hostos/guestos/upgrade-guestos.service"): "/etc/systemd/system/upgrade-guestos.service", + Label("multi-hostos/start-guestos.sh"): "/opt/ic/bin/start-guestos.sh", + Label("multi-hostos/start-guestos.service"): "/etc/systemd/system/start-guestos.service", + Label("multi-hostos/guestos@.service"): "/etc/systemd/system/guestos@.service", + Label("multi-hostos/guestos.target"): "/etc/systemd/system/guestos.target", Label("hostos/libvirt/setup-libvirt.sh"): "/opt/ic/bin/setup-libvirt.sh", Label("hostos/libvirt/setup-libvirt.service"): "/etc/systemd/system/setup-libvirt.service", Label("hostos/misc/setup-var.sh"): "/opt/ic/bin/setup-var.sh", @@ -42,6 +46,7 @@ component_files = { Label("misc/output-wrapper.sh"): "/opt/ic/bin/output-wrapper.sh", Label("misc/vsock/vsock-agent.service"): "/etc/systemd/system/vsock-agent.service", Label("misc/vsock/10-vhost-vsock.rules"): "/etc/udev/rules.d/10-vhost-vsock.rules", + Label("multi-hostos/skip-vsock.sh"): "/opt/ic/bin/skip-vsock.sh", Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf", Label("misc/chrony/chrony-var.service"): "/etc/systemd/system/chrony-var.service", Label("hostos/misc/sudoers"): "/etc/sudoers", diff --git a/ic-os/components/hostos/guestos/guestos.service b/ic-os/components/hostos/guestos/guestos.service index fc9c73f5a7a2..ec2949da7cf2 100644 --- a/ic-os/components/hostos/guestos/guestos.service +++ b/ic-os/components/hostos/guestos/guestos.service @@ -15,6 +15,3 @@ ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm Restart=always RestartSec=60 TimeoutStopSec=130s - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/hostos/update-config/update-config.service b/ic-os/components/hostos/update-config/update-config.service index b17f43638763..c322c4e96ca7 100644 --- a/ic-os/components/hostos/update-config/update-config.service +++ b/ic-os/components/hostos/update-config/update-config.service @@ -2,7 +2,7 @@ Description=Update HostOS Configuration RequiresMountsFor=/boot/config Before=node_exporter.service -Before=guestos.service +Before=guestos.target Before=log-config.service [Service] diff --git a/ic-os/components/hostos/verbose-logging/verbose-logging.service b/ic-os/components/hostos/verbose-logging/verbose-logging.service index 09bee517f73a..fdcc193474a9 100644 --- a/ic-os/components/hostos/verbose-logging/verbose-logging.service +++ b/ic-os/components/hostos/verbose-logging/verbose-logging.service @@ -1,7 +1,7 @@ [Unit] Description=If verbose flag enabled, pipe GuestOS console to the Host terminal -Wants=guestos.service -After=guestos.service +Wants=guestos.target +After=guestos.target [Service] ExecStart=/opt/ic/bin/verbose-logging.sh diff --git a/ic-os/components/misc/vsock/vsock-agent.service b/ic-os/components/misc/vsock/vsock-agent.service index 6d1bd8c9c4a5..6c98068d8f06 100644 --- a/ic-os/components/misc/vsock/vsock-agent.service +++ b/ic-os/components/misc/vsock/vsock-agent.service @@ -2,8 +2,9 @@ Description=VSOCK agent daemon [Service] +ExecCondition=/opt/ic/bin/skip-vsock.sh ExecStart=/opt/ic/bin/vsock_host -Restart=always +Restart=on-failure [Install] WantedBy=multi-user.target diff --git a/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh b/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh index b64a79578017..8a0b0c740fcd 100644 --- a/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh +++ b/ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh @@ -1,10 +1,34 @@ #!/bin/bash +set -euo pipefail # Export GuestOS serial logs to journald +# Strip ANSI color/escape codes from serial console output before forwarding to journald -set -e +source /opt/ic/bin/config.sh -# Strip ANSI color/escape codes from serial console output before forwarding to journald -tail -F /var/log/libvirt/qemu/guestos-serial.log | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t guestos-serial -p info & +node_reward_type=$(get_config_value '.icos_settings.node_reward_type') + +case "${node_reward_type}" in + type4.1) COUNT=32 ;; + type4.2) COUNT=8 ;; + type4.3) COUNT=4 ;; + type4.4) COUNT=2 ;; + *) COUNT=1 ;; +esac + +# Forward all the GuestOS logs +for i in $(seq 0 "$((COUNT - 1))"); do + # A single GuestOS keeps the guestos-serial.log name + if [ "$COUNT" -eq 1 ]; then + s="" + else + s=$i + fi + + tail -F "/var/log/libvirt/qemu/guestos-serial$s.log" | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t "guestos-serial$s" -p info & +done + +# And the upgrade VM tail -F /var/log/libvirt/qemu/upgrade-guestos-serial.log | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t upgrade-guestos-serial -p info & + wait diff --git a/ic-os/components/monitoring/hostos/monitor-guestos.service b/ic-os/components/monitoring/hostos/monitor-guestos.service index 358b7201f0c6..bc52951fc203 100644 --- a/ic-os/components/monitoring/hostos/monitor-guestos.service +++ b/ic-os/components/monitoring/hostos/monitor-guestos.service @@ -1,6 +1,7 @@ [Unit] Description=Monitor GuestOS virtual machine -After=guestos.service +Wants=guestos.target +After=guestos.target [Service] Type=oneshot diff --git a/ic-os/components/multi-hostos.bzl b/ic-os/components/multi-hostos.bzl deleted file mode 100644 index e85c5c530296..000000000000 --- a/ic-os/components/multi-hostos.bzl +++ /dev/null @@ -1,20 +0,0 @@ -""" -Override a few components for multi-hostos -""" - -load("hostos.bzl", hostos_component_files = "component_files") - -component_files = hostos_component_files | { - Label("multi-hostos/start-guestos.sh"): "/opt/ic/bin/start-guestos.sh", - Label("multi-hostos/start-guestos.service"): "/etc/systemd/system/start-guestos.service", - Label("multi-hostos/guestos@.service"): "/etc/systemd/system/guestos@.service", - Label("multi-hostos/verbose-logging.service"): "/etc/systemd/system/verbose-logging.service", - Label("multi-hostos/monitor-guestos.service"): "/etc/systemd/system/monitor-guestos.service", - Label("multi-hostos/update-config.service"): "/etc/systemd/system/update-config.service", -} -component_files.pop(Label("hostos/guestos/guestos.service")) -component_files.pop(Label("hostos/verbose-logging/verbose-logging.service")) -component_files.pop(Label("monitoring/hostos/monitor-guestos.service")) -component_files.pop(Label("hostos/update-config/update-config.service")) -component_files.pop(Label("misc/vsock/vsock-agent.service")) -component_files.pop(Label("misc/vsock/10-vhost-vsock.rules")) diff --git a/ic-os/components/multi-hostos/guestos.target b/ic-os/components/multi-hostos/guestos.target new file mode 100644 index 000000000000..0bef6956c108 --- /dev/null +++ b/ic-os/components/multi-hostos/guestos.target @@ -0,0 +1,2 @@ +[Unit] +Description=All GuestOS VMs diff --git a/ic-os/components/multi-hostos/guestos@.service b/ic-os/components/multi-hostos/guestos@.service index 44c72d627ead..752bf816d3b2 100644 --- a/ic-os/components/multi-hostos/guestos@.service +++ b/ic-os/components/multi-hostos/guestos@.service @@ -3,8 +3,8 @@ Description=Manage GuestOS %i virtual machine # Not Requires, otherwise guestos will restart when libvirtd restarts Wants=libvirtd.service After=libvirtd.service - RequiresMountsFor=/var +PartOf=guestos.target [Service] Type=notify @@ -15,6 +15,3 @@ ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm Restart=always RestartSec=60 TimeoutStopSec=130s - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/monitor-guestos.service b/ic-os/components/multi-hostos/monitor-guestos.service deleted file mode 100644 index bc52951fc203..000000000000 --- a/ic-os/components/multi-hostos/monitor-guestos.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Monitor GuestOS virtual machine -Wants=guestos.target -After=guestos.target - -[Service] -Type=oneshot -ExecStart=/opt/ic/bin/monitor-guestos.sh - -# Disable systemd start and stop logs -LogLevelMax=1 -SyslogLevel=2 - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/skip-vsock.sh b/ic-os/components/multi-hostos/skip-vsock.sh new file mode 100644 index 000000000000..307c357305de --- /dev/null +++ b/ic-os/components/multi-hostos/skip-vsock.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +source /opt/ic/bin/config.sh + +node_reward_type="$(get_config_value '.icos_settings.node_reward_type')" + +if [[ "$node_reward_type" =~ ^type4(\.[0-9]+)?$ ]]; then + echo "node_reward_type=${node_reward_type}; skipping vsock-agent" + exit 1 +fi + +exit 0 diff --git a/ic-os/components/multi-hostos/start-guestos.sh b/ic-os/components/multi-hostos/start-guestos.sh index bd9dd256b6d4..f72a4ba8c210 100644 --- a/ic-os/components/multi-hostos/start-guestos.sh +++ b/ic-os/components/multi-hostos/start-guestos.sh @@ -1,17 +1,24 @@ #!/bin/bash +set -euo pipefail -# This script dynamically starts "guestos@" services to create the right number of VMs per node type +# This script dynamically starts guestos services to create the right number of VMs per node type source /opt/ic/bin/config.sh node_reward_type=$(get_config_value '.icos_settings.node_reward_type') -if [[ $node_reward_type =~ ^type4.1$ ]]; then - systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service guestos@4.service guestos@5.service guestos@6.service guestos@7.service guestos@8.service guestos@9.service guestos@10.service guestos@11.service guestos@12.service guestos@13.service guestos@14.service guestos@15.service guestos@16.service guestos@17.service guestos@18.service guestos@19.service guestos@20.service guestos@21.service guestos@22.service guestos@23.service guestos@24.service guestos@25.service guestos@26.service guestos@27.service guestos@28.service guestos@29.service guestos@30.service guestos@31.service -elif [[ $node_reward_type =~ ^type4.2$ ]]; then - systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service guestos@4.service guestos@5.service guestos@6.service guestos@7.service -elif [[ $node_reward_type =~ ^type4.3$ ]]; then - systemctl start guestos@0.service guestos@1.service guestos@2.service guestos@3.service -elif [[ $node_reward_type =~ ^type4.4$ ]]; then - systemctl start guestos@0.service guestos@1.service +case "${node_reward_type}" in + type4.1) COUNT=32 ;; + type4.2) COUNT=8 ;; + type4.3) COUNT=4 ;; + type4.4) COUNT=2 ;; + *) COUNT=1 ;; +esac + +# If not a type4 node, fall to the default GuestOS flow +if ((COUNT == 1)); then + systemctl start guestos.service || true + exit 0 fi + +eval systemctl start guestos@{0..$((COUNT - 1))}.service || true diff --git a/ic-os/components/multi-hostos/update-config.service b/ic-os/components/multi-hostos/update-config.service deleted file mode 100644 index c322c4e96ca7..000000000000 --- a/ic-os/components/multi-hostos/update-config.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Update HostOS Configuration -RequiresMountsFor=/boot/config -Before=node_exporter.service -Before=guestos.target -Before=log-config.service - -[Service] -Type=oneshot -ExecStart=/opt/ic/bin/config_tool update-config -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/multi-hostos/verbose-logging.service b/ic-os/components/multi-hostos/verbose-logging.service deleted file mode 100644 index fdcc193474a9..000000000000 --- a/ic-os/components/multi-hostos/verbose-logging.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=If verbose flag enabled, pipe GuestOS console to the Host terminal -Wants=guestos.target -After=guestos.target - -[Service] -ExecStart=/opt/ic/bin/verbose-logging.sh -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/setupos/install-hostos.sh b/ic-os/components/setupos/install-hostos.sh index 15f85e0bfd59..c0ee5dd6530d 100755 --- a/ic-os/components/setupos/install-hostos.sh +++ b/ic-os/components/setupos/install-hostos.sh @@ -108,38 +108,27 @@ function resize_partition() { # And set up new split volumes free=$(vgs --noheadings -o vg_free_count hostlvm | tr -d ' ') - if [[ $node_reward_type =~ ^type4.1$ ]]; then - each=$((free / 32)) - for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do - lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 + create_guestos_lvs() { + local total="$1" + local each=$((free / total)) + local last=$((total - 1)) + local i + + for ((i = 0; i < last; i++)); do + lvcreate -i "${count}" --type striped -l "${each}" -n "guestos${i}" hostlvm >/dev/null 2>&1 log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" done - lvcreate -i "${count}" --type striped -l 100%FREE -n guestos31 hostlvm >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - elif [[ $node_reward_type =~ ^type4.2$ ]]; then - each=$((free / 8)) - for i in 0 1 2 3 4 5 6; do - lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - done - lvcreate -i "${count}" --type striped -l 100%FREE -n guestos7 hostlvm >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - elif [[ $node_reward_type =~ ^type4.3$ ]]; then - each=$((free / 4)) - for i in 0 1 2; do - lvcreate -i "${count}" --type striped -l $each -n guestos$i hostlvm >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - done - lvcreate -i "${count}" --type striped -l 100%FREE -n guestos3 hostlvm >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - elif [[ $node_reward_type =~ ^type4.4$ ]]; then - each=$((free / 2)) - lvcreate -i "${count}" --type striped -l $each -n guestos0 hostlvm >/dev/null 2>&1 - log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - lvcreate -i "${count}" --type striped -l 100%FREE -n guestos1 hostlvm >/dev/null 2>&1 + lvcreate -i "${count}" --type striped -l 100%FREE -n "guestos${last}" hostlvm >/dev/null 2>&1 log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS" - fi + } + + case "${node_reward_type}" in + type4.1) create_guestos_lvs 32 ;; + type4.2) create_guestos_lvs 8 ;; + type4.3) create_guestos_lvs 4 ;; + type4.4) create_guestos_lvs 2 ;; + esac # "Normal" behavior else # Extend GuestOS LV to fill VG space diff --git a/ic-os/hostos/defs.bzl b/ic-os/hostos/defs.bzl index 30ad54274a0e..5de2eedd3570 100644 --- a/ic-os/hostos/defs.bzl +++ b/ic-os/hostos/defs.bzl @@ -2,8 +2,7 @@ Hold manifest common to all HostOS variants. """ -load("//ic-os/components:hostos.bzl", default_component_files = "component_files") -load("//ic-os/components:multi-hostos.bzl", multi_component_files = "component_files") +load("//ic-os/components:hostos.bzl", "component_files") load("//toolchains/sysimage:toolchain.bzl", "lvm_image") # Declare the dependencies that we will have for the built filesystem images. @@ -16,7 +15,7 @@ def image_deps(mode, _malicious = False): Define all HostOS inputs. Args: - mode: Variant to be built, dev or prod (or multi). + mode: Variant to be built, dev or prod. _malicious: Unused, but currently needed to fit generic build structure. Returns: A dict containing inputs to build this image. @@ -43,7 +42,7 @@ def image_deps(mode, _malicious = False): # Set various configuration values "container_context_files": Label("//ic-os/hostos/context:context-files"), - "component_files": dict(default_component_files if mode != "multi" else multi_component_files), # Make a copy because we might update it later + "component_files": dict(component_files), # Make a copy because we might update it later "partition_table": Label("//ic-os/hostos:partitions.csv"), "volume_table": Label("//ic-os/hostos:volumes.csv"), "rootfs_size": "3G", diff --git a/ic-os/hostos/envs/multi/BUILD.bazel b/ic-os/hostos/envs/multi/BUILD.bazel deleted file mode 100644 index dd48a8613e08..000000000000 --- a/ic-os/hostos/envs/multi/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("//ic-os:defs.bzl", "icos_build") -load("//ic-os/hostos:defs.bzl", "image_deps") - -icos_build( - name = "multi", - image_deps_func = image_deps, - visibility = [ - "//rs:ic-os-pkg", - "//rs:system-tests-pkg", - ], - vuln_scan = False, -) diff --git a/ic-os/setupos/defs.bzl b/ic-os/setupos/defs.bzl index 1feb0e51a32c..3ca1dcf86a37 100644 --- a/ic-os/setupos/defs.bzl +++ b/ic-os/setupos/defs.bzl @@ -17,7 +17,7 @@ def image_deps(mode, _malicious = False): Define all SetupOS inputs. Args: - mode: Variant to be built, dev or prod (or multi). + mode: Variant to be built, dev or prod. _malicious: Unused, but currently needed to fit generic build structure. Returns: A dict containing inputs to build this image. @@ -99,12 +99,6 @@ def _custom_partitions(mode): nns_urls = '["https://icp-api.io", "https://icp0.io", "https://ic0.app"]' include_nns_public_key_override = False deployment_environment = "mainnet" - elif mode == "multi": - guest_image = Label("//ic-os/guestos/envs/prod:disk-img.tar.zst") - host_image = Label("//ic-os/hostos/envs/multi:disk-img.tar.zst") - nns_urls = '["https://icp-api.io", "https://icp0.io", "https://ic0.app"]' - include_nns_public_key_override = False - deployment_environment = "mainnet" else: fail("Unkown mode detected: " + mode) diff --git a/ic-os/setupos/envs/multi/BUILD.bazel b/ic-os/setupos/envs/multi/BUILD.bazel deleted file mode 100644 index b7cbbb021fc5..000000000000 --- a/ic-os/setupos/envs/multi/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -load("//ic-os:defs.bzl", "icos_build") -load("//ic-os/dev-tools/bare_metal_deployment:tools.bzl", "launch_bare_metal") -load("//ic-os/setupos:defs.bzl", "image_deps") - -icos_build( - name = "multi", - image_deps_func = image_deps, - upgrades = False, - vuln_scan = False, -) - -launch_bare_metal( - name = "launch_bare_metal", - image_zst_file = ":disk-img.tar.zst", -) diff --git a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs index fdf92c28e8fb..a9cd83e6c845 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs +++ b/rs/ic_os/os_tools/guest_vm_runner/src/guest_vm_config.rs @@ -33,9 +33,7 @@ pub struct GuestOSTemplateProps { pub console_log_path: String, pub vm_memory: u32, pub nr_of_vcpus: u32, - pub nr_of_sockets: u32, - pub nr_of_cores: u32, - pub nr_of_threads: u32, + pub topology: Topology, pub mac_address: macaddr::MacAddr6, pub disk_device: PathBuf, pub config_media_path: PathBuf, @@ -56,6 +54,12 @@ pub struct DirectBootConfig { pub kernel_cmdline: String, } +pub struct Topology { + pub nr_of_sockets: u32, + pub nr_of_cores: u32, + pub nr_of_threads: u32, +} + pub fn assemble_config_media( hostos_config: &HostOSConfig, guest_vm_slot: Option, @@ -144,20 +148,21 @@ pub fn generate_vm_config( ) }; - let (cpu_domain, total_vm_memory, nr_of_vcpus, nr_of_sockets, nr_of_cores, nr_of_threads) = - vm_resources(config); + let (cpu_domain, total_vm_memory, nr_of_vcpus) = vm_resources(config); + let (vm_memory, nr_of_vcpus, topology) = + split_resources_for_type_4(config, total_vm_memory, nr_of_vcpus); // We need 4GB for the upgrade VM. We subtract that from the total memory. This is not // necessary when SEV is disabled (since no upgrade VM is needed) but mixed subnets that // contain nodes with and without SEV should have the same memory settings for consistency // across nodes. ensure!( - total_vm_memory >= UPGRADE_VM_MEMORY_GIB, + vm_memory >= UPGRADE_VM_MEMORY_GIB, "GuestOS VM memory must be at least {UPGRADE_VM_MEMORY_GIB}GiB but is {total_vm_memory}GiB." ); let vm_memory_gib = match (guest_vm_type, &config.icos_settings.node_reward_type) { - (GuestVMType::Default, Some(val)) if val.starts_with("type4") => total_vm_memory, - (GuestVMType::Default, _) => total_vm_memory - UPGRADE_VM_MEMORY_GIB, + (GuestVMType::Default, Some(val)) if val.starts_with("type4") => vm_memory, + (GuestVMType::Default, _) => vm_memory - UPGRADE_VM_MEMORY_GIB, (GuestVMType::Upgrade, _) => UPGRADE_VM_MEMORY_GIB, }; @@ -176,9 +181,7 @@ pub fn generate_vm_config( console_log_path: serial_log_path.display().to_string(), vm_memory: vm_memory_gib, nr_of_vcpus, - nr_of_sockets, - nr_of_cores, - nr_of_threads, + topology, mac_address, config_media_path: media_path.to_path_buf(), direct_boot, @@ -190,7 +193,7 @@ pub fn generate_vm_config( } #[cfg(feature = "dev")] -pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32, u32, u32, u32) { +pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32) { let cpu_domain = if config.hostos_settings.hostos_dev_settings.vm_cpu == "qemu" { "qemu".to_string() } else { @@ -200,60 +203,41 @@ pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32, u32, u32 let total_vm_memory = config.hostos_settings.hostos_dev_settings.vm_memory; let vm_nr_of_vcpus = config.hostos_settings.hostos_dev_settings.vm_nr_of_vcpus; - ( - cpu_domain, - total_vm_memory, - vm_nr_of_vcpus, - 2, - vm_nr_of_vcpus / 4, - 2, - ) + (cpu_domain, total_vm_memory, vm_nr_of_vcpus) } #[cfg(not(feature = "dev"))] -pub(crate) fn vm_resources(config: &HostOSConfig) -> (String, u32, u32, u32, u32, u32) { - match &config.icos_settings.node_reward_type { - Some(val) if val == "type4.1" => ( - "kvm".to_string(), - DEFAULT_VM_MEMORY_GIB / 32, - DEFAULT_VM_VCPUS / 32, - 1, - DEFAULT_VM_VCPUS / 32, - 1, - ), - Some(val) if val == "type4.2" => ( - "kvm".to_string(), - DEFAULT_VM_MEMORY_GIB / 8, - DEFAULT_VM_VCPUS / 8, - 2, - DEFAULT_VM_VCPUS / 8 / 4, - 2, - ), - Some(val) if val == "type4.3" => ( - "kvm".to_string(), - DEFAULT_VM_MEMORY_GIB / 4, - DEFAULT_VM_VCPUS / 4, - 2, - DEFAULT_VM_VCPUS / 4 / 4, - 2, - ), - Some(val) if val == "type4.4" => ( - "kvm".to_string(), - DEFAULT_VM_MEMORY_GIB / 2, - DEFAULT_VM_VCPUS / 2, - 2, - DEFAULT_VM_VCPUS / 2 / 4, - 2, - ), - _ => ( - "kvm".to_string(), - DEFAULT_VM_MEMORY_GIB, - DEFAULT_VM_VCPUS, - 2, - DEFAULT_VM_VCPUS / 4, - 2, - ), - } +pub(crate) fn vm_resources(_config: &HostOSConfig) -> (String, u32, u32) { + ("kvm".to_string(), DEFAULT_VM_MEMORY_GIB, DEFAULT_VM_VCPUS) +} + +fn split_resources_for_type_4( + config: &HostOSConfig, + memory: u32, + vcpus: u32, +) -> (u32, u32, Topology) { + let (memory, vcpus) = match &config.icos_settings.node_reward_type { + Some(val) if val == "type4.1" => (memory / 32, vcpus / 32), + Some(val) if val == "type4.2" => (memory / 8, vcpus / 8), + Some(val) if val == "type4.3" => (memory / 4, vcpus / 4), + Some(val) if val == "type4.4" => (memory / 2, vcpus / 2), + _ => (memory, vcpus), + }; + + let topology = match &config.icos_settings.node_reward_type { + Some(val) if val == "type4.1" => Topology { + nr_of_sockets: 1, + nr_of_cores: vcpus, + nr_of_threads: 1, + }, + _ => Topology { + nr_of_sockets: 2, + nr_of_cores: vcpus / 4, + nr_of_threads: 2, + }, + }; + + (memory, vcpus, topology) } pub fn vm_domain_name(guest_vm_type: GuestVMType, slot: Option) -> String { diff --git a/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml b/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml index 6bdf6805b950..cea127e09348 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml +++ b/rs/ic_os/os_tools/guest_vm_runner/templates/guestos_vm_template.xml @@ -20,7 +20,7 @@ {% else %} - + {%- if enable_sev %}