Eero/multi hostos ci

ic-os/components/hostos.bzl

@@ -11,6 +11,10 @@ component_files = {
     # hostos components
     Label("hostos/guestos/guestos.service"): "/etc/systemd/system/guestos.service",
     Label("hostos/guestos/upgrade-guestos.service"): "/etc/systemd/system/upgrade-guestos.service",
+    Label("multi-hostos/start-guestos.sh"): "/opt/ic/bin/start-guestos.sh",
+    Label("multi-hostos/start-guestos.service"): "/etc/systemd/system/start-guestos.service",
+    Label("multi-hostos/guestos@.service"): "/etc/systemd/system/guestos@.service",
+    Label("multi-hostos/guestos.target"): "/etc/systemd/system/guestos.target",
     Label("hostos/libvirt/setup-libvirt.sh"): "/opt/ic/bin/setup-libvirt.sh",
     Label("hostos/libvirt/setup-libvirt.service"): "/etc/systemd/system/setup-libvirt.service",
     Label("hostos/misc/setup-var.sh"): "/opt/ic/bin/setup-var.sh",
@@ -42,6 +46,7 @@ component_files = {
     Label("misc/output-wrapper.sh"): "/opt/ic/bin/output-wrapper.sh",
     Label("misc/vsock/vsock-agent.service"): "/etc/systemd/system/vsock-agent.service",
     Label("misc/vsock/10-vhost-vsock.rules"): "/etc/udev/rules.d/10-vhost-vsock.rules",
+    Label("multi-hostos/skip-vsock.sh"): "/opt/ic/bin/skip-vsock.sh",
     Label("misc/chrony/chrony.conf"): "/etc/chrony/chrony.conf",
     Label("misc/chrony/chrony-var.service"): "/etc/systemd/system/chrony-var.service",
     Label("hostos/misc/sudoers"): "/etc/sudoers",

ic-os/components/hostos/guestos/guestos.service

@@ -15,6 +15,3 @@ ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm
 Restart=always
 RestartSec=60
 TimeoutStopSec=130s
-
-[Install]
-WantedBy=multi-user.target

ic-os/components/hostos/update-config/update-config.service

@@ -2,7 +2,7 @@
 Description=Update HostOS Configuration
 RequiresMountsFor=/boot/config
 Before=node_exporter.service
-Before=guestos.service
+Before=guestos.target
 Before=log-config.service
 
 [Service]

ic-os/components/hostos/verbose-logging/verbose-logging.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=If verbose flag enabled, pipe GuestOS console to the Host terminal
-Wants=guestos.service
-After=guestos.service
+Wants=guestos.target
+After=guestos.target
 
 [Service]
 ExecStart=/opt/ic/bin/verbose-logging.sh

ic-os/components/misc/vsock/vsock-agent.service

@@ -2,8 +2,9 @@
 Description=VSOCK agent daemon
 
 [Service]
+ExecCondition=/opt/ic/bin/skip-vsock.sh
 ExecStart=/opt/ic/bin/vsock_host
-Restart=always
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

ic-os/components/monitoring/hostos/export-guestos-serial-logs.sh

@@ -1,10 +1,34 @@
 #!/bin/bash
+set -euo pipefail
 
 # Export GuestOS serial logs to journald
+# Strip ANSI color/escape codes from serial console output before forwarding to journald
 
-set -e
+source /opt/ic/bin/config.sh
 
-# Strip ANSI color/escape codes from serial console output before forwarding to journald
-tail -F /var/log/libvirt/qemu/guestos-serial.log | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t guestos-serial -p info &
+node_reward_type=$(get_config_value '.icos_settings.node_reward_type')
+
+case "${node_reward_type}" in
+    type4.1) COUNT=32 ;;
+    type4.2) COUNT=8 ;;
+    type4.3) COUNT=4 ;;
+    type4.4) COUNT=2 ;;
+    *) COUNT=1 ;;
+esac
+
+# Forward all the GuestOS logs
+for i in $(seq 0 "$((COUNT - 1))"); do
+    # A single GuestOS keeps the guestos-serial.log name
+    if [ "$COUNT" -eq 1 ]; then
+        s=""
+    else
+        s=$i
+    fi
+
+    tail -F "/var/log/libvirt/qemu/guestos-serial$s.log" | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t "guestos-serial$s" -p info &
+done
+
+# And the upgrade VM
 tail -F /var/log/libvirt/qemu/upgrade-guestos-serial.log | sed --unbuffered 's/\x1b\[[0-9;]*[a-zA-Z]//g; s/\[[0-9]\+;[0-9;]*m//g' | systemd-cat -t upgrade-guestos-serial -p info &
+
 wait

ic-os/components/monitoring/hostos/monitor-guestos.service

@@ -1,6 +1,7 @@
 [Unit]
 Description=Monitor GuestOS virtual machine
-After=guestos.service
+Wants=guestos.target
+After=guestos.target
 
 [Service]
 Type=oneshot

ic-os/components/multi-hostos/guestos.target

@@ -0,0 +1,2 @@
+[Unit]
+Description=All GuestOS VMs

ic-os/components/multi-hostos/guestos@.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Manage GuestOS %i virtual machine
+# Not Requires, otherwise guestos will restart when libvirtd restarts
+Wants=libvirtd.service
+After=libvirtd.service
+RequiresMountsFor=/var
+PartOf=guestos.target
+
+[Service]
+Type=notify
+ExecStartPre=/bin/sh -c 'echo -e "\nStarting GuestOS %i service...\n\n" | tee /dev/tty1 > /dev/ttyS0'
+ExecStartPre=/opt/ic/bin/detect-first-boot.sh
+ExecStart=/opt/ic/bin/guest_vm_runner run --slot %i
+ExecStartPost=/opt/ic/bin/manageboot.sh hostos confirm
+Restart=always
+RestartSec=60
+TimeoutStopSec=130s

ic-os/components/multi-hostos/skip-vsock.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+source /opt/ic/bin/config.sh
+
+node_reward_type="$(get_config_value '.icos_settings.node_reward_type')"
+
+if [[ "$node_reward_type" =~ ^type4(\.[0-9]+)?$ ]]; then
+    echo "node_reward_type=${node_reward_type}; skipping vsock-agent"
+    exit 1
+fi
+
+exit 0

ic-os/components/multi-hostos/start-guestos.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Startup GuestOS VMs
+RequiresMountsFor=/boot/config
+Wants=guestos.target
+Before=guestos.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/opt/ic/bin/start-guestos.sh
+
+[Install]
+WantedBy=multi-user.target

ic-os/components/multi-hostos/start-guestos.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+
+# This script dynamically starts guestos services to create the right number of VMs per node type
+
+source /opt/ic/bin/config.sh
+
+node_reward_type=$(get_config_value '.icos_settings.node_reward_type')
+
+case "${node_reward_type}" in
+    type4.1) COUNT=32 ;;
+    type4.2) COUNT=8 ;;
+    type4.3) COUNT=4 ;;
+    type4.4) COUNT=2 ;;
+    *) COUNT=1 ;;
+esac
+
+# If not a type4 node, fall to the default GuestOS flow
+if ((COUNT == 1)); then
+    systemctl start guestos.service || true
+    exit 0
+fi
+
+eval systemctl start guestos@{0..$((COUNT - 1))}.service || true

ic-os/components/setupos/install-guestos.sh

@@ -11,7 +11,7 @@ source /opt/ic/bin/functions.sh
 LV="/dev/mapper/hostlvm-guestos"
 
 function install_guestos() {
-    echo "* Installing GuestOS disk-image..."
+    echo "* Installing GuestOS disk-images..."
 
     vgchange -ay hostlvm
     log_and_halt_installation_on_error "${?}" "Unable to activate HostOS volume group."
@@ -21,9 +21,11 @@ function install_guestos() {
     tar xaf /data/guest-os.img.tar.zst -C "${TMPDIR}" disk.img
     log_and_halt_installation_on_error "${?}" "Unable to extract GuestOS disk-image."
 
-    echo "* Writing the GuestOS image to ${LV}..."
-    dd if="${TMPDIR}/disk.img" of=${LV} bs=10M conv=sparse status=progress
-    log_and_halt_installation_on_error "${?}" "Unable to install GuestOS disk-image."
+    for lv in "${LV}"*; do
+        echo "* Writing the GuestOS image to ${lv}..."
+        dd if="${TMPDIR}/disk.img" of="${lv}" bs=10M conv=sparse status=progress
+        log_and_halt_installation_on_error "${?}" "Unable to install GuestOS disk-image to ${lv}."
+    done
 
     rm -rf "${TMPDIR}"
 

ic-os/components/setupos/install-hostos.sh

@@ -6,6 +6,7 @@ set -o pipefail
 SHELL="/bin/bash"
 PATH="/sbin:/bin:/usr/sbin:/usr/bin"
 
+source /opt/ic/bin/config.sh
 source /opt/ic/bin/functions.sh
 
 function install_hostos() {
@@ -96,9 +97,44 @@ function resize_partition() {
         log_and_halt_installation_on_error "${?}" "Unable to include PV '/dev/${drive}' in VG."
     done
 
-    # Extend GuestOS LV to fill VG space
-    lvextend -i "${count}" --type striped -l +100%FREE /dev/hostlvm/guestos >/dev/null 2>&1
-    log_and_halt_installation_on_error "${?}" "Unable to extend logical volume: /dev/hostlvm/guestos"
+    local node_reward_type=$(get_config_value '.icos_settings.node_reward_type')
+
+    # Configure multiple GuestOS if type4.X
+    if [[ $node_reward_type =~ ^type4(\.[0-9]+)?$ ]]; then
+        # Cleanup the initial GuestOS
+        lvremove -f hostlvm/guestos >/dev/null 2>&1
+        log_and_halt_installation_on_error "${?}" "Unable to cleanup initial GuestOS volume"
+
+        # And set up new split volumes
+        free=$(vgs --noheadings -o vg_free_count hostlvm | tr -d ' ')
+
+        create_guestos_lvs() {
+            local total="$1"
+            local each=$((free / total))
+            local last=$((total - 1))
+            local i
+
+            for ((i = 0; i < last; i++)); do
+                lvcreate -i "${count}" --type striped -l "${each}" -n "guestos${i}" hostlvm >/dev/null 2>&1
+                log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS"
+            done
+
+            lvcreate -i "${count}" --type striped -l 100%FREE -n "guestos${last}" hostlvm >/dev/null 2>&1
+            log_and_halt_installation_on_error "${?}" "Unable to create new GuestOS"
+        }
+
+        case "${node_reward_type}" in
+            type4.1) create_guestos_lvs 32 ;;
+            type4.2) create_guestos_lvs 8 ;;
+            type4.3) create_guestos_lvs 4 ;;
+            type4.4) create_guestos_lvs 2 ;;
+        esac
+    # "Normal" behavior
+    else
+        # Extend GuestOS LV to fill VG space
+        lvextend -i "${count}" --type striped -l +100%FREE /dev/hostlvm/guestos >/dev/null 2>&1
+        log_and_halt_installation_on_error "${?}" "Unable to extend logical volume: /dev/hostlvm/guestos"
+    fi
 }
 
 # Establish run order

rs/ic_os/config/tool/src/hostos/guestos_config.rs

@@ -4,7 +4,7 @@ use config_types::{
     GuestVMType, HostOSConfig, Ipv6Config, RecoveryConfig, TrustedExecutionEnvironmentConfig,
 };
 use deterministic_ips::node_type::NodeType;
-use deterministic_ips::{MacAddr6Ext, calculate_deterministic_mac};
+use deterministic_ips::{MacAddr6Ext, calculate_deterministic_mac_w_slot};
 use std::net::Ipv6Addr;
 use std::path::Path;
 use utils::to_cidr;
@@ -18,6 +18,18 @@ pub fn generate_guestos_config(
     hostos_config: &HostOSConfig,
     guest_vm_type: GuestVMType,
     sev_certificate_chain_pem: Option<String>,
+) -> Result<GuestOSConfig> {
+    generate_guestos_config_w_slot(hostos_config, 0, guest_vm_type, sev_certificate_chain_pem)
+}
+
+/// Generate the GuestOS configuration based on the provided HostOS configuration.
+/// If hostos_config.icos_settings.enable_trusted_execution_environment is true,
+/// sev_certificate_chain_pem must be provided.
+pub fn generate_guestos_config_w_slot(
+    hostos_config: &HostOSConfig,
+    guest_vm_slot: usize,
+    guest_vm_type: GuestVMType,
+    sev_certificate_chain_pem: Option<String>,
 ) -> Result<GuestOSConfig> {
     ensure!(
         !hostos_config
@@ -47,9 +59,14 @@ pub fn generate_guestos_config(
         }
     };
 
-    let guestos_ipv6_address =
-        node_ipv6_address(node_type, hostos_config, deterministic_ipv6_config)?;
+    let guestos_ipv6_address = node_ipv6_address(
+        guest_vm_slot,
+        node_type,
+        hostos_config,
+        deterministic_ipv6_config,
+    )?;
     let peer_ipv6_address = node_ipv6_address(
+        0,
         upgrade_peer_node_type,
         hostos_config,
         deterministic_ipv6_config,
@@ -90,13 +107,15 @@ pub fn generate_guestos_config(
 }
 
 fn node_ipv6_address(
+    slot: usize,
     node_type: NodeType,
     hostos_config: &HostOSConfig,
     deterministic_config: &DeterministicIpv6Config,
 ) -> Result<Ipv6Addr> {
-    let mac = calculate_deterministic_mac(
+    let mac = calculate_deterministic_mac_w_slot(
         &hostos_config.icos_settings.mgmt_mac,
         hostos_config.icos_settings.deployment_environment,
+        slot,
         node_type,
     );
 

rs/ic_os/networking/deterministic_ips/src/lib.rs

@@ -65,20 +65,47 @@ pub fn calculate_deterministic_mac(
     deployment_environment: DeploymentEnvironment,
     node_type: NodeType,
 ) -> MacAddr6 {
-    let index = node_type.to_index();
+    calculate_deterministic_mac_w_slot(mgmt_mac, deployment_environment, 0, node_type)
+}
 
-    // NOTE: In order to be backwards compatible with existing scripts, this
-    // **MUST** have a newline.
-    let seed = format!(
-        "{}{}\n",
-        mgmt_mac.to_string().to_lowercase(),
-        deployment_environment
-    );
+pub fn calculate_deterministic_mac_w_slot(
+    mgmt_mac: &MacAddr6,
+    deployment_environment: DeploymentEnvironment,
+    slot: usize,
+    node_type: NodeType,
+) -> MacAddr6 {
+    if slot < 16 {
+        let index = node_type.to_index() | (slot as u8) << 4; // Mix in the slot index
+
+        // NOTE: In order to be backwards compatible with existing scripts, this
+        // **MUST** have a newline.
+        let seed = format!(
+            "{}{}\n",
+            mgmt_mac.to_string().to_lowercase(),
+            deployment_environment
+        );
 
-    let hash = Sha256::hash(seed.as_bytes());
+        let hash = Sha256::hash(seed.as_bytes());
 
-    // 0x6a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing.
-    [0x6a, index, hash[0], hash[1], hash[2], hash[3]].into()
+        // 0x6a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing.
+        [0x6a, index, hash[0], hash[1], hash[2], hash[3]].into()
+    } else {
+        let index = node_type.to_index();
+
+        // NOTE: In order to be backwards compatible with existing scripts, this
+        // **MUST** have a newline.
+        let seed = format!(
+            "{}{}\n",
+            mgmt_mac.to_string().to_lowercase(),
+            deployment_environment
+        );
+
+        let hash = Sha256::hash(seed.as_bytes());
+
+        // NOTE: We extend to 7a to shrink the hash, so we can fit more slot info into the address.
+        // 0x7a: locally administered, unicast MAC prefix chosen for IPv6 deterministic addressing.
+        [0x7a, index, slot as u8, hash[0], hash[1], hash[2]].into()
+    }
 }
 
 #[cfg(test)]