diff --git a/Cargo.lock b/Cargo.lock index 5ef1c14e5d4a..36ced8afd831 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1146,6 +1146,7 @@ dependencies = [ "anyhow", "clap 4.6.0", "deterministic_ips", + "grub", "http 1.4.0", "macaddr", "rand 0.8.5", diff --git a/ic-os/testing/BUILD.bazel b/ic-os/testing/BUILD.bazel index d9a0a11052d8..4790eba713b6 100644 --- a/ic-os/testing/BUILD.bazel +++ b/ic-os/testing/BUILD.bazel @@ -1,12 +1,16 @@ -load("//ic-os/components:setupos.bzl", "component_files") - package(default_visibility = ["//rs:ic-os-pkg"]) # Components required by the reload_icos script to reload HostOS -all_component_files = component_files | { +# Should be kept in sync with STAGED_OPT_IC_BIN_FILES in reload_icos.sh +all_component_files = { "//ic-os/testing:reload_icos.sh": "/opt/ic/bin/reload_icos.sh", "//ic-os/components:upgrade/boot-state.sh": "/opt/ic/bin/boot-state.sh", + "//ic-os/components/setupos:functions.sh": "/opt/ic/bin/functions.sh", + "//ic-os/components/setupos:setup-hostos-config.sh": "/opt/ic/bin/setup-hostos-config.sh", + "//ic-os/components:misc/guestos-recovery/guestos-recovery-upgrader/guestos-recovery-launcher.sh": "/opt/ic/bin/guestos-recovery-launcher.sh", + "//ic-os/components:misc/guestos-recovery/guestos-recovery-upgrader/guestos-recovery-upgrader.sh": "/opt/ic/bin/guestos-recovery-upgrader.sh", "//rs/ic_os/config/tool:config_tool_dev": "/opt/ic/bin/config_tool", + "//ic-os/components/setupos:preload-config.sh": "/opt/ic/bin/preload-config.sh", "//ic-os/components:upgrade/manageboot/manageboot.sh": "/opt/ic/bin/manageboot.sh", "//ic-os/components:monitoring/metrics.sh": "/opt/ic/bin/metrics.sh", } @@ -34,6 +38,7 @@ genrule( # Self-extracting bash script containing all dependencies to reload a HostOS (and the GuestOS in it) # Usage: # ./reload_icos_cmd --setupos-config-img= --hostos-upgrade-img= --guestos-img= +# ./reload_icos_cmd --guestos-upgrade-img= --guestos-target-boot-alternative= [--guestos-wipe-var-partition] genrule( name = "reload_icos", srcs = [":reload_icos_tar"], diff --git a/ic-os/testing/reload_icos.sh b/ic-os/testing/reload_icos.sh index b49fffb37f8a..d2f95f59bcd4 100755 --- a/ic-os/testing/reload_icos.sh +++ b/ic-os/testing/reload_icos.sh @@ -1,10 +1,24 @@ #!/bin/bash -set -e +set -euo pipefail # Directory containing this script SCRIPT_DIR="$(dirname -- "${BASH_SOURCE[0]}")" +# These are files that are already on the HostOS but we ship our own version during reload_icos, so +# the workspace version is used instead of the one already on the HostOS. +STAGED_OPT_IC_BIN_FILES=( + boot-state.sh + config_tool + functions.sh + guestos-recovery-launcher.sh + guestos-recovery-upgrader.sh + preload-config.sh + manageboot.sh + metrics.sh + setup-hostos-config.sh +) + # Mount points and paths export SETUPOS_CONFIG_MOUNT=/tmp/setupos TARGET_BOOT_PARTITION_MOUNT=/tmp/target_boot_partition @@ -13,16 +27,28 @@ GUESTOS_EXTRACT_DIR=/tmp/guestos_img # Temporary download paths SETUPOS_CONFIG_IMG_PATH=/tmp/setupos-config.img HOSTOS_UPGRADE_IMG_TAR_PATH=/tmp/hostos-upgrade-img.tar.zst -GUESTOS_IMG_TAR_PATH=/tmp/guestos.tar.zst +GUESTOS_FULL_IMG_TAR_PATH=/tmp/guestos.tar.zst +GUESTOS_UPGRADE_IMG_TAR_PATH=/tmp/guestos-upgrade.tar.zst +GUESTOS_RECOVERY_STAGE_DIR=/run/guestos-recovery/stage + +SETUPOS_CONFIG_IMG_SRC="" +HOSTOS_UPGRADE_IMG_TAR_SRC="" +GUESTOS_FULL_IMG_TAR_SRC="" +GUESTOS_UPGRADE_IMG_TAR_SRC="" +GUESTOS_DEPLOYMENT_MODE="none" +GUESTOS_TARGET_BOOT_ALTERNATIVE="" +GUESTOS_WIPE_VAR_PARTITION=false usage() { - echo "Usage: $0 [--setupos-config-img=] [--hostos-upgrade-img=] [--guestos-img=]" >&2 - echo "At least one of --hostos-upgrade-img or --guestos-img must be provided." >&2 + echo "Usage: $0 [--setupos-config-img=] [--hostos-upgrade-img=] [--guestos-img=] [--guestos-upgrade-img= --guestos-target-boot-alternative= [--guestos-wipe-var-partition]]" >&2 + echo "At least one image or config input must be provided." >&2 } cleanup() { echo "Cleaning up..." - umount /opt/ic/bin 2>/dev/null || true + for staged_file in "${STAGED_OPT_IC_BIN_FILES[@]}"; do + umount "/opt/ic/bin/${staged_file}" 2>/dev/null || true + done umount /var/ic 2>/dev/null || true umount /config 2>/dev/null || true umount /data 2>/dev/null || true @@ -30,7 +56,7 @@ cleanup() { umount "$SETUPOS_CONFIG_MOUNT" 2>/dev/null || true umount "$TARGET_BOOT_PARTITION_MOUNT" 2>/dev/null || true rm -rf "$TARGET_BOOT_PARTITION_MOUNT" "$GUESTOS_EXTRACT_DIR" 2>/dev/null || true - rm -f "$SETUPOS_CONFIG_IMG_PATH" "$HOSTOS_UPGRADE_IMG_TAR_PATH" "$GUESTOS_IMG_TAR_PATH" 2>/dev/null || true + rm -f "$SETUPOS_CONFIG_IMG_PATH" "$HOSTOS_UPGRADE_IMG_TAR_PATH" "$GUESTOS_FULL_IMG_TAR_PATH" "$GUESTOS_UPGRADE_IMG_TAR_PATH" 2>/dev/null || true } fetch_file() { @@ -39,7 +65,7 @@ fetch_file() { local dest="$2" local name="$3" - if echo "$source" | grep -q "://"; then + if [[ "$source" == *"://"* ]]; then echo "Downloading $name from $source..." curl "$source" -o "$dest" --fail --silent --show-error --clobber else @@ -63,9 +89,15 @@ fetch_images() { fetch_file "$HOSTOS_UPGRADE_IMG_TAR_SRC" "$HOSTOS_UPGRADE_IMG_TAR_PATH" "HostOS upgrade image" fi - if [ -n "$GUESTOS_IMG_TAR_SRC" ]; then - fetch_file "$GUESTOS_IMG_TAR_SRC" "$GUESTOS_IMG_TAR_PATH" "GuestOS image" - fi + case "$GUESTOS_DEPLOYMENT_MODE" in + full) + fetch_file "$GUESTOS_FULL_IMG_TAR_SRC" "$GUESTOS_FULL_IMG_TAR_PATH" "GuestOS full disk image" + ;; + upgrade) + fetch_file "$GUESTOS_UPGRADE_IMG_TAR_SRC" "$GUESTOS_UPGRADE_IMG_TAR_PATH" "GuestOS upgrade image" + ;; + none) ;; + esac } setup_temp_mounts() { @@ -73,8 +105,19 @@ setup_temp_mounts() { mkdir -p /config /data mount -t tmpfs tmpfs /config mount -t tmpfs tmpfs /data - # Mount over existing script dir and use the shipped scripts instead of the ones already deployed on the node - mount --bind "$SCRIPT_DIR" /opt/ic/bin + + for staged_file in "${STAGED_OPT_IC_BIN_FILES[@]}"; do + local source_path="${SCRIPT_DIR}/${staged_file}" + local target_path="/opt/ic/bin/${staged_file}" + + if [ ! -f "$source_path" ]; then + echo "Missing staged file: ${source_path}" >&2 + exit 1 + fi + + [ -e "$target_path" ] || touch "$target_path" + mount --bind "$source_path" "$target_path" + done } install_new_hostos() { @@ -87,23 +130,75 @@ install_new_hostos() { /opt/ic/bin/manageboot.sh --nocheck hostos upgrade-install "$HOSTOS_UPGRADE_IMG_TAR_PATH" } -install_new_guestos() { - if [ -z "$GUESTOS_IMG_TAR_SRC" ]; then - echo "No GuestOS image specified, skipping GuestOS upgrade." +install_guestos_full_image() { + if [ "$GUESTOS_DEPLOYMENT_MODE" != "full" ]; then + echo "No GuestOS full disk image specified, skipping full GuestOS installation." return fi - echo "Installing GuestOS image..." + echo "Installing GuestOS full disk image..." mkdir -p "$GUESTOS_EXTRACT_DIR" - tar -xavf "$GUESTOS_IMG_TAR_PATH" -C "$GUESTOS_EXTRACT_DIR" + tar -xavf "$GUESTOS_FULL_IMG_TAR_PATH" -C "$GUESTOS_EXTRACT_DIR" echo "Stopping GuestOS service..." systemctl stop guestos.service || true systemctl stop upgrade-guestos.service || true - echo "Writing GuestOS disk image..." + echo "Writing GuestOS full disk image..." dd if="$GUESTOS_EXTRACT_DIR/disk.img" of=/dev/mapper/hostlvm-guestos bs=4M - echo "Successfully installed GuestOS image." + echo "Successfully installed GuestOS full disk image." +} + +write_guestos_upgrade_prep_info() { + cat >"$GUESTOS_RECOVERY_STAGE_DIR/prep-info" <&2 + usage + exit 1 + ;; + esac + done +} + +validate_guestos_args() { + if [ -n "$GUESTOS_FULL_IMG_TAR_SRC" ] && [ -n "$GUESTOS_UPGRADE_IMG_TAR_SRC" ]; then + echo "--guestos-img and --guestos-upgrade-img are mutually exclusive" >&2 + usage + exit 1 + fi + + if [ -n "$GUESTOS_FULL_IMG_TAR_SRC" ]; then + GUESTOS_DEPLOYMENT_MODE="full" + elif [ -n "$GUESTOS_UPGRADE_IMG_TAR_SRC" ]; then + GUESTOS_DEPLOYMENT_MODE="upgrade" + else + GUESTOS_DEPLOYMENT_MODE="none" + fi -while [ "$#" -gt 0 ]; do - case "$1" in - --setupos-config-img=*) - SETUPOS_CONFIG_IMG_SRC="${1#*=}" - shift + case "$GUESTOS_DEPLOYMENT_MODE" in + full) + if [ -n "$GUESTOS_TARGET_BOOT_ALTERNATIVE" ] || [ "$GUESTOS_WIPE_VAR_PARTITION" = "true" ]; then + echo "--guestos-target-boot-alternative and --guestos-wipe-var-partition require --guestos-upgrade-img" >&2 + usage + exit 1 + fi ;; - --hostos-upgrade-img=*) - HOSTOS_UPGRADE_IMG_TAR_SRC="${1#*=}" - shift + upgrade) + if [ -z "$GUESTOS_TARGET_BOOT_ALTERNATIVE" ]; then + echo "--guestos-target-boot-alternative is required with --guestos-upgrade-img" >&2 + usage + exit 1 + fi + if [ "$GUESTOS_TARGET_BOOT_ALTERNATIVE" != "A" ] && [ "$GUESTOS_TARGET_BOOT_ALTERNATIVE" != "B" ]; then + echo "--guestos-target-boot-alternative must be A or B" >&2 + usage + exit 1 + fi ;; - --guestos-img=*) - GUESTOS_IMG_TAR_SRC="${1#*=}" - shift + none) + if [ -n "$GUESTOS_TARGET_BOOT_ALTERNATIVE" ] || [ "$GUESTOS_WIPE_VAR_PARTITION" = "true" ]; then + echo "--guestos-target-boot-alternative and --guestos-wipe-var-partition require --guestos-upgrade-img" >&2 + usage + exit 1 + fi ;; - --) - shift - break + esac +} + +validate_inputs() { + if [ -z "$HOSTOS_UPGRADE_IMG_TAR_SRC" ] && [ -z "$GUESTOS_FULL_IMG_TAR_SRC" ] && [ -z "$GUESTOS_UPGRADE_IMG_TAR_SRC" ] && [ -z "$SETUPOS_CONFIG_IMG_SRC" ]; then + echo "At least one of --hostos-upgrade-img, --guestos-img, --guestos-upgrade-img or --setupos-config-img must be provided" >&2 + usage + exit 1 + fi + + validate_guestos_args +} + +finalize_deployment() { + if [ -n "$HOSTOS_UPGRADE_IMG_TAR_SRC" ]; then + # Determine target alternative and mount its boot partition + target_alternative="$(/opt/ic/bin/manageboot.sh hostos target)" + echo "Will update HostOS into: $target_alternative" + mount_target_boot_partition "$target_alternative" + + # Read boot arguments from new HostOS + eval "$(cat "$TARGET_BOOT_PARTITION_MOUNT/boot_args")" + boot_args_var=BOOT_ARGS_${target_alternative} + + commit_and_reboot "${!boot_args_var}" + return + fi + + if [ -n "$SETUPOS_CONFIG_IMG_SRC" ]; then + # Config-only update: reboot with current kernel + echo "Preparing kexec reboot with current kernel..." + kexec -l /boot/vmlinuz --initrd=/boot/initrd.img --reuse-cmdline + nohup bash -c 'sleep 2; systemctl start kexec.target' >/dev/null 2>&1 & + return + fi + + case "$GUESTOS_DEPLOYMENT_MODE" in + full) + echo "Only a GuestOS full disk image was updated." + echo "Starting GuestOS..." + systemctl start guestos.service ;; - *) - echo "Unknown option: $1" >&2 - usage - exit 1 + upgrade) + echo "Only a GuestOS upgrade image was applied." ;; + none) ;; esac -done - -if [ -z "$HOSTOS_UPGRADE_IMG_TAR_SRC" ] && [ -z "$GUESTOS_IMG_TAR_SRC" ] && [ -z "$SETUPOS_CONFIG_IMG_SRC" ]; then - echo "At least one of --hostos-upgrade-img or --guestos-img or --setupos-config-img must be provided" >&2 - usage - exit 1 -fi +} trap cleanup EXIT +parse_args "$@" +validate_inputs + mount -o remount,rw / cleanup fetch_images setup_temp_mounts install_new_hostos -install_new_guestos +install_guestos setup_config - -if [ -n "$HOSTOS_UPGRADE_IMG_TAR_SRC" ]; then - # Determine target alternative and mount its boot partition - target_alternative="$(/opt/ic/bin/manageboot.sh hostos target)" - echo "Will update HostOS into: $target_alternative" - mount_target_boot_partition "$target_alternative" - - # Read boot arguments from new HostOS - eval "$(cat "$TARGET_BOOT_PARTITION_MOUNT/boot_args")" - boot_args_var=BOOT_ARGS_${target_alternative} - - commit_and_reboot "${!boot_args_var}" -elif [ -n "$SETUPOS_CONFIG_IMG_SRC" ]; then - # Config-only update: reboot with current kernel - echo "Preparing kexec reboot with current kernel..." - kexec -l /boot/vmlinuz --initrd=/boot/initrd.img --reuse-cmdline - nohup bash -c 'sleep 2; systemctl start kexec.target' >/dev/null 2>&1 & -elif [ -n "$GUESTOS_IMG_TAR_SRC" ]; then - echo "Only GuestOS was updated." - echo "Starting GuestOS..." - systemctl start guestos.service -fi +finalize_deployment diff --git a/rs/ic_os/dev_test_tools/bare_metal_deployment/BUILD.bazel b/rs/ic_os/dev_test_tools/bare_metal_deployment/BUILD.bazel index 4568e2edba7b..c4b272ef3238 100644 --- a/rs/ic_os/dev_test_tools/bare_metal_deployment/BUILD.bazel +++ b/rs/ic_os/dev_test_tools/bare_metal_deployment/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_rust//cargo:defs.bzl", "cargo_build_script") -load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library") +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") package(default_visibility = ["//rs:ic-os-pkg"]) @@ -20,6 +20,7 @@ rust_library( ], deps = [ ":build_script", + "//rs/ic_os/boot/grub", "//rs/ic_os/networking/deterministic_ips", "@crate_index//:anyhow", "@crate_index//:http", @@ -48,6 +49,7 @@ rust_binary( target_compatible_with = ["@platforms//os:linux"], deps = [ ":bare_metal_deployment", + "//rs/ic_os/boot/grub", "@crate_index//:anyhow", "@crate_index//:clap", "@crate_index//:slog", @@ -55,3 +57,27 @@ rust_binary( "@crate_index//:slog-term", ], ) + +rust_test( + name = "bare_metal_deployment_test", + crate = ":bare_metal_deployment", + deps = [ + ":build_script", + "//rs/ic_os/boot/grub", + "//rs/ic_os/networking/deterministic_ips", + "@crate_index//:anyhow", + "@crate_index//:http", + "@crate_index//:macaddr", + "@crate_index//:rand", + "@crate_index//:rexpect", + "@crate_index//:rust_ini", + "@crate_index//:slog", + "@crate_index//:ssh2", + "@crate_index//:thiserror", + ], +) + +rust_test( + name = "deploy_test", + crate = ":deploy", +) diff --git a/rs/ic_os/dev_test_tools/bare_metal_deployment/Cargo.toml b/rs/ic_os/dev_test_tools/bare_metal_deployment/Cargo.toml index d341a16e2b83..8bd8b7a20854 100644 --- a/rs/ic_os/dev_test_tools/bare_metal_deployment/Cargo.toml +++ b/rs/ic_os/dev_test_tools/bare_metal_deployment/Cargo.toml @@ -10,6 +10,7 @@ path = "src/main.rs" anyhow = { workspace = true } deterministic_ips = { path = "../../networking/deterministic_ips" } clap = { workspace = true } +grub = { path = "../../boot/grub" } http = { workspace = true } macaddr = { workspace = true } rust-ini = { workspace = true } diff --git a/rs/ic_os/dev_test_tools/bare_metal_deployment/src/deploy.rs b/rs/ic_os/dev_test_tools/bare_metal_deployment/src/deploy.rs index dd1b81626487..ddecbe92f1ab 100644 --- a/rs/ic_os/dev_test_tools/bare_metal_deployment/src/deploy.rs +++ b/rs/ic_os/dev_test_tools/bare_metal_deployment/src/deploy.rs @@ -1,6 +1,8 @@ use anyhow::Result; +use grub::BootAlternative; use ssh2::Session; -use std::io::{Read, Write}; +use std::fmt::Write as _; +use std::io::Read; use std::net::{IpAddr, SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; @@ -34,24 +36,68 @@ include!(concat!(env!("OUT_DIR"), "/reload_icos_cmd.rs")); const SSH_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); -/// Source of an OS image +/// Source of an OS image. #[derive(Debug, Clone)] pub enum ImageSource { - /// URL to download the image from + /// URL to download the image from. Url(http::Uri), - /// Local file path to copy via SCP + /// Local file path to copy via SCP. File(PathBuf), } -/// Configuration for bare metal deployment. At least one of hostos_image or guestos_image or +/// Configuration for deploying GuestOS. +#[derive(Debug, Clone)] +pub struct GuestOsDeploymentConfig { + /// The GuestOS image source. + pub image: ImageSource, + /// How the GuestOS image should be deployed. + pub mode: GuestOsDeploymentMode, +} + +impl GuestOsDeploymentConfig { + pub fn full(image: ImageSource) -> Self { + Self { + image, + mode: GuestOsDeploymentMode::Full, + } + } + + pub fn upgrade( + image: ImageSource, + target_boot_alternative: BootAlternative, + wipe_var_partition: bool, + ) -> Self { + Self { + image, + mode: GuestOsDeploymentMode::Upgrade { + target_boot_alternative, + wipe_var_partition, + }, + } + } +} + +/// Supported GuestOS deployment modes. +#[derive(Debug, Clone, Copy)] +pub enum GuestOsDeploymentMode { + /// Replace the full GuestOS disk image. + Full, + /// Install a GuestOS upgrade image into a specific boot alternative. + Upgrade { + target_boot_alternative: BootAlternative, + wipe_var_partition: bool, + }, +} + +/// Configuration for bare metal deployment. At least one of hostos_upgrade_image or guestos or /// setupos_config_image must be specified. #[derive(Debug, Clone)] pub struct DeploymentConfig { - /// The HostOS upgrade image (.tar.zst) + /// The HostOS upgrade image (.tar.zst). pub hostos_upgrade_image: Option, - /// The GuestOS image (.tar.zst) - pub guestos_image: Option, - /// The SetupOS config image (created by build-setupos-config-image.sh) + /// The GuestOS deployment configuration. + pub guestos: Option, + /// The SetupOS config image (created by build-setupos-config-image.sh). pub setupos_config_image: Option, } @@ -62,17 +108,18 @@ pub fn deploy_to_bare_metal( ssh_auth_method: &SshAuthMethod, ) -> Result<(), DeploymentError> { if config.hostos_upgrade_image.is_none() - && config.guestos_image.is_none() + && config.guestos.is_none() && config.setupos_config_image.is_none() { return Err(DeploymentError::Other(anyhow::anyhow!( - "hostos_image, guestos_image or setupos_config_image must be specified" + "hostos_upgrade_image, guestos or setupos_config_image must be specified" ))); } println!("Starting bare metal deployment to {ip}"); let ssh_session = establish_ssh_connection(ip, ssh_auth_method)?; + copy_via_scp( &ssh_session, RELOAD_ICOS_CMD, @@ -81,52 +128,70 @@ pub fn deploy_to_bare_metal( 0o755, )?; - let mut reload_icos_cmd = String::from("/tmp/reload_icos_cmd"); - - let image_configs = [ - ( - &config.setupos_config_image, - "SetupOS config", - "--setupos-config-img", - ), - ( - &config.hostos_upgrade_image, - "HostOS", - "--hostos-upgrade-img", - ), - (&config.guestos_image, "GuestOS", "--guestos-img"), - ]; - - for (image, name, flag) in image_configs { - if let Some(image) = image { - let source = match image { - ImageSource::Url(uri) => { - println!("Using {} URL: {}", name, uri); - uri.to_string() - } - ImageSource::File(path) => { - println!("Copying {} file...", name); - let epoch_secs = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(); - let random: u32 = rand::random(); - let remote_path = format!("/tmp/deploy-{}-{random:x}", epoch_secs); - copy_file_via_scp(&ssh_session, path, Path::new(&remote_path), 0o644)?; - remote_path + let mut cmd = String::from("/tmp/reload_icos_cmd"); + if let Some(image) = &config.setupos_config_image { + let source = prepare_image_source_for_reload_icos(&ssh_session, image, "SetupOS config")?; + write!(cmd, " --setupos-config-img={source}").unwrap(); + } + if let Some(image) = &config.hostos_upgrade_image { + let source = prepare_image_source_for_reload_icos(&ssh_session, image, "HostOS")?; + write!(cmd, " --hostos-upgrade-img={source}").unwrap(); + } + + if let Some(guestos) = &config.guestos { + let source = prepare_image_source_for_reload_icos(&ssh_session, &guestos.image, "GuestOS")?; + match guestos.mode { + GuestOsDeploymentMode::Full => { + write!(cmd, " --guestos-img={source}").unwrap(); + } + GuestOsDeploymentMode::Upgrade { + target_boot_alternative, + wipe_var_partition, + } => { + write!(cmd, " --guestos-upgrade-img={source}").unwrap(); + write!( + cmd, + " --guestos-target-boot-alternative={target_boot_alternative}" + ) + .unwrap(); + + if wipe_var_partition { + write!(cmd, " --guestos-wipe-var-partition").unwrap(); } - }; - reload_icos_cmd.push_str(&format!(" {}={}", flag, source)); + } } } println!("Executing reload_icos..."); - execute_bash_script(&ssh_session, &reload_icos_cmd)?; + execute_remote_command(&ssh_session, &cmd)?; println!("Deployment completed"); Ok(()) } +fn prepare_image_source_for_reload_icos( + session: &Session, + image: &ImageSource, + name: &str, +) -> Result { + match image { + ImageSource::Url(uri) => { + println!("Using {name} URL: {uri}"); + Ok(uri.to_string()) + } + ImageSource::File(path) => { + println!("Copying {} file...", name); + let epoch_secs = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let remote_path = format!("/tmp/deploy-{}-{:x}", epoch_secs, rand::random::()); + copy_file_via_scp(session, path, Path::new(&remote_path), 0o644)?; + Ok(remote_path) + } + } +} + pub fn establish_ssh_connection( host_ip: IpAddr, auth_method: &SshAuthMethod, @@ -219,18 +284,12 @@ fn copy_file_via_scp( copy_via_scp(session, file, size, remote_path, mode) } -fn execute_bash_script(session: &Session, script: &str) -> Result<(), DeploymentError> { +fn execute_remote_command(session: &Session, command: &str) -> Result<(), DeploymentError> { let mut channel = session .channel_session() .map_err(|e| DeploymentError::Other(e.into()))?; channel - .exec("bash") - .map_err(|e| DeploymentError::Other(e.into()))?; - channel - .write_all(script.as_bytes()) - .map_err(|e| DeploymentError::Other(e.into()))?; - channel - .send_eof() + .exec(command) .map_err(|e| DeploymentError::Other(e.into()))?; let mut out = String::new(); @@ -249,7 +308,7 @@ fn execute_bash_script(session: &Session, script: &str) -> Result<(), Deployment != 0 { return Err(DeploymentError::Other(anyhow::anyhow!( - "Script failed. Output: {out}\nError: {err}" + "Command failed. Output: {out}\nError: {err}" ))); } diff --git a/rs/ic_os/dev_test_tools/bare_metal_deployment/src/main.rs b/rs/ic_os/dev_test_tools/bare_metal_deployment/src/main.rs index c722f00d212f..c36c11a4db6a 100644 --- a/rs/ic_os/dev_test_tools/bare_metal_deployment/src/main.rs +++ b/rs/ic_os/dev_test_tools/bare_metal_deployment/src/main.rs @@ -1,11 +1,13 @@ use anyhow::{Context, Result, bail, ensure}; use bare_metal_deployment::deploy::{ - DeploymentConfig, DeploymentError, ImageSource, deploy_to_bare_metal, establish_ssh_connection, + DeploymentConfig, DeploymentError, GuestOsDeploymentConfig, ImageSource, deploy_to_bare_metal, + establish_ssh_connection, }; use bare_metal_deployment::{ BareMetalIpmiSession, LoginInfo, SshAuthMethod, parse_login_info_from_ini, }; -use clap::Parser; +use clap::{Parser, ValueEnum}; +use grub::BootAlternative; use std::fmt::Display; use std::path::{Path, PathBuf}; use std::process::Command; @@ -21,16 +23,44 @@ struct Args { #[arg(long)] login_info: PathBuf, - /// HostOS environment to build (e.g., "dev", "prod"). + /// HostOS image spec. + /// + /// Can be an environment name to build locally (e.g., "dev", "prod") or a 40-character + /// git hash to download a HostOS update image from download.dfinity.systems. /// If left empty, HostOS will not be deployed. #[arg(long)] hostos: Option, - /// GuestOS environment to build (e.g., "dev", "prod"). + /// GuestOS image spec. + /// + /// Can be an environment name to build locally (e.g., "dev", "prod"). When used with + /// `--guestos-mode=upgrade`, it can also be a 40-character git hash to download a GuestOS + /// update image from download.dfinity.systems. Git-hash specs are not supported for + /// `--guestos-mode=full`. /// If left empty, GuestOS will not be deployed. #[arg(long)] guestos: Option, + /// How to deploy the GuestOS image (via full image or upgrade image). + /// Upgrade image replaces just the root+boot partitions appointed by + /// `--guestos-target-boot-alternative` and optionally wipes the corresponding var partition + /// (with `--guestos-wipe-var-partition`). + #[arg(long, value_enum, requires = "guestos")] + guestos_mode: Option, + + /// Target GuestOS boot alternative for GuestOS upgrade-image deployment. + #[arg( + long, + value_enum, + requires_all = ["guestos", "guestos_mode"], + required_if_eq("guestos_mode", "upgrade") + )] + guestos_target_boot_alternative: Option, + + /// Wipe the GuestOS var partition during GuestOS upgrade-image deployment. + #[arg(long, requires_all = ["guestos", "guestos_mode", "guestos_target_boot_alternative"])] + guestos_wipe_var_partition: bool, + /// Skip building images with Bazel, use a previously built image. /// Fails if images are not found at expected paths. #[arg(long)] @@ -48,6 +78,12 @@ enum OsType { GuestOs, } +#[derive(Copy, Clone, Debug, Eq, PartialEq, ValueEnum)] +enum ImageType { + Full, + Upgrade, +} + impl Display for OsType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -112,10 +148,17 @@ fn get_ssh_keys(public_key_path: Option<&Path>) -> Result<(String, SshAuthMethod ) } -fn build_image(os_type: OsType, env: &str) -> Result<()> { +/// Builds a local image artifact for an environment-based image spec such as `dev` or `prod`. +/// Git-hash specs are resolved to download URLs by `resolve_image_source` instead. +fn build_image(os_type: OsType, spec: &str, guestos_mode: ImageType) -> Result<()> { let bazel_target = match os_type { - OsType::HostOs => format!("//ic-os/hostos/envs/{env}:update-img.tar.zst"), - OsType::GuestOs => format!("//ic-os/guestos/envs/{env}:disk-img.tar.zst"), + OsType::HostOs => format!("//ic-os/hostos/envs/{spec}:update-img.tar.zst"), + OsType::GuestOs => match guestos_mode { + ImageType::Full => format!("//ic-os/guestos/envs/{spec}:disk-img.tar.zst"), + ImageType::Upgrade => { + format!("//ic-os/guestos/envs/{spec}:update-img.tar.zst") + } + }, }; println!("Building {bazel_target}"); @@ -132,24 +175,68 @@ fn build_image(os_type: OsType, env: &str) -> Result<()> { Ok(()) } -fn artifact_path(os_type: OsType, env: &str) -> PathBuf { - match os_type { - OsType::HostOs => PathBuf::from(format!( - "bazel-bin/ic-os/hostos/envs/{env}/update-img.tar.zst", - )), - OsType::GuestOs => PathBuf::from(format!( - "bazel-bin/ic-os/guestos/envs/{env}/disk-img.tar.zst", - )), +/// Resolves an image spec to either a local build artifact or a remote update-image URL. +/// +/// Environment specs such as `dev` and `prod` are built/resolved locally. A 40-character git hash +/// resolves to a download URL on download.dfinity.systems, but only for update-image flows. +fn resolve_image_source( + os_type: OsType, + spec: &str, + build: bool, + image_type: ImageType, +) -> Result { + if is_git_hash_spec(spec) { + if os_type == OsType::GuestOs { + ensure!( + image_type == ImageType::Upgrade, + "GuestOS git-hash specs are only supported with --guestos-mode=upgrade \ + (we only publish upgrade images)" + ); + } + + let url = get_released_upgrade_image_url(os_type, spec); + println!("Using {os_type} update image URL for git revision {spec}: {url}"); + return Ok(ImageSource::Url( + url.parse() + .with_context(|| format!("Failed to parse image URL: {url}"))?, + )); } + + Ok(ImageSource::File(get_or_build_image( + os_type, spec, build, image_type, + )?)) +} + +/// Returns true if `spec` looks like a 40-character git hash. +fn is_git_hash_spec(spec: &str) -> bool { + spec.len() == 40 && spec.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn get_released_upgrade_image_url(os_type: OsType, git_hash: &str) -> String { + let directory = match os_type { + OsType::HostOs => "host-os", + OsType::GuestOs => "guest-os", + }; + + format!( + "https://download.dfinity.systems/ic/{git_hash}/{directory}/update-img/update-img.tar.zst" + ) } -fn get_or_build_image(os_type: OsType, env: &str, build: bool) -> Result { +/// Builds (unless `build` is false) and resolves the local artifact path for an environment-based +/// image spec. Git-hash specs are handled by `resolve_image_source` instead. +fn get_or_build_image( + os_type: OsType, + spec: &str, + build: bool, + guestos_mode: ImageType, +) -> Result { if build { - build_image(os_type, env)?; + build_image(os_type, spec, guestos_mode)?; } else { println!("Skipping {} build (--nobuild specified)", os_type); } - let path = artifact_path(os_type, env); + let path = artifact_path(os_type, spec, guestos_mode); ensure!( path.exists(), "{os_type} image not found at {}.", @@ -159,23 +246,75 @@ fn get_or_build_image(os_type: OsType, env: &str, build: bool) -> Result PathBuf { + match os_type { + OsType::HostOs => PathBuf::from(format!( + "bazel-bin/ic-os/hostos/envs/{spec}/update-img.tar.zst", + )), + OsType::GuestOs => match guestos_mode { + ImageType::Full => PathBuf::from(format!( + "bazel-bin/ic-os/guestos/envs/{spec}/disk-img.tar.zst", + )), + ImageType::Upgrade => PathBuf::from(format!( + "bazel-bin/ic-os/guestos/envs/{spec}/update-img.tar.zst", + )), + }, + } +} + +fn get_guestos_deployment_config(args: &Args) -> Result> { + if args.guestos.is_none() { + return Ok(None); + } + + let guestos_image_type = args.guestos_mode.unwrap_or(ImageType::Full); + let guestos_spec = args.guestos.as_deref().unwrap(); + let image = resolve_image_source( + OsType::GuestOs, + guestos_spec, + !args.nobuild, + guestos_image_type, + )?; + + let config = match guestos_image_type { + ImageType::Full => { + ensure!( + args.guestos_target_boot_alternative.is_none(), + "--guestos-target-boot-alternative can only be used with --guestos-mode=upgrade" + ); + ensure!( + !args.guestos_wipe_var_partition, + "--guestos-wipe-var-partition can only be used with --guestos-mode=upgrade" + ); + GuestOsDeploymentConfig::full(image) + } + ImageType::Upgrade => GuestOsDeploymentConfig::upgrade( + image, + args.guestos_target_boot_alternative.unwrap(), + args.guestos_wipe_var_partition, + ), + }; + + Ok(Some(config)) +} + fn main() -> Result<()> { let args = Args::parse(); if let Some(working_dir) = env::var_os("BUILD_WORKING_DIRECTORY") { env::set_current_dir(working_dir)?; } + let guestos = get_guestos_deployment_config(&args)?; + let config = DeploymentConfig { hostos_upgrade_image: args .hostos - .map(|env| get_or_build_image(OsType::HostOs, &env, !args.nobuild)) - .transpose()? - .map(ImageSource::File), - guestos_image: args - .guestos - .map(|env| get_or_build_image(OsType::GuestOs, &env, !args.nobuild)) - .transpose()? - .map(ImageSource::File), + .map(|spec| { + resolve_image_source(OsType::HostOs, &spec, !args.nobuild, ImageType::Upgrade) + }) + .transpose()?, + guestos, setupos_config_image: None, }; @@ -195,7 +334,7 @@ fn main() -> Result<()> { // Check if we need to deploy or just verify SSH connection let has_images = config.hostos_upgrade_image.is_some() - || config.guestos_image.is_some() + || config.guestos.is_some() || config.setupos_config_image.is_some(); let final_host_ip = if has_images { diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 9feb0c65d5e6..dc9aab9fb8b2 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -27,7 +27,9 @@ use crate::driver::{ }; use anyhow::{Context, Result, bail}; use bare_metal_deployment::SshAuthMethod; -use bare_metal_deployment::deploy::{DeploymentConfig, ImageSource, deploy_to_bare_metal}; +use bare_metal_deployment::deploy::{ + DeploymentConfig, GuestOsDeploymentConfig, ImageSource, deploy_to_bare_metal, +}; use config_tool::hostos::guestos_bootstrap_image::BootstrapOptions; use config_tool::setupos::{ config_ini::ConfigIniSettings, @@ -680,7 +682,7 @@ pub fn setup_baremetal_instance( let config = DeploymentConfig { hostos_upgrade_image: Some(ImageSource::Url(hostos_url)), - guestos_image: Some(guestos_image_source), + guestos: Some(GuestOsDeploymentConfig::full(guestos_image_source)), setupos_config_image: Some(ImageSource::File(config_image.to_path_buf())), };