From e4b94e59402c0e1b49c00fff12d309eb261392ec Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Tue, 23 Jun 2026 18:24:23 +0200 Subject: [PATCH 1/2] wip --- BUILD.bazel | 29 ++++++++++++ bazel/defs.bzl | 38 +++++++++++++++ ci/container/files/packages.common | 2 - .../dev-tools/bare_metal_deployment/deploy.py | 11 +++++ .../dev-tools/bare_metal_deployment/tools.bzl | 5 ++ ic-os/dev-tools/build-setupos-config-image.sh | 12 +++-- .../build_tools/partition_tools/BUILD.bazel | 10 ++++ .../build_tools/partition_tools/src/fat.rs | 26 ++++++++--- rs/ic_os/config/tool/BUILD.bazel | 22 ++++++++- .../src/hostos/guestos_bootstrap_image.rs | 25 +++++++++- rs/ic_os/os_tools/guest_vm_runner/BUILD.bazel | 24 ++++++++-- .../create-universal-vm-config-image.sh | 8 +++- rs/tests/driver/src/driver/bootstrap.rs | 2 + rs/tests/driver/src/driver/universal_vm.rs | 2 + rs/tests/idx/colocate_test.rs | 46 ++++++++++++++++++- rs/tests/run_systest.sh | 20 +++++++- rs/tests/system_tests.bzl | 7 +++ 17 files changed, 265 insertions(+), 24 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 68e98bac510b..546f4fdfa10d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//rules:common_settings.bzl", "string_setting") load("@gazelle//:def.bzl", "gazelle") load("@rules_python//python:pip.bzl", "compile_pip_requirements") +load("//bazel:defs.bzl", "tool_file") load("//ci/src/artifacts:upload.bzl", "upload_artifacts") package(default_visibility = ["//visibility:public"]) @@ -248,6 +249,34 @@ alias( }), ) +# Single-file, executable handles for the FAT image tools above. The aliases +# resolve to multi-file configure_make bundles; these expose just the binary so +# it can be passed around by path (e.g. fed to the system-test driver as a +# $(rootpath) runtime dep for creating UVM/SetupOS config images). +# These resolve to @platforms//:incompatible on non-(linux, x86_64) platforms (see +# the aliases above), so gate the extractors to the same platforms; otherwise the +# rule would run with an empty bundle and fail. Consumers propagate this and are +# skipped accordingly off-platform. +tool_file( + name = "mkfs.fat.bin", + binary = "mkfs.fat", + bundle = "//:mkfs.fat", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + +tool_file( + name = "mtools.bin", + binary = "mtools", + bundle = "//:mtools", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + ### e2fsdroid, used to populate ext4 filesystem images (fs_config + SELinux contexts) alias( diff --git a/bazel/defs.bzl b/bazel/defs.bzl index 881cc5d7319c..95b5c55878d1 100644 --- a/bazel/defs.bzl +++ b/bazel/defs.bzl @@ -129,6 +129,44 @@ mcopy = rule( }, ) +def _tool_file(ctx): + """Extracts a single named binary out of a multi-file tool bundle. + + Our `mkfs.fat`/`mtools` targets are `configure_make` bundles (the binary plus + e.g. an include dir), so they can't be passed around by a single path (e.g. + as a `$(rootpath)` runtime dep). This picks the requested binary out by + basename and exposes it as a standalone, executable, single-file target. + + The output keeps the binary's original basename (placed under a per-target + directory to avoid colliding with the bundle alias of the same name). This + matters for multi-call binaries like `mtools`, which dispatch on argv[0]: + it must still be invoked as `mtools -c `. + """ + tool = None + for f in ctx.files.bundle: + if f.basename == ctx.attr.binary: + tool = f + break + if not tool: + fail("could not locate '{}' binary among {} outputs".format(ctx.attr.binary, ctx.attr.bundle.label)) + + out = ctx.actions.declare_file("{}/{}".format(ctx.label.name, ctx.attr.binary)) + ctx.actions.run_shell( + command = "cp -p {src} {out} && chmod +x {out}".format(src = tool.path, out = out.path), + inputs = [tool], + outputs = [out], + ) + return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))] + +tool_file = rule( + implementation = _tool_file, + doc = "Exposes a single named binary from a multi-file tool bundle as a standalone executable file.", + attrs = { + "bundle": attr.label(mandatory = True, allow_files = True), + "binary": attr.string(mandatory = True), + }, +) + # Binaries needed for testing with canister_sandbox _SANDBOX_DATA = [ "//rs/canister_sandbox", diff --git a/ci/container/files/packages.common b/ci/container/files/packages.common index ba20acd4cb1e..701aa7c41446 100644 --- a/ci/container/files/packages.common +++ b/ci/container/files/packages.common @@ -27,7 +27,6 @@ llvm # IC-OS clang cryptsetup-bin -dosfstools # provides mkfs.vfat faketime fdisk iasl # to build OVMF @@ -38,7 +37,6 @@ libselinux-dev libsystemd-dev # Linked in by IC-OS binaries for managing virtual machines programmatically. libvirt-dev -mtools # used for mcopy and mmd nasm # to build OVMF podman rsync diff --git a/ic-os/dev-tools/bare_metal_deployment/deploy.py b/ic-os/dev-tools/bare_metal_deployment/deploy.py index 5b19c413f3b0..30bc8a32fb33 100755 --- a/ic-os/dev-tools/bare_metal_deployment/deploy.py +++ b/ic-os/dev-tools/bare_metal_deployment/deploy.py @@ -109,6 +109,10 @@ class Args: # Path to the setupos-inject-config tool. Necessary if any inject* args are present inject_configuration_tool: Optional[str] = None + # Path to the Bazel-built mtools binary, used by setupos-inject-config to write the + # FAT config partition. + mtools_tool: Optional[str] = None + # Time to wait between each remote deployment, in minutes wait_time: int = field(default=DEFAULT_SETUPOS_WAIT_TIME_MINS, alias="-t") @@ -160,6 +164,7 @@ def __post_init__(self): ), "Both ipv6_prefix and ipv6_gateway flags must be present or none" if self.inject_image_ipv6_prefix: assert self.inject_configuration_tool, "setupos_inject_config tool required to modify image" + assert self.mtools_tool, "mtools tool required to modify image" ipv4_args = [ self.inject_image_ipv4_address, self.inject_image_ipv4_gateway, @@ -706,6 +711,7 @@ def upload_to_file_share( def inject_config_into_image( setupos_inject_config_path: Path, + mtools_path: Path, working_dir: Path, compressed_image_path: Path, node_reward_type: str, @@ -732,6 +738,9 @@ def is_executable(p: Path) -> bool: assert setupos_inject_config_path.exists() and is_executable(setupos_inject_config_path) + # Absolute path: setupos-inject-config runs mtools with its own working directory. + mtools = os.path.abspath(mtools_path) + invoke.run(f"tar --extract --zstd --file {compressed_image_path} --directory {working_dir}", echo=True) img_path = Path(f"{working_dir}/disk.img") @@ -764,6 +773,7 @@ def is_executable(p: Path) -> bool: invoke.run( f"{setupos_inject_config_path} {image_part} {reward_part} {prefix_part} {gateway_part} {ipv4_part} {enable_trusted_execution_environment_part} {verbose_part} {admin_key_part}", echo=True, + env={"MTOOLS": mtools}, ) # Reuse the name of the compressed image path in the working directory @@ -836,6 +846,7 @@ def main(): tmpdir = tempfile.mkdtemp() modified_image_path = inject_config_into_image( Path(args.inject_configuration_tool), + Path(args.mtools_tool), Path(tmpdir), Path(args.upload_img), args.inject_image_node_reward_type, diff --git a/ic-os/dev-tools/bare_metal_deployment/tools.bzl b/ic-os/dev-tools/bare_metal_deployment/tools.bzl index 962fc47d4a0f..b9d82f8e8059 100644 --- a/ic-os/dev-tools/bare_metal_deployment/tools.bzl +++ b/ic-os/dev-tools/bare_metal_deployment/tools.bzl @@ -34,6 +34,10 @@ def launch_bare_metal(name, image_zst_file): "$(location :" + binary_name + ")", "--inject_configuration_tool", "$(location //rs/ic_os/dev_test_tools/setupos-image-config:setupos-inject-config)", + # Bazel-built mtools (the build container no longer ships system mtools), + # used by setupos-inject-config to write the FAT config partition. + "--mtools_tool", + "$(location //:mtools.bin)", "--upload_img", "$(location " + image_zst_file + ")", "--deterministic_ips_tool", @@ -51,6 +55,7 @@ def launch_bare_metal(name, image_zst_file): data = [ ":" + binary_name, image_zst_file, + "//:mtools.bin", "//rs/ic_os/dev_test_tools/setupos-image-config:setupos-inject-config", "//ic-os/dev-tools/bare_metal_deployment:redfish_scripts", "//ic-os/dev-tools/bare_metal_deployment:benchmark_runner.sh", diff --git a/ic-os/dev-tools/build-setupos-config-image.sh b/ic-os/dev-tools/build-setupos-config-image.sh index 7d8a6f521eec..18d422e73e13 100755 --- a/ic-os/dev-tools/build-setupos-config-image.sh +++ b/ic-os/dev-tools/build-setupos-config-image.sh @@ -6,6 +6,10 @@ CONFIG_DIR="${1}" DATA_DIR="${2}" OUTPUT_IMAGE="${3}" +# MKFS_VFAT and MTOOLS are the Bazel-built mkfs.fat / mtools binaries, set in the +# environment by the system_test rule (so the build container doesn't need system +# dosfstools/mtools). mtools is a multi-call binary, driven as `mtools -c `. + TMPDIR=$(mktemp -d) tar cf "${TMPDIR}/config.tar" -C "${CONFIG_DIR}" . @@ -13,11 +17,11 @@ tar cf "${TMPDIR}/data.tar" -C "${DATA_DIR}" . truncate -s 10M "${OUTPUT_IMAGE}" -/usr/sbin/mkfs.vfat "${OUTPUT_IMAGE}" +"$MKFS_VFAT" "${OUTPUT_IMAGE}" -mlabel -i "${OUTPUT_IMAGE}" ::OVERRIDE +"$MTOOLS" -c mlabel -i "${OUTPUT_IMAGE}" ::OVERRIDE -mcopy -i "${OUTPUT_IMAGE}" -o "${TMPDIR}/config.tar" :: -mcopy -i "${OUTPUT_IMAGE}" -o "${TMPDIR}/data.tar" :: +"$MTOOLS" -c mcopy -i "${OUTPUT_IMAGE}" -o "${TMPDIR}/config.tar" :: +"$MTOOLS" -c mcopy -i "${OUTPUT_IMAGE}" -o "${TMPDIR}/data.tar" :: rm -rf "${TMPDIR}" diff --git a/rs/ic_os/build_tools/partition_tools/BUILD.bazel b/rs/ic_os/build_tools/partition_tools/BUILD.bazel index 1c5a86061941..655c1f6a480c 100644 --- a/rs/ic_os/build_tools/partition_tools/BUILD.bazel +++ b/rs/ic_os/build_tools/partition_tools/BUILD.bazel @@ -23,6 +23,16 @@ rust_library( rust_test( name = "partition_tools_test", crate = ":partition_tools", + # Bazel-built FAT tools used by the tests (instead of system dosfstools/mtools); + # fat.rs reads MKFS_VFAT/MTOOLS from the environment. + data = [ + "//:mkfs.fat.bin", + "//:mtools.bin", + ], + env = { + "MKFS_VFAT": "$(rootpath //:mkfs.fat.bin)", + "MTOOLS": "$(rootpath //:mtools.bin)", + }, proc_macro_deps = [ # Keep sorted. "@crate_index//:indoc", diff --git a/rs/ic_os/build_tools/partition_tools/src/fat.rs b/rs/ic_os/build_tools/partition_tools/src/fat.rs index df22e4d7ea1d..fcd332a1416a 100644 --- a/rs/ic_os/build_tools/partition_tools/src/fat.rs +++ b/rs/ic_os/build_tools/partition_tools/src/fat.rs @@ -6,6 +6,16 @@ use anyhow::{Context, Result, anyhow, ensure}; use crate::Partition; use crate::gpt; +/// Build a `Command` that runs mtools' `mcopy`. `MTOOLS` is the Bazel-built +/// mtools binary, provided via the environment; it is a multi-call binary, so +/// `mcopy` is invoked as `mtools -c mcopy ...`. +fn mcopy_command() -> Command { + let mtools = std::env::var("MTOOLS").expect("MTOOLS environment variable must be set"); + let mut command = Command::new(mtools); + command.arg("-c").arg("mcopy"); + command +} + pub struct FatPartition { offset_bytes: Option, original: PathBuf, @@ -41,7 +51,7 @@ impl Partition for FatPartition { /// Copy a file into place fn write_file(&mut self, input: &Path, output: &Path) -> Result<()> { let out = if let Some(offset) = self.offset_bytes { - Command::new("mcopy") + mcopy_command() .args([ "-o", "-i", @@ -52,7 +62,7 @@ impl Partition for FatPartition { .output() .context("failed to run mcopy")? } else { - Command::new("mcopy") + mcopy_command() .args([ "-o", "-i", @@ -83,7 +93,7 @@ impl Partition for FatPartition { ); let out = if let Some(offset) = self.offset_bytes { - Command::new("mcopy") + mcopy_command() .args([ "-s", // recursive copy "-o", // overwrite existing files @@ -95,7 +105,7 @@ impl Partition for FatPartition { .output() .context("failed to run mcopy")? } else { - Command::new("mcopy") + mcopy_command() .args([ "-s", // recursive copy "-o", // overwrite existing files @@ -147,7 +157,7 @@ impl FatPartition { // Capture and return stdout, which may be used to "read" the file directly fn copy_file_inner(&mut self, from: &Path, to: &Path) -> Result> { let out = if let Some(offset) = self.offset_bytes { - Command::new("mcopy") + mcopy_command() .args([ "-o", "-i", @@ -158,7 +168,7 @@ impl FatPartition { .output() .context("failed to run mcopy")? } else { - Command::new("mcopy") + mcopy_command() .args([ "-o", "-i", @@ -194,7 +204,9 @@ mod test { ]) .status()?; - Command::new("/usr/sbin/mkfs.fat") + let mkfs_vfat = + std::env::var("MKFS_VFAT").expect("MKFS_VFAT environment variable must be set"); + Command::new(mkfs_vfat) .args(["-F", "32", "-i", "0"]) .arg(path.as_os_str()) .status()?; diff --git a/rs/ic_os/config/tool/BUILD.bazel b/rs/ic_os/config/tool/BUILD.bazel index d83f920ebbdc..3fc45566a4de 100644 --- a/rs/ic_os/config/tool/BUILD.bazel +++ b/rs/ic_os/config/tool/BUILD.bazel @@ -103,7 +103,17 @@ rust_test( name = "config_lib_test_dev", crate = ":config_lib_dev", crate_features = ["dev"], - env = {"CARGO_MANIFEST_DIR": "rs/ic_os/generate_guestos_vm_config"}, + # build_bootstrap_config_image() reads mkfs.vfat/mtools from MKFS_VFAT/MTOOLS + # (the build container no longer ships system dosfstools/mtools). + data = [ + "//:mkfs.fat.bin", + "//:mtools.bin", + ], + env = { + "CARGO_MANIFEST_DIR": "rs/ic_os/generate_guestos_vm_config", + "MKFS_VFAT": "$(rootpath //:mkfs.fat.bin)", + "MTOOLS": "$(rootpath //:mtools.bin)", + }, deps = [ # Keep sorted. "@crate_index//:once_cell", @@ -114,6 +124,16 @@ rust_test( rust_test( name = "config_lib_test", crate = ":config_lib", + # build_bootstrap_config_image() reads mkfs.vfat/mtools from MKFS_VFAT/MTOOLS + # (the build container no longer ships system dosfstools/mtools). + data = [ + "//:mkfs.fat.bin", + "//:mtools.bin", + ], + env = { + "MKFS_VFAT": "$(rootpath //:mkfs.fat.bin)", + "MTOOLS": "$(rootpath //:mtools.bin)", + }, deps = [ # Keep sorted. "@crate_index//:once_cell", diff --git a/rs/ic_os/config/tool/src/hostos/guestos_bootstrap_image.rs b/rs/ic_os/config/tool/src/hostos/guestos_bootstrap_image.rs index eb13104f9515..d7c0d9995fb8 100644 --- a/rs/ic_os/config/tool/src/hostos/guestos_bootstrap_image.rs +++ b/rs/ic_os/config/tool/src/hostos/guestos_bootstrap_image.rs @@ -5,6 +5,27 @@ use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::process::Command; +/// `mkfs.vfat` command. The Bazel build provides the binary via the `MKFS_VFAT` +/// environment variable (the build container no longer ships system dosfstools); +/// inside the IC-OS images at runtime it falls back to the system tool. +fn mkfs_vfat_command() -> Command { + Command::new(std::env::var("MKFS_VFAT").unwrap_or_else(|_| "/usr/sbin/mkfs.vfat".to_string())) +} + +/// `mcopy` command. The Bazel build provides mtools via the `MTOOLS` environment +/// variable (a multi-call binary, driven as `mtools -c mcopy`); inside the IC-OS +/// images at runtime it falls back to the system `mcopy`. +fn mcopy_command() -> Command { + match std::env::var("MTOOLS") { + Ok(mtools) if !mtools.is_empty() => { + let mut command = Command::new(mtools); + command.arg("-c").arg("mcopy"); + command + } + _ => Command::new("/usr/bin/mcopy"), + } +} + /// Configuration options for GuestOS bootstrap image/tar creation. #[derive(Default, Debug, Clone, Eq, PartialEq)] pub struct BootstrapOptions { @@ -61,7 +82,7 @@ impl BootstrapOptions { .context("Failed to set output file size")?; // Format the disk image as FAT - if !Command::new("/usr/sbin/mkfs.vfat") + if !mkfs_vfat_command() .arg("-n") .arg("CONFIG") .arg(out_file) @@ -124,7 +145,7 @@ impl BootstrapOptions { .collect::>>() .context("Failed to collect config directory entries")?; - let output = Command::new("/usr/bin/mcopy") + let output = mcopy_command() .arg("-i") .arg(vfat_image) .arg("-s") diff --git a/rs/ic_os/os_tools/guest_vm_runner/BUILD.bazel b/rs/ic_os/os_tools/guest_vm_runner/BUILD.bazel index 0390df61dc5d..aa4e5919ade8 100644 --- a/rs/ic_os/os_tools/guest_vm_runner/BUILD.bazel +++ b/rs/ic_os/os_tools/guest_vm_runner/BUILD.bazel @@ -103,7 +103,14 @@ rust_test( name = "guest_vm_runner_test", crate = ":guest_vm_runner_dev", crate_features = ["dev"], - data = glob(["golden/*"]), + data = glob(["golden/*"]) + [ + "//:mkfs.fat.bin", + "//:mtools.bin", + ], + env = { + "MKFS_VFAT": "$(rootpath //:mkfs.fat.bin)", + "MTOOLS": "$(rootpath //:mtools.bin)", + }, deps = [ # Keep sorted. "@crate_index//:goldenfile", @@ -121,8 +128,19 @@ rust_test( "dev", "integration_tests", ], - data = glob(["golden/*"]) + ["//ic-os/guestos/envs/prod:disk-img-for-tests.tar"], - env = {"ICOS_IMAGE": "$(rootpath //ic-os/guestos/envs/prod:disk-img-for-tests.tar)"}, + # Launching a GuestOS VM calls assemble_config_media() -> + # build_bootstrap_config_image(), which reads mkfs.vfat/mtools from + # MKFS_VFAT/MTOOLS (the build container no longer ships system dosfstools/mtools). + data = glob(["golden/*"]) + [ + "//:mkfs.fat.bin", + "//:mtools.bin", + "//ic-os/guestos/envs/prod:disk-img-for-tests.tar", + ], + env = { + "ICOS_IMAGE": "$(rootpath //ic-os/guestos/envs/prod:disk-img-for-tests.tar)", + "MKFS_VFAT": "$(rootpath //:mkfs.fat.bin)", + "MTOOLS": "$(rootpath //:mtools.bin)", + }, # Mark it manual here and expose it in //ic-os/tests:integration_tests tags = ["manual"], deps = [ diff --git a/rs/tests/driver/assets/create-universal-vm-config-image.sh b/rs/tests/driver/assets/create-universal-vm-config-image.sh index 920f6989b17e..3c4524074688 100755 --- a/rs/tests/driver/assets/create-universal-vm-config-image.sh +++ b/rs/tests/driver/assets/create-universal-vm-config-image.sh @@ -62,6 +62,10 @@ if [[ -z "${INPUT_DIR:-}" || -z "${OUTPUT_FILE:-}" || -z "${LABEL:-}" ]]; then usage fi +# MKFS_VFAT and MTOOLS are the Bazel-built mkfs.fat / mtools binaries, set in the +# environment by the system_test rule (so the build container doesn't need system +# dosfstools/mtools). mtools is a multi-call binary, driven as `mtools -c `. + tmp=$(mktemp) finalize() { @@ -74,6 +78,6 @@ size=$(du --bytes -s "$INPUT_DIR" | awk '{print $1}') size=$((2 * size + 1048576)) echo "image size: $size" truncate -s $size "$tmp" -/usr/sbin/mkfs.vfat -n "$LABEL" "$tmp" -mcopy -i "$tmp" -sQ "$INPUT_DIR"/* :: +"$MKFS_VFAT" -n "$LABEL" "$tmp" +"$MTOOLS" -c mcopy -i "$tmp" -sQ "$INPUT_DIR"/* :: zstd --threads=0 -10 -i "$tmp" -o "$OUTPUT_FILE" --force diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 1acdf210a02a..1de22c74ec98 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -826,6 +826,8 @@ fn create_setupos_config_image( // Pack dirs into config image let config_image = nested_vm.get_setupos_config_image_path()?; + // The MKFS_VFAT/MTOOLS env vars (Bazel-built tool paths) are set on the test + // process by the system_test rule and inherited by this child script. let status = Command::new(build_setupos_config_image) .arg(config_dir) .arg(data_dir) diff --git a/rs/tests/driver/src/driver/universal_vm.rs b/rs/tests/driver/src/driver/universal_vm.rs index adf6f2f1afd7..ecda968c8437 100644 --- a/rs/tests/driver/src/driver/universal_vm.rs +++ b/rs/tests/driver/src/driver/universal_vm.rs @@ -203,6 +203,8 @@ fn create_universal_vm_config_image( label: &str, ) -> Result<()> { // pipe the uvm creation script into bash + // The MKFS_VFAT/MTOOLS/ZSTD env vars (Bazel-built tool paths) are set on the + // test process by the system_test rule and inherited by this child. let mut cmd = Command::new("/bin/bash") .stdin(Stdio::piped()) // with .spawn() the parent's stdout & stderr are inherited diff --git a/rs/tests/idx/colocate_test.rs b/rs/tests/idx/colocate_test.rs index 6ad2de186135..e3abd3f63dfe 100644 --- a/rs/tests/idx/colocate_test.rs +++ b/rs/tests/idx/colocate_test.rs @@ -32,6 +32,11 @@ pub const RUNTIME_DEPS_TAR_ZST: &str = "runtime_deps.tar.zst"; pub const ENV_TAR_ZST: &str = "env.tar.zst"; const DASHBOARDS_TAR_ZST: &str = "dashboards.tar.zst"; +/// Where the runtime-dep tarball is unpacked inside the UVM container: the host +/// dir /home/admin/test is bind-mounted at /home/root/test (see the `docker run` +/// below), and the deps land under runtime_deps/ there. +const CONTAINER_RUNTIME_DEPS_DIR: &str = "/home/root/test/runtime_deps"; + pub const TEST_STATUS_CHECK_RETRY: Duration = Duration::from_secs(5); type ExitCode = i32; @@ -166,7 +171,13 @@ fn setup(env: TestEnv) { .output() .unwrap_or_else(|e| panic!("Failed to list env: {e}")); - file.write_all(&output.stdout).expect("Could not write env"); + // run_systest.sh leaves MKFS_VFAT/MTOOLS pointing at their host location + // (the wrapper dereferences them locally, see UniversalVm::start). Inside + // the UVM container the runtime deps live under CONTAINER_RUNTIME_DEPS_DIR, + // so rewrite just those two variables for the env we forward to the + // container's (inner) test driver. + let env_file = rewrite_fat_tool_paths_for_container(&output.stdout); + file.write_all(&env_file).expect("Could not write env"); scp_send_to( log.clone(), @@ -327,6 +338,39 @@ chmod +x /home/admin/run info!(log, "test execution has finished successfully"); } +/// Rewrites the `MKFS_VFAT`/`MTOOLS` variables (the Bazel-built FAT tools) in a +/// `KEY=VALUE`-per-line env dump so they point at the runtime-deps location +/// inside the UVM container instead of their host path. `run_systest.sh` +/// intentionally leaves them host-pathed because the colocate wrapper +/// dereferences them locally (building the driver VM's SSH config image); the +/// inner test driver, running in the container, needs the in-container path. The +/// symlink basename is identical on both sides, so only the directory changes. +/// Operates on raw bytes so a non-UTF-8 entry elsewhere in the dump is preserved +/// verbatim rather than corrupted. +fn rewrite_fat_tool_paths_for_container(env_dump: &[u8]) -> Vec { + const PREFIXES: [&[u8]; 2] = [b"MKFS_VFAT=", b"MTOOLS="]; + let mut out = Vec::with_capacity(env_dump.len()); + for (i, line) in env_dump.split(|&b| b == b'\n').enumerate() { + if i > 0 { + out.push(b'\n'); + } + match PREFIXES + .iter() + .find_map(|prefix| line.strip_prefix(*prefix).map(|value| (*prefix, value))) + { + Some((prefix, value)) => { + let name = value.rsplit(|&b| b == b'/').next().unwrap_or(value); + out.extend_from_slice(prefix); + out.extend_from_slice(CONTAINER_RUNTIME_DEPS_DIR.as_bytes()); + out.push(b'/'); + out.extend_from_slice(name); + } + None => out.extend_from_slice(line), + } + } + out +} + fn start_test(env: TestEnv, uvm: &DeployedUniversalVm) { let run_test_script = r#" set -E diff --git a/rs/tests/run_systest.sh b/rs/tests/run_systest.sh index 7ff207b2b330..131b91071a26 100755 --- a/rs/tests/run_systest.sh +++ b/rs/tests/run_systest.sh @@ -72,10 +72,26 @@ for env_var in "${runtime_dep_env_vars[@]}"; do old_dep_hash="$(sha256sum <<<"$old_dep" | cut -d' ' -f1)" old_dep_name="$(basename "$old_dep")" new_dep="$old_dep_hash-$old_dep_name" + if [ "$old_dep_name" = "mtools" ]; then + new_dep="$old_dep_name" + fi old_dep_abs="$(realpath $old_dep)" - echo "Linking runtime dependency for $env_var: $runtime_dep_base/$new_dep -> $old_dep_abs" >&2 ln -sf "$old_dep_abs" "$runtime_deps/$new_dep" - export "$env_var=$runtime_dep_base/$new_dep" + + # By default the env var points at where the dep lives for the process that + # consumes it: $runtime_dep_base, i.e. the in-container path for colocated + # tests and the local path otherwise. MKFS_VFAT/MTOOLS are an exception: the + # colocated *wrapper* itself dereferences them on the host (UniversalVm::start + # builds the driver VM's SSH config image locally), so they must resolve + # locally. colocate_test.rs rewrites them to the in-container path in the + # environment it forwards to the UVM container, where the inner test driver + # consumes them. + dep_base="$runtime_dep_base" + case "$env_var" in + MKFS_VFAT | MTOOLS) dep_base="$runtime_deps" ;; + esac + echo "Linking runtime dependency for $env_var: $dep_base/$new_dep -> $old_dep_abs" >&2 + export "$env_var=$dep_base/$new_dep" done # Set environment variables based on volatile status variables: diff --git a/rs/tests/system_tests.bzl b/rs/tests/system_tests.bzl index d7caecdc127c..33d56f50cc6b 100644 --- a/rs/tests/system_tests.bzl +++ b/rs/tests/system_tests.bzl @@ -129,6 +129,13 @@ def system_test( _runtime_deps["TEST_BIN"] = test_driver_target + # Bazel-built tools the driver uses to assemble FAT config images for + # universal VMs / SetupOS, instead of system mkfs.vfat/mtools. These are set + # on the test process and read (by name) from the driver, the config-image + # scripts it spawns, and partition_tools. + _runtime_deps["MKFS_VFAT"] = "//:mkfs.fat.bin" + _runtime_deps["MTOOLS"] = "//:mtools.bin" + env_var_files = {} icos_images = dict() From 7000d37ebaef3939b5963801a702c37c360a9647 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation <> Date: Tue, 23 Jun 2026 16:28:38 +0000 Subject: [PATCH 2/2] Updating container images to tag: f96c20ebb316dcc7dd4e537b40fd2b88a1d2fa43cf2ba80c243076c23ab8e136 ic-build: sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 ic-dev: sha256:b894f45fd9ed37fcb19c8cbe55cba88ca0b003b1f2350b403829b54a233077f5 --- .devcontainer/devcontainer.json | 2 +- .github/workflows/api-bn-recovery-test.yml | 2 +- .github/workflows/ci-main.yml | 2 +- .github/workflows/ci-pr-only.yml | 2 +- .github/workflows/container-api-bn-recovery.yml | 2 +- .github/workflows/container-scan-nightly.yml | 2 +- .github/workflows/pocket-ic-tests-windows.yml | 2 +- .github/workflows/rate-limits-backend-release.yml | 2 +- .github/workflows/release-testing.yml | 2 +- .github/workflows/rosetta-release.yml | 2 +- .github/workflows/salt-sharing-canister-release.yml | 2 +- .github/workflows/schedule-daily.yml | 2 +- .github/workflows/schedule-rust-bench.yml | 2 +- .github/workflows/system-tests-benchmarks-nightly.yml | 2 +- .github/workflows/update-mainnet-canister-revisions.yaml | 2 +- ci/container/TAG | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cf638c7cfca4..9b1ee40334c3 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "ghcr.io/dfinity/ic-dev@sha256:0b0e1cf7202f83a29c645a84e984aea8fc2f18c3ff61869b4fd80f30949d7aa0", + "image": "ghcr.io/dfinity/ic-dev@sha256:b894f45fd9ed37fcb19c8cbe55cba88ca0b003b1f2350b403829b54a233077f5", "remoteUser": "ubuntu", "privileged": true, "runArgs": [ diff --git a/.github/workflows/api-bn-recovery-test.yml b/.github/workflows/api-bn-recovery-test.yml index 712859748acb..51e53b9fa1e5 100644 --- a/.github/workflows/api-bn-recovery-test.yml +++ b/.github/workflows/api-bn-recovery-test.yml @@ -22,7 +22,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index d578dfa42d93..1a55511f9184 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -33,7 +33,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 90 diff --git a/.github/workflows/ci-pr-only.yml b/.github/workflows/ci-pr-only.yml index 2be2c80357fc..5699e9a9d122 100644 --- a/.github/workflows/ci-pr-only.yml +++ b/.github/workflows/ci-pr-only.yml @@ -37,7 +37,7 @@ jobs: runs-on: &dind-small-setup labels: dind-small container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --mount type=tmpfs,target="/tmp/containers" steps: diff --git a/.github/workflows/container-api-bn-recovery.yml b/.github/workflows/container-api-bn-recovery.yml index f4fb0bed917b..b596bda0c53f 100644 --- a/.github/workflows/container-api-bn-recovery.yml +++ b/.github/workflows/container-api-bn-recovery.yml @@ -28,7 +28,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/home/buildifier/.local/share/containers" diff --git a/.github/workflows/container-scan-nightly.yml b/.github/workflows/container-scan-nightly.yml index 816162aaf64f..2af0646b82e1 100644 --- a/.github/workflows/container-scan-nightly.yml +++ b/.github/workflows/container-scan-nightly.yml @@ -12,7 +12,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 60 diff --git a/.github/workflows/pocket-ic-tests-windows.yml b/.github/workflows/pocket-ic-tests-windows.yml index 124cac133534..db15a86610f2 100644 --- a/.github/workflows/pocket-ic-tests-windows.yml +++ b/.github/workflows/pocket-ic-tests-windows.yml @@ -45,7 +45,7 @@ jobs: bazel-build-pocket-ic: name: Bazel Build PocketIC container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 90 diff --git a/.github/workflows/rate-limits-backend-release.yml b/.github/workflows/rate-limits-backend-release.yml index d3fa85fec87b..784f3ef5bbd7 100644 --- a/.github/workflows/rate-limits-backend-release.yml +++ b/.github/workflows/rate-limits-backend-release.yml @@ -32,7 +32,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers" diff --git a/.github/workflows/release-testing.yml b/.github/workflows/release-testing.yml index b415025eb2fd..eff8a1a5f1a0 100644 --- a/.github/workflows/release-testing.yml +++ b/.github/workflows/release-testing.yml @@ -35,7 +35,7 @@ jobs: group: dm1 labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 180 diff --git a/.github/workflows/rosetta-release.yml b/.github/workflows/rosetta-release.yml index cae49e54a702..618ca055b9f5 100644 --- a/.github/workflows/rosetta-release.yml +++ b/.github/workflows/rosetta-release.yml @@ -22,7 +22,7 @@ jobs: runs-on: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" environment: DockerHub diff --git a/.github/workflows/salt-sharing-canister-release.yml b/.github/workflows/salt-sharing-canister-release.yml index 3a2a6895db4f..6e3b4c4c7d48 100644 --- a/.github/workflows/salt-sharing-canister-release.yml +++ b/.github/workflows/salt-sharing-canister-release.yml @@ -32,7 +32,7 @@ jobs: labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers" diff --git a/.github/workflows/schedule-daily.yml b/.github/workflows/schedule-daily.yml index 5a3d9cf55215..914f995e636f 100644 --- a/.github/workflows/schedule-daily.yml +++ b/.github/workflows/schedule-daily.yml @@ -14,7 +14,7 @@ jobs: runs-on: &dind-large-setup labels: dind-large container: &container-setup - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/schedule-rust-bench.yml b/.github/workflows/schedule-rust-bench.yml index 0158d3469974..5269e9d27a6e 100644 --- a/.github/workflows/schedule-rust-bench.yml +++ b/.github/workflows/schedule-rust-bench.yml @@ -24,7 +24,7 @@ jobs: # see linux-x86-64 runner group labels: rust-benchmarks container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 # running on bare metal machine using ubuntu user options: --user ubuntu --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 720 # 12 hours diff --git a/.github/workflows/system-tests-benchmarks-nightly.yml b/.github/workflows/system-tests-benchmarks-nightly.yml index d411e50f8eaf..5bef21cafb26 100644 --- a/.github/workflows/system-tests-benchmarks-nightly.yml +++ b/.github/workflows/system-tests-benchmarks-nightly.yml @@ -17,7 +17,7 @@ jobs: group: dm1 labels: dind-large container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host --mount type=tmpfs,target="/tmp/containers" timeout-minutes: 480 diff --git a/.github/workflows/update-mainnet-canister-revisions.yaml b/.github/workflows/update-mainnet-canister-revisions.yaml index 915f48d98080..dd145283e948 100644 --- a/.github/workflows/update-mainnet-canister-revisions.yaml +++ b/.github/workflows/update-mainnet-canister-revisions.yaml @@ -25,7 +25,7 @@ jobs: labels: dind-small environment: CREATE_PR container: - image: ghcr.io/dfinity/ic-build@sha256:688586207f0119e428d56b710538a610533f5cb400ca527baabc0b122536ce6e + image: ghcr.io/dfinity/ic-build@sha256:b2146fafb387bbb8eb325a6e4c49c76678d001d14016482dd5d9c4c0cb0d8d63 options: >- -e NODE_NAME --privileged --cgroupns host -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info --mount type=tmpfs,target="/tmp/containers" env: diff --git a/ci/container/TAG b/ci/container/TAG index c28db26d9038..846d57438511 100644 --- a/ci/container/TAG +++ b/ci/container/TAG @@ -1 +1 @@ -85500ab8d8fa12c3fe8dd2930023d65b9184b3f19e3789d22a9b6d282c160556 +f96c20ebb316dcc7dd4e537b40fd2b88a1d2fa43cf2ba80c243076c23ab8e136