diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh new file mode 100755 index 00000000..74180026 --- /dev/null +++ b/.github/scripts/provision-kvm-host.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Provision a GitHub-hosted ubuntu runner as a single-node Firecracker host +# for the `:integration` E2E suite. Mirrors docs/cookbook/install.md; keep +# the two in sync. Assumes: passwordless sudo, repo compiled (the firecracker +# and suidhelper install tasks are mix tasks), MIX_ENV matching the test run. +# +# Deliberate CI-only deltas from install.md — do NOT remove these in a future +# sync pass: +# - the 0666 udev kvm rule (install.md assumes a real host with a `kvm` +# group an operator is added to; the ephemeral runner has neither) +# - the `[budget]` stanza raising cpu_max_load to 4.0 (this host is a +# dedicated ephemeral runner, so the default's contention guard against +# other workloads doesn't apply, and compile/provision load would +# otherwise trip :no_capacity) +# - pre-creating /srv/hyper/layers (install.md documents this too, but it's +# load-bearing here since nothing else primes it before the suite boots +# the node) +# - DEBIAN_FRONTEND=noninteractive (install.md is written for an interactive +# operator session; CI has no tty to prompt on) +set -euo pipefail + +export DEBIAN_FRONTEND=noninteractive + +# The jailed VM uid (900000+) opens /dev/kvm; 0666 is GitHub's own documented +# rule for KVM access on hosted runners. +echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null +sudo udevadm control --reload-rules +sudo udevadm trigger --name-match=kvm + +sudo apt-get update +sudo apt-get install -y \ + coreutils e2fsprogs libc-bin lvm2 skopeo util-linux \ + "linux-modules-extra-$(uname -r)" + +# -a is load-bearing: without it modprobe reads the 2nd+ names as module +# PARAMETERS of the first, and the load fails. +sudo modprobe -av dm_snapshot dm_thin_pool loop +targets="$(sudo dmsetup targets)" +echo "dmsetup targets: ${targets}" +grep -q thin-pool <<<"${targets}" || { echo "ERROR: thin-pool dm target missing" >&2; exit 1; } + +sudo mkdir -p /etc/hyper +sudo tee /etc/hyper/config.toml >/dev/null <<'EOF' +work_dir = "/srv/hyper" + +[tools] +firecracker = "/opt/firecracker/firecracker" +jailer = "/opt/firecracker/jailer" + +[jails] +uid_gid_range = [900000, 999999] +cgroup = "hyper" + +# The runner is a dedicated ephemeral host: the soft budget's default +# cpu_max_load (0.8) refuses VMs with :no_capacity while compile/provision +# load is still decaying, and nothing else runs here worth protecting. +[budget] +cpu_max_load = 4.0 +EOF +sudo chown root:root /etc/hyper/config.toml +sudo chmod 0644 /etc/hyper/config.toml + +sudo mkdir -p /sys/fs/cgroup/hyper +echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control >/dev/null + +# The BEAM (the `runner` user — Hyper refuses to run as root) owns work_dir. +# layers/ must pre-exist: boot validation (Layer.Repo.test_system) checks it, +# and the node only creates it lazily on first image load. +sudo mkdir -p /srv/hyper +sudo chown "$(id -u):$(id -g)" /srv/hyper +mkdir -p /srv/hyper/layers + +# firecracker.install writes into /opt/firecracker as the invoking user, then +# the helper requires both binaries root-owned and not group/world-writable. +sudo mkdir -p /opt/firecracker +sudo chown "$(id -u)" /opt/firecracker +mix firecracker.install +sudo chown root:root /opt/firecracker/firecracker /opt/firecracker/jailer +sudo chmod 0755 /opt/firecracker/firecracker /opt/firecracker/jailer + +# With passwordless sudo the task installs setuid-root itself. +mix suidhelper.install +[ -u /usr/local/bin/hyper-suidhelper ] || { echo "ERROR: hyper-suidhelper missing or not setuid-root" >&2; exit 1; } + +echo "provisioned: kvm=$(ls -l /dev/kvm)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13e5895d..69cf4a92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,6 +254,115 @@ jobs: sudo -E env "PATH=$PATH" \ cargo nextest run --features insecure_test_seams -E 'test(/_as_root$/)' + integration: + name: Integration (KVM E2E) + # ubuntu-latest x64 exposes /dev/kvm (all Linux runners with 2+ vCPUs, + # per GitHub's Apr 2024 changelog); arm64 runners do NOT — keep this x64. + runs-on: ubuntu-latest + # Worst case: two 10-minute tests (vm_lifecycle + crash_recovery) plus + # ~4 minutes of provisioning/compile/setup. 25 could cancel the job right + # as it finishes, and `!cancelled()` steps then skip result uploads — + # exactly when flake history matters most. + timeout-minutes: 35 + env: + MIX_ENV: test + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v7 + + # Fail fast and loudly if GitHub ever withdraws nested virt, rather + # than timing out inside a VM boot. + - name: Assert KVM is available + run: '[ -e /dev/kvm ]' + + - uses: erlef/setup-beam@v1 + id: beam + with: + elixir-version: "1.20" + otp-version: "28" + + # Separate cache key from the elixir job: this _build is test-env only + # and the two jobs would otherwise race the same cache entry. + - name: Cache deps and _build + uses: actions/cache@v6 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-integration-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix-integration-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}- + + - name: Install dependencies + run: mix deps.get + + # protoc + protoc-gen-elixir drive the :grpc_gen Mix compiler (same as + # the elixir job). + - name: Install protoc + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install protoc-gen-elixir + run: mix escript.install hex protobuf 0.17.0 --force + + - name: Install musl target for the guest-agent build + run: rustup target add x86_64-unknown-linux-musl + + - name: Cache rust builds + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + native/guest-agent + native/suidhelper + + - name: Compile (warnings as errors) + run: mix compile --warnings-as-errors + + # After compile: firecracker.install / suidhelper.install are mix tasks, + # and the helper's checksum stamp must come from this build. + - name: Provision Firecracker host + run: .github/scripts/provision-kvm-host.sh + + - name: Create and migrate test DB + run: | + mix ecto.create -r Hyper.Img.Db.Repo + mix ecto.migrate -r Hyper.Img.Db.Repo + + # No --no-start: the app boots its real supervision tree, which runs + # Hyper.Node.test_system/0 host validation before any test executes. + # Multiple --only flags OR together: this runs ONLY the gated tags. + - name: Integration + external tests + run: mix test --only integration --only external --warnings-as-errors + + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: _build/test/junit.xml + flags: integration + report_type: test_results + + # test-results.yml globs artifacts/**/*.xml, so this artifact joins the + # unified Test Results check automatically. + - name: Upload integration test results artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: integration-test-results + path: _build/test/junit.xml + # Uploads the triggering event payload so the workflow_run publisher # (test-results.yml) can map results back to the originating PR -- required # for PR comments on fork PRs. Tiny job, always runs. diff --git a/AGENTS.md b/AGENTS.md index ed34a468..ffb1020a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,12 +23,23 @@ mix check # THE gate. Must pass before any PR. Runs, in order: # dialyzer (strict; @specs required) mix test # Elixir suite (needs Postgres for DB tests) mix test test/unit test/controls # pure tests, no Postgres/Firecracker needed +mix test --only integration --only external + # live E2E: boots real Firecracker VMs. Needs a provisioned + # KVM host (docs/cookbook/install.md) + passwordless sudo. + # CI runs this in ci.yml's `integration` job on ubuntu-latest. cargo nextest run # Rust suite (run inside native/suidhelper/) ``` Pure tests under `test/unit` and `test/controls` need neither Postgres nor Firecracker. DB-backed tests need `mix ecto.create && mix ecto.migrate` first. +The `:integration` suite (`test/e2e/`) is the live Firecracker E2E: it starts +the full supervision tree (never combine with `--no-start`) and drives real +VMs. `:external` tests pull real OCI images. Both are excluded by default and +run in CI's `integration` job, which provisions the GitHub-hosted runner via +`.github/scripts/provision-kvm-host.sh` — keep that script in lockstep with +`docs/cookbook/install.md`. + ## Layout - `lib/` — Elixir source. Tests in `test/` mirror this tree. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 317bc586..06d31e88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,6 +113,8 @@ happy path is often enough. Do **not** add mechanical, one-assertion-per-getter tests that exist only to inflate the count -- they will be asked to be removed. Match the style of the tests already in `test/`. +The `:integration` and `:external` test suites are excluded by default. The `:integration` suite (`test/e2e/`) contains live Firecracker E2E tests that boot real VMs and require a provisioned KVM host (see `docs/cookbook/install.md`) and passwordless sudo. The `:external` suite pulls real OCI images. Both run automatically in CI's `integration` job on every PR, so you do not need to run them locally unless you are developing them -- `mix check` runs the default test set, which is sufficient for most contributions. + ## Generated code The Firecracker API bindings under diff --git a/config/runtime.exs b/config/runtime.exs index 6df3f4e9..93b48220 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1,22 +1,11 @@ import Config -# Per-node resource budget. Lives in runtime config because it builds `Unit.*` -# values, which are only loadable once the app's modules are on the code path. -config :hyper, Hyper.Cfg.Budget, - mem_max: Unit.Information.gib(4), - disk_max: Unit.Information.gib(4), - cpu_max_load: 0.8, - cpu_max_cap: 4.0, - disk_bw_cap: Unit.Bandwidth.gibps(1), - disk_bw_max_load: 0.8, - net_bw_cap: Unit.Bandwidth.gibps(1), - net_bw_max_load: 0.8 - # Operator overrides from a well-known location. An optional Elixir config file # at /etc/hyper/config.exs (override the path with HYPER_CONFIG) is merged in -# last, so its values win over every default set above. An absent file is a -# no-op — the normal case in dev and CI. Skipped under :test so the suite never -# reads host state. +# as app env, which every `Hyper.Cfg.*` module treats as the highest-priority +# layer — above the `[budget]`-style TOML tables and above built-in defaults. +# An absent file is a no-op — the normal case in dev and CI. Skipped under +# :test so the suite never reads host state. # # OpenTelemetry exporter wiring is resolved through Hyper.Cfg.Otel so that the # operator's `config :hyper, Hyper.Cfg.Otel, ...` stanza (if present) takes diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md index 5521f5ca..84a91b03 100644 --- a/docs/cookbook/config.md +++ b/docs/cookbook/config.md @@ -185,18 +185,19 @@ headers = { "authorization" = "Bearer YOUR_TOKEN" } ## Budget Configuration Hyper allows you to control the absolute maximal budgets that are available to -all VMs on a particular node. Every field is required except `cpu_max_cap`. - -| Config Key | `config.exs` | `config.toml` | Default | Notes | -| ------------------ | ------------------- | ------------------- | ------- | ----- | -| `mem_max` | `.mem_max` | `.mem_max` | - | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of RAM the node offers VMs. Must not exceed available system memory. | -| `disk_max` | `.disk_max` | `.disk_max` | - | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of disk the node offers VMs. Must not exceed available system disk. | -| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): instantaneous CPU load threshold, a fraction `0.0`–`1.0` of the cap. | -| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | `nil` | [$\beta$ (soft) budget](./architecture.md#budgets): soft CPU capacity in cores (a float). Optional. | -| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | - | [$\beta$ (soft) budget](./architecture.md#budgets): soft disk-bandwidth capacity ([unit](#unit)). | -| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): disk-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | -| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ (soft) budget](./architecture.md#budgets): soft network-bandwidth capacity ([unit](#unit)). | -| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): network-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | +all VMs on a particular node. Every field has a built-in default; set only the +keys you need to change. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------------ | ------------------- | ------------------- | --------- | ----- | +| `mem_max` | `.mem_max` | `.mem_max` | `4 GiB` | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of RAM the node offers VMs. Must not exceed available system memory. | +| `disk_max` | `.disk_max` | `.disk_max` | `4 GiB` | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of disk the node offers VMs. Must not exceed available system disk. | +| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | `0.8` | [$\beta$ (soft) budget](./architecture.md#budgets): instantaneous CPU load threshold, a fraction `0.0`–`1.0` of the cap. | +| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | `4.0` | [$\beta$ (soft) budget](./architecture.md#budgets): soft CPU capacity in cores (a float). Optional — an explicit `nil` in `config.exs` disables the cap. | +| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | `1 GiBps` | [$\beta$ (soft) budget](./architecture.md#budgets): soft disk-bandwidth capacity ([unit](#unit)). | +| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | `0.8` | [$\beta$ (soft) budget](./architecture.md#budgets): disk-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | +| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | `1 GiBps` | [$\beta$ (soft) budget](./architecture.md#budgets): soft network-bandwidth capacity ([unit](#unit)). | +| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | `0.8` | [$\beta$ (soft) budget](./architecture.md#budgets): network-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | ### `config.exs` diff --git a/docs/cookbook/install.md b/docs/cookbook/install.md index 1116ea69..0acdaf1a 100644 --- a/docs/cookbook/install.md +++ b/docs/cookbook/install.md @@ -83,7 +83,7 @@ Hyper relies on `dm-snapshot` and `dm-thin` to build COW filesystems. Load the modules and confirm the targets are present: ```sh -sudo modprobe dm_snapshot dm_thin_pool loop +sudo modprobe -a dm_snapshot dm_thin_pool loop sudo dmsetup targets # must list snapshot, thin, and thin-pool ``` @@ -237,11 +237,14 @@ and the ones it deliberately does **not**. The node builds its entire on-disk tree (`jails`, `socks`, `scratch`, `layers`, `redist`) under `work_dir` (from `/etc/hyper/config.toml`, default `/srv/hyper`) -**as this user**. It must therefore own that directory: +**as this user**. It must therefore own that directory. Boot validation +(`Hyper.Node.Layer.Repo.test_system/0`) refuses to start unless the `layers` +subdirectory already exists — the node only creates it lazily on first image +load, so pre-create it now: ```sh -sudo mkdir -p /srv/hyper -sudo chown hyper:hyper /srv/hyper +sudo mkdir -p /srv/hyper/layers +sudo chown -R hyper:hyper /srv/hyper ``` ## Installation diff --git a/lib/hyper/cfg/budget.ex b/lib/hyper/cfg/budget.ex index 2d01799e..91dcb812 100644 --- a/lib/hyper/cfg/budget.ex +++ b/lib/hyper/cfg/budget.ex @@ -1,9 +1,11 @@ defmodule Hyper.Cfg.Budget do @moduledoc """ This node's resource budget. Each field reads from `config.exs` - (`config :hyper, Hyper.Cfg.Budget, ...`), then the `[budget]` table in - `/etc/hyper/config.toml`, then its default. `Unit.*` quantities may be given - as Elixir terms in `config.exs` or as strings (`"4GiB"`, `"1GiBps"`) in TOML. + (`config :hyper, Hyper.Cfg.Budget, ...`, typically set via the operator + override file `/etc/hyper/config.exs`), then the `[budget]` table in + `/etc/hyper/config.toml`, then its built-in default. `Unit.*` quantities may + be given as Elixir terms in `config.exs` or as strings (`"4GiB"`, `"1GiBps"`) + in TOML. """ import Hyper.Cfg, only: [fetch_cfg: 1] @@ -29,20 +31,33 @@ defmodule Hyper.Cfg.Budget do :net_bw_max_load ] + @default_mem_max Unit.Information.gib(4) + @default_disk_max Unit.Information.gib(4) + @default_cpu_max_load 0.8 + @default_cpu_max_cap 4.0 + @default_disk_bw_cap Unit.Bandwidth.gibps(1) + @default_disk_bw_max_load 0.8 + @default_net_bw_cap Unit.Bandwidth.gibps(1) + @default_net_bw_max_load 0.8 + @spec load :: {:ok, t()} | {:error, term()} def load do - with {:ok, mem_max} <- information(:mem_max, "budget.mem_max"), - {:ok, disk_max} <- information(:disk_max, "budget.disk_max"), - {:ok, cpu_max_load} <- number(:cpu_max_load, "budget.cpu_max_load"), - {:ok, disk_bw_cap} <- bandwidth(:disk_bw_cap, "budget.disk_bw_cap"), - {:ok, disk_bw_max_load} <- number(:disk_bw_max_load, "budget.disk_bw_max_load"), - {:ok, net_bw_cap} <- bandwidth(:net_bw_cap, "budget.net_bw_cap"), - {:ok, net_bw_max_load} <- number(:net_bw_max_load, "budget.net_bw_max_load") do + with {:ok, mem_max} <- information(:mem_max, "budget.mem_max", @default_mem_max), + {:ok, disk_max} <- information(:disk_max, "budget.disk_max", @default_disk_max), + {:ok, cpu_max_load} <- + number(:cpu_max_load, "budget.cpu_max_load", @default_cpu_max_load), + {:ok, disk_bw_cap} <- + bandwidth(:disk_bw_cap, "budget.disk_bw_cap", @default_disk_bw_cap), + {:ok, disk_bw_max_load} <- + number(:disk_bw_max_load, "budget.disk_bw_max_load", @default_disk_bw_max_load), + {:ok, net_bw_cap} <- bandwidth(:net_bw_cap, "budget.net_bw_cap", @default_net_bw_cap), + {:ok, net_bw_max_load} <- + number(:net_bw_max_load, "budget.net_bw_max_load", @default_net_bw_max_load) do config = %__MODULE__{ mem_max: mem_max, disk_max: disk_max, cpu_max_load: cpu_max_load, - cpu_max_cap: optional_number(:cpu_max_cap, "budget.cpu_max_cap"), + cpu_max_cap: optional_number(:cpu_max_cap, "budget.cpu_max_cap", @default_cpu_max_cap), disk_bw_cap: disk_bw_cap, disk_bw_max_load: disk_bw_max_load, net_bw_cap: net_bw_cap, @@ -57,40 +72,46 @@ defmodule Hyper.Cfg.Budget do @spec get :: t() def get, do: :persistent_term.get(__MODULE__) - @spec information(atom(), String.t()) :: {:ok, Unit.Information.t()} | {:error, term()} - defp information(key, toml) do - with {:ok, v} <- required(key, toml) do + @spec information(atom(), String.t(), Unit.Information.t()) :: + {:ok, Unit.Information.t()} | {:error, term()} + defp information(key, toml, default) do + with {:ok, v} <- required(key, toml, default) do coerce(v, &Unit.Information.parse/1, Unit.Information, key) end end - @spec bandwidth(atom(), String.t()) :: {:ok, Unit.Bandwidth.t()} | {:error, term()} - defp bandwidth(key, toml) do - with {:ok, v} <- required(key, toml) do + @spec bandwidth(atom(), String.t(), Unit.Bandwidth.t()) :: + {:ok, Unit.Bandwidth.t()} | {:error, term()} + defp bandwidth(key, toml, default) do + with {:ok, v} <- required(key, toml, default) do coerce(v, &Unit.Bandwidth.parse/1, Unit.Bandwidth, key) end end - @spec number(atom(), String.t()) :: {:ok, number()} | {:error, term()} - defp number(key, toml) do - case required(key, toml) do + @spec number(atom(), String.t(), number()) :: {:ok, number()} | {:error, term()} + defp number(key, toml, default) do + case required(key, toml, default) do {:ok, n} when is_number(n) -> {:ok, n} {:ok, other} -> {:error, {:not_a_number, key, other}} {:error, _} = e -> e end end - @spec optional_number(atom(), String.t()) :: number() | nil - defp optional_number(key, toml) do - case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do + # cpu_max_cap stays nilable: an operator may explicitly set + # `cpu_max_cap: nil` in config.exs to disable the cap outright, so a + # non-number resolution (including that explicit nil) falls through to nil + # rather than failing the whole load. + @spec optional_number(atom(), String.t(), number()) :: number() | nil + defp optional_number(key, toml, default) do + case fetch_cfg(runtime: {__MODULE__, key}, toml: toml, default: default) do {:ok, n} when is_number(n) -> n _ -> nil end end - @spec required(atom(), String.t()) :: {:ok, term()} | {:error, term()} - defp required(key, toml) do - case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do + @spec required(atom(), String.t(), term()) :: {:ok, term()} | {:error, term()} + defp required(key, toml, default) do + case fetch_cfg(runtime: {__MODULE__, key}, toml: toml, default: default) do {:ok, v} -> {:ok, v} :error -> {:error, {:missing, key}} end diff --git a/test/e2e/crash_recovery_test.exs b/test/e2e/crash_recovery_test.exs new file mode 100644 index 00000000..8713539d --- /dev/null +++ b/test/e2e/crash_recovery_test.exs @@ -0,0 +1,90 @@ +defmodule Hyper.E2e.CrashRecoveryTest do + @moduledoc """ + Crash-recovery contract: SIGKILLing the jailed Firecracker process behind a + running VM must NOT kill the VM. `Core`'s `:one_for_all` deliberately + cold-boots the daemon + controller pair on the same mutable rootfs + (lib/hyper/node/fire_vmm/core.ex), so the guest comes back and answers + `exec` again on the SAME writable dm volume — proven not just by the dm + device name still existing post-recovery (a destroy+recreate under the same + deterministic name would also pass that check) but by a file written to the + guest's rootfs before the crash still being there after recovery, which only + survives if the underlying block device itself was preserved. Reclaim + happens only on explicit stop: `stop_image_vm/1` must then remove the + volume — the crash/recover cycle must not leak the device or the routing + entry past the stop. + + Runs only under `--only integration` on a provisioned host (see + VmLifecycleTest for the environment contract). + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + + test "SIGKILL of firecracker cold-boots the VM; explicit stop reclaims the volume" do + assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + + # :micro, not the :base default — :base asks for 32 GiB of disk budget, + # which the default node budget (4 GiB) refuses with :no_capacity on the + # small CI runner. + assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + # A failed assertion anywhere below must not leak a live VM into other + # tests in the same run; stop_image_vm/1 treats :not_found as :ok, so + # this is safe alongside the explicit stop the test itself performs. + on_exit(fn -> Hyper.Node.stop_image_vm(vm) end) + vm_id = Hyper.id(vm) + assert vm_id, "Hyper.id/1 returned nil for a freshly-created VM" + rw_dev = Hyper.Node.Img.Mutable.dm_name(vm_id) + + # Prove the VM is fully live first — otherwise the kill would race boot + # and the test could pass without exercising crash recovery at all. The + # second guest of a run has come up slower than the first on nested-virt + # CI runners, hence the extended deadline. The marker write doubles as the + # volume-identity proof: `sync` forces it past the page cache (which dies + # with firecracker) and onto the dm volume itself, so it can only survive + # the crash if that volume — not just a same-named replacement — does. + assert {:ok, %{exit_code: 0}} = + await_exec( + vm, + ["/bin/sh", "-c", "echo persisted > /marker && sync"], + :timer.minutes(3) + ) + + assert MapSet.member?(dm_devices(), rw_dev) + + # The jailer/firecracker cmdline carries the vm_id (--id). The [v]... + # bracket regex still matches it, but not this command's own sudo wrapper — + # whose cmdline contains the pattern text, not the raw id (a bare -f vm_id + # SIGKILLs its own sudo and System.cmd reports 137 instead of 0). + kill_pattern = "[" <> String.first(vm_id) <> "]" <> String.slice(vm_id, 1..-1//1) + assert {_, 0} = System.cmd("sudo", ["pkill", "-9", "-f", kill_pattern]) + + # Self-heal: Core cold-boots the daemon/controller pair; the relay child + # is untouched, so exec reaches the rebooted guest once its agent is back. + assert {:ok, %{stdout: marker, exit_code: 0}} = + await_exec(vm, ["/bin/cat", "/marker"], :timer.minutes(3)), + "VM did not recover from a firecracker SIGKILL" + + assert marker =~ "persisted", + "recovered VM lost pre-crash writes — not the same volume" + + # The recovered VM still runs on ITS OWN volume — the crash must not have + # torn it down or swapped it. Supplementary to the marker-file assertion + # above: this only proves the dm device name persisted, which a + # destroy+recreate under the same deterministic name would also satisfy. + assert MapSet.member?(dm_devices(), rw_dev), + "writable dm volume #{rw_dev} vanished across the crash/recovery cycle" + + assert :ok = Hyper.Node.stop_image_vm(vm) + + assert poll_until(fn -> not MapSet.member?(dm_devices(), rw_dev) end, :timer.seconds(90)), + "writable dm volume #{rw_dev} leaked after stop_image_vm" + + assert poll_until(fn -> Hyper.whereis(vm_id) == nil end, :timer.minutes(1)), + "routing entry for #{vm_id} survived stop_image_vm" + end +end diff --git a/test/e2e/vm_lifecycle_test.exs b/test/e2e/vm_lifecycle_test.exs new file mode 100644 index 00000000..9b020d2f --- /dev/null +++ b/test/e2e/vm_lifecycle_test.exs @@ -0,0 +1,54 @@ +defmodule Hyper.E2e.VmLifecycleTest do + @moduledoc """ + Live end-to-end contract of the VM lifecycle on a provisioned host: + + - `OciLoader.load/1` of a real registry image yields a bootable image id; + - `create_vm/1` boots a guest whose per-VM writable dm volume + (`Mutable.dm_name/1`) exists while the VM runs; + - the guest agent answers `exec` with the command's captured output; + - `stop_image_vm/1` reclaims the writable volume (no dm leak). + + Runs only under `--only integration` / `--include integration` on a host + provisioned per docs/cookbook/install.md (CI: the `integration` job). + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + # public.ecr.aws mirrors library images without Docker Hub's per-IP pull + # limits, which shared GHA egress IPs routinely exhaust. + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + + test "load -> create_vm -> exec -> stop reclaims the VM's dm volume" do + assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + + # :micro, not the :base default — :base asks for 32 GiB of disk budget, + # which the default node budget (4 GiB) refuses with :no_capacity on the + # small CI runner. + assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + # A failed assertion must not leak a live VM into the next E2E test in + # the same run; stop_image_vm/1 is idempotent, so this is safe alongside + # the explicit stop below (which is itself the behavior under test). + on_exit(fn -> Hyper.Node.stop_image_vm(vm) end) + + vm_id = Hyper.id(vm) + assert vm_id, "Hyper.id/1 returned nil for a freshly-created VM" + rw_dev = Hyper.Node.Img.Mutable.dm_name(vm_id) + + assert MapSet.member?(dm_devices(), rw_dev), + "expected writable dm volume #{rw_dev} while the VM is running" + + assert {:ok, %{stdout: out, exit_code: 0}} = + await_exec(vm, ["/bin/echo", "hello from guest"]) + + assert out =~ "hello from guest" + + assert :ok = Hyper.Node.stop_image_vm(vm) + + assert poll_until(fn -> not MapSet.member?(dm_devices(), rw_dev) end, :timer.seconds(90)), + "writable dm volume #{rw_dev} leaked after stop_image_vm" + end +end diff --git a/test/hyper/cfg/budget_test.exs b/test/hyper/cfg/budget_test.exs new file mode 100644 index 00000000..881efa94 --- /dev/null +++ b/test/hyper/cfg/budget_test.exs @@ -0,0 +1,61 @@ +defmodule Hyper.Cfg.BudgetTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Budget + alias Hyper.Cfg.Toml + + setup do + Application.delete_env(:hyper, Budget) + Toml.put_cache(%{}) + + on_exit(fn -> + Application.delete_env(:hyper, Budget) + Toml.reload() + # load/0 caches the struct; erase it so no later test observes this + # test's budget by accident. + :persistent_term.erase(Budget) + end) + + :ok + end + + test "load/0 with no app env and no TOML table returns the documented built-in defaults" do + assert {:ok, config} = Budget.load() + + assert config == %Budget{ + mem_max: Unit.Information.gib(4), + disk_max: Unit.Information.gib(4), + cpu_max_load: 0.8, + cpu_max_cap: 4.0, + disk_bw_cap: Unit.Bandwidth.gibps(1), + disk_bw_max_load: 0.8, + net_bw_cap: Unit.Bandwidth.gibps(1), + net_bw_max_load: 0.8 + } + end + + test "a [budget] TOML table overrides the built-in default" do + Toml.put_cache(%{"budget" => %{"cpu_max_load" => 0.5, "mem_max" => "2GiB"}}) + + assert {:ok, config} = Budget.load() + assert config.cpu_max_load == 0.5 + assert config.mem_max == Unit.Information.gib(2) + # Fields absent from the TOML table still fall back to their defaults. + assert config.disk_max == Unit.Information.gib(4) + end + + test "an app-env override (config.exs) wins over a conflicting TOML value" do + Toml.put_cache(%{"budget" => %{"cpu_max_load" => 0.5}}) + Application.put_env(:hyper, Budget, cpu_max_load: 0.9) + + assert {:ok, config} = Budget.load() + assert config.cpu_max_load == 0.9 + end + + test "cpu_max_cap can be explicitly disabled via app env, overriding its default" do + Application.put_env(:hyper, Budget, cpu_max_cap: nil) + + assert {:ok, config} = Budget.load() + assert config.cpu_max_cap == nil + end +end diff --git a/test/support/e2e.ex b/test/support/e2e.ex new file mode 100644 index 00000000..091c480e --- /dev/null +++ b/test/support/e2e.ex @@ -0,0 +1,75 @@ +defmodule Hyper.E2e do + @moduledoc """ + Helpers for the `:integration` E2E suite, which runs against a fully + provisioned single-node Firecracker host (KVM, device-mapper, setuid + helper, Postgres) — in CI, the `integration` job of ci.yml provisions the + runner via .github/scripts/provision-kvm-host.sh. + + Requires passwordless sudo: `dm_devices/0` shells out to `sudo dmsetup ls` + to observe kernel state Hyper itself owns via the setuid helper. + """ + + @doc "Names of all live device-mapper devices, via `sudo dmsetup ls`." + @spec dm_devices() :: MapSet.t(String.t()) + def dm_devices do + {out, 0} = System.cmd("sudo", ["dmsetup", "ls"], stderr_to_stdout: true) + + out + |> String.split("\n", trim: true) + |> Enum.reject(&String.starts_with?(&1, "No devices found")) + |> Enum.map(fn line -> line |> String.split() |> hd() end) + |> MapSet.new() + end + + @doc "Polls `fun` every 500 ms until it returns truthy or `deadline_ms` elapses." + @spec poll_until((-> boolean()), non_neg_integer()) :: boolean() + def poll_until(fun, deadline_ms) do + deadline = System.monotonic_time(:millisecond) + deadline_ms + do_poll(fun, deadline) + end + + defp do_poll(fun, deadline) do + cond do + fun.() -> true + System.monotonic_time(:millisecond) > deadline -> false + true -> Process.sleep(500) && do_poll(fun, deadline) + end + end + + @doc """ + `Hyper.exec/3`, retried while the guest agent is still coming up. + + Boot-race errors that get retried: `:agent_unavailable`, `:timeout`, and + `GRPC.RPCError` — the relay's vsock stream can connect and drop while the + in-guest agent is still starting, which surfaces as a gRPC stream error + rather than `:agent_unavailable`. Anything else returns immediately. + """ + @spec await_exec(pid() | binary(), [String.t()], non_neg_integer()) :: + {:ok, map()} | {:error, term()} + def await_exec(vm, argv, deadline_ms \\ :timer.seconds(60)) do + deadline = System.monotonic_time(:millisecond) + deadline_ms + do_await_exec(vm, argv, deadline) + end + + defp do_await_exec(vm, argv, deadline) do + case Hyper.exec(vm, argv) do + {:error, reason} = err when reason in [:agent_unavailable, :timeout] -> + retry_or_return(err, vm, argv, deadline) + + {:error, %GRPC.RPCError{}} = err -> + retry_or_return(err, vm, argv, deadline) + + other -> + other + end + end + + defp retry_or_return(err, vm, argv, deadline) do + if System.monotonic_time(:millisecond) > deadline do + err + else + Process.sleep(1_000) + do_await_exec(vm, argv, deadline) + end + end +end