From ccd2e537286f04a77ca05283cfd0f15af8b13be2 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:01:54 +0000
Subject: [PATCH 01/22] test(e2e): add Hyper.E2e helpers for the integration
 suite

---
 test/support/e2e.ex | 65 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100644 test/support/e2e.ex

diff --git a/test/support/e2e.ex b/test/support/e2e.ex
new file mode 100644
index 00000000..b3c08286
--- /dev/null
+++ b/test/support/e2e.ex
@@ -0,0 +1,65 @@
+defmodule Hyper.E2e do
+  @moduledoc """
+  Helpers for the `:integration` E2E suite, which runs against a fully
+  provisioned single-node Firecracker host (KVM, device-mapper, setuid
+  helper, Postgres) — in CI, the `integration` job of ci.yml provisions the
+  runner via .github/scripts/provision-kvm-host.sh.
+
+  Requires passwordless sudo: `dm_devices/0` shells out to `sudo dmsetup ls`
+  to observe kernel state Hyper itself owns via the setuid helper.
+  """
+
+  @doc "Names of all live device-mapper devices, via `sudo dmsetup ls`."
+  @spec dm_devices() :: MapSet.t(String.t())
+  def dm_devices do
+    {out, 0} = System.cmd("sudo", ["dmsetup", "ls"], stderr_to_stdout: true)
+
+    out
+    |> String.split("\n", trim: true)
+    |> Enum.reject(&String.starts_with?(&1, "No devices found"))
+    |> Enum.map(fn line -> line |> String.split() |> hd() end)
+    |> MapSet.new()
+  end
+
+  @doc "Polls `fun` every 500 ms until it returns truthy or `deadline_ms` elapses."
+  @spec poll_until((-> boolean()), non_neg_integer()) :: boolean()
+  def poll_until(fun, deadline_ms) do
+    deadline = System.monotonic_time(:millisecond) + deadline_ms
+    do_poll(fun, deadline)
+  end
+
+  defp do_poll(fun, deadline) do
+    cond do
+      fun.() -> true
+      System.monotonic_time(:millisecond) > deadline -> false
+      true -> Process.sleep(500) && do_poll(fun, deadline)
+    end
+  end
+
+  @doc """
+  `Hyper.exec/3`, retried while the guest agent is still coming up
+  (`:agent_unavailable` / `:timeout` are the boot-race errors; anything else
+  returns immediately).
+  """
+  @spec await_exec(pid() | binary(), [String.t()], non_neg_integer()) ::
+          {:ok, map()} | {:error, term()}
+  def await_exec(vm, argv, deadline_ms \\ :timer.seconds(60)) do
+    deadline = System.monotonic_time(:millisecond) + deadline_ms
+    do_await_exec(vm, argv, deadline)
+  end
+
+  defp do_await_exec(vm, argv, deadline) do
+    case Hyper.exec(vm, argv) do
+      {:error, reason} when reason in [:agent_unavailable, :timeout] ->
+        if System.monotonic_time(:millisecond) > deadline do
+          {:error, reason}
+        else
+          Process.sleep(1_000)
+          do_await_exec(vm, argv, deadline)
+        end
+
+      other ->
+        other
+    end
+  end
+end

From 26fe188e50117db204b7fd4d8273a5a3ab1b6600 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:08:31 +0000
Subject: [PATCH 02/22] test(e2e): live VM lifecycle E2E (load, boot, exec,
 clean stop)

---
 test/e2e/vm_lifecycle_test.exs | 45 ++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 test/e2e/vm_lifecycle_test.exs

diff --git a/test/e2e/vm_lifecycle_test.exs b/test/e2e/vm_lifecycle_test.exs
new file mode 100644
index 00000000..020c4d3c
--- /dev/null
+++ b/test/e2e/vm_lifecycle_test.exs
@@ -0,0 +1,45 @@
+defmodule Hyper.E2e.VmLifecycleTest do
+  @moduledoc """
+  Live end-to-end contract of the VM lifecycle on a provisioned host:
+
+  - `OciLoader.load/1` of a real registry image yields a bootable image id;
+  - `create_vm/1` boots a guest whose per-VM writable dm volume
+    (`Mutable.dm_name/1`) exists while the VM runs;
+  - the guest agent answers `exec` with the command's captured output;
+  - `stop_image_vm/1` reclaims the writable volume (no dm leak).
+
+  Runs only under `--only integration` / `--include integration` on a host
+  provisioned per docs/cookbook/install.md (CI: the `integration` job).
+  """
+  use ExUnit.Case, async: false
+
+  import Hyper.E2e
+
+  @moduletag :integration
+  @moduletag timeout: :timer.minutes(10)
+
+  # public.ecr.aws mirrors library images without Docker Hub's per-IP pull
+  # limits, which shared GHA egress IPs routinely exhaust.
+  @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19")
+
+  test "load -> create_vm -> exec -> stop reclaims the VM's dm volume" do
+    assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image)
+
+    assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id})
+    vm_id = Hyper.id(vm)
+    rw_dev = Hyper.Node.Img.Mutable.dm_name(vm_id)
+
+    assert MapSet.member?(dm_devices(), rw_dev),
+           "expected writable dm volume #{rw_dev} while the VM is running"
+
+    assert {:ok, %{stdout: out, exit_code: 0}} =
+             await_exec(vm, ["/bin/echo", "hello from guest"])
+
+    assert out =~ "hello from guest"
+
+    assert :ok = Hyper.Node.stop_image_vm(vm)
+
+    assert poll_until(fn -> not MapSet.member?(dm_devices(), rw_dev) end, :timer.seconds(90)),
+           "writable dm volume #{rw_dev} leaked after stop_image_vm"
+  end
+end

From 0c4aa4f7a3afaa97f2cce734233f308e8a1a4ea7 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:13:50 +0000
Subject: [PATCH 03/22] test(e2e): tear down the VM on lifecycle test failure

---
 test/e2e/vm_lifecycle_test.exs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/e2e/vm_lifecycle_test.exs b/test/e2e/vm_lifecycle_test.exs
index 020c4d3c..6386914c 100644
--- a/test/e2e/vm_lifecycle_test.exs
+++ b/test/e2e/vm_lifecycle_test.exs
@@ -26,7 +26,13 @@ defmodule Hyper.E2e.VmLifecycleTest do
     assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image)
 
     assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id})
+    # A failed assertion must not leak a live VM into the next E2E test in
+    # the same run; stop_image_vm/1 is idempotent, so this is safe alongside
+    # the explicit stop below (which is itself the behavior under test).
+    on_exit(fn -> Hyper.Node.stop_image_vm(vm) end)
+
     vm_id = Hyper.id(vm)
+    assert vm_id, "Hyper.id/1 returned nil for a freshly-created VM"
     rw_dev = Hyper.Node.Img.Mutable.dm_name(vm_id)
 
     assert MapSet.member?(dm_devices(), rw_dev),

From dacc99942ac281695a55ae93a12ba537e309179e Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:16:27 +0000
Subject: [PATCH 04/22] test(e2e): crash-reclaim E2E (SIGKILL firecracker,
 assert no dm leak)

---
 test/e2e/crash_teardown_test.exs | 46 ++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 test/e2e/crash_teardown_test.exs

diff --git a/test/e2e/crash_teardown_test.exs b/test/e2e/crash_teardown_test.exs
new file mode 100644
index 00000000..27b1c4de
--- /dev/null
+++ b/test/e2e/crash_teardown_test.exs
@@ -0,0 +1,46 @@
+defmodule Hyper.E2e.CrashTeardownTest do
+  @moduledoc """
+  Crash-reclaim contract: SIGKILLing the jailed Firecracker process behind a
+  running VM must tear the whole VM down and reclaim its writable dm volume —
+  monitors free the uid and mutable layer on :DOWN, and the Reaper (60 s
+  tick, two-strike confirm) is the backstop for anything they miss. Nothing
+  may resurrect the device (the idle-reaper-restart-resurrection regression).
+
+  Runs only under `--only integration` on a provisioned host (see
+  VmLifecycleTest for the environment contract).
+  """
+  use ExUnit.Case, async: false
+
+  import Hyper.E2e
+
+  @moduletag :integration
+  @moduletag timeout: :timer.minutes(10)
+
+  @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19")
+
+  test "SIGKILL of firecracker reclaims the dm volume and the routing entry" do
+    assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image)
+    assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id})
+    on_exit(fn -> Hyper.Node.stop_image_vm(vm) end)
+    vm_id = Hyper.id(vm)
+    assert vm_id, "Hyper.id/1 returned nil for a freshly-created VM"
+    rw_dev = Hyper.Node.Img.Mutable.dm_name(vm_id)
+
+    # Prove the VM is fully live first — otherwise the kill would race boot
+    # and the test could pass without exercising crash reclaim at all.
+    assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"])
+    assert MapSet.member?(dm_devices(), rw_dev)
+
+    # The jailer/firecracker cmdline carries the vm_id (--id); the BEAM's
+    # own cmdline does not, so -f cannot match the test runner itself.
+    assert {_, 0} = System.cmd("sudo", ["pkill", "-9", "-f", vm_id])
+
+    # Monitor-driven teardown is immediate; the Reaper backstop is 60 s ticks
+    # with two-strike confirmation, hence the generous deadline.
+    assert poll_until(fn -> not MapSet.member?(dm_devices(), rw_dev) end, :timer.minutes(4)),
+           "writable dm volume #{rw_dev} survived firecracker SIGKILL"
+
+    assert poll_until(fn -> Hyper.whereis(vm_id) == nil end, :timer.minutes(1)),
+           "routing entry for #{vm_id} survived firecracker SIGKILL"
+  end
+end

From 7f2e91ae98e246a2fb6287469abf67a53a18e59d Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:20:37 +0000
Subject: [PATCH 05/22] test(e2e): explain the crash test's on_exit teardown
 guard

---
 test/e2e/crash_teardown_test.exs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/e2e/crash_teardown_test.exs b/test/e2e/crash_teardown_test.exs
index 27b1c4de..4619c160 100644
--- a/test/e2e/crash_teardown_test.exs
+++ b/test/e2e/crash_teardown_test.exs
@@ -21,6 +21,9 @@ defmodule Hyper.E2e.CrashTeardownTest do
   test "SIGKILL of firecracker reclaims the dm volume and the routing entry" do
     assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image)
     assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id})
+    # A failed assertion before (or after) the SIGKILL must not leak a live VM
+    # into other tests in the same run; stop_image_vm/1 treats :not_found as
+    # :ok, so this is safe even after crash-reclaim already tore the VM down.
     on_exit(fn -> Hyper.Node.stop_image_vm(vm) end)
     vm_id = Hyper.id(vm)
     assert vm_id, "Hyper.id/1 returned nil for a freshly-created VM"

From fd66ae8dcea16eb5f66fba38eb77db9aed17424f Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:22:17 +0000
Subject: [PATCH 06/22] ci: add KVM host provisioning script for the
 integration job

---
 .github/scripts/provision-kvm-host.sh | 57 +++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100755 .github/scripts/provision-kvm-host.sh

diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh
new file mode 100755
index 00000000..192537db
--- /dev/null
+++ b/.github/scripts/provision-kvm-host.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Provision a GitHub-hosted ubuntu runner as a single-node Firecracker host
+# for the `:integration` E2E suite. Mirrors docs/cookbook/install.md; keep
+# the two in sync. Assumes: passwordless sudo, repo compiled (the firecracker
+# and suidhelper install tasks are mix tasks), MIX_ENV matching the test run.
+set -euo pipefail
+
+# The jailed VM uid (900000+) opens /dev/kvm; 0666 is GitHub's own documented
+# rule for KVM access on hosted runners.
+echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+  | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+sudo udevadm control --reload-rules
+sudo udevadm trigger --name-match=kvm
+
+sudo apt-get update
+sudo apt-get install -y \
+  coreutils e2fsprogs libc-bin lvm2 skopeo util-linux \
+  "linux-modules-extra-$(uname -r)"
+
+sudo modprobe dm_snapshot dm_thin_pool loop
+sudo dmsetup targets | grep -q thin-pool
+
+sudo mkdir -p /etc/hyper
+sudo tee /etc/hyper/config.toml >/dev/null <<'EOF'
+work_dir = "/srv/hyper"
+
+[tools]
+firecracker = "/opt/firecracker/firecracker"
+jailer = "/opt/firecracker/jailer"
+
+[jails]
+uid_gid_range = [900000, 999999]
+cgroup = "hyper"
+EOF
+sudo chown root:root /etc/hyper/config.toml
+sudo chmod 0644 /etc/hyper/config.toml
+
+sudo mkdir -p /sys/fs/cgroup/hyper
+echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control
+
+# The BEAM (the `runner` user — Hyper refuses to run as root) owns work_dir.
+sudo mkdir -p /srv/hyper
+sudo chown "$(id -u):$(id -g)" /srv/hyper
+
+# firecracker.install writes into /opt/firecracker as the invoking user, then
+# the helper requires both binaries root-owned and not group/world-writable.
+sudo mkdir -p /opt/firecracker
+sudo chown "$(id -u)" /opt/firecracker
+mix firecracker.install
+sudo chown root:root /opt/firecracker/firecracker /opt/firecracker/jailer
+sudo chmod 0755 /opt/firecracker/firecracker /opt/firecracker/jailer
+
+# With passwordless sudo the task installs setuid-root itself.
+mix suidhelper.install
+[ -u /usr/local/bin/hyper-suidhelper ]
+
+echo "provisioned: kvm=$(ls -l /dev/kvm)"

From 46405081974d0d2c165fc6954a3f2fbd26babfd7 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:26:07 +0000
Subject: [PATCH 07/22] ci: make provisioning apt calls non-interactive

---
 .github/scripts/provision-kvm-host.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh
index 192537db..440b9da6 100755
--- a/.github/scripts/provision-kvm-host.sh
+++ b/.github/scripts/provision-kvm-host.sh
@@ -5,10 +5,12 @@
 # and suidhelper install tasks are mix tasks), MIX_ENV matching the test run.
 set -euo pipefail
 
+export DEBIAN_FRONTEND=noninteractive
+
 # The jailed VM uid (900000+) opens /dev/kvm; 0666 is GitHub's own documented
 # rule for KVM access on hosted runners.
 echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
-  | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+  | sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null
 sudo udevadm control --reload-rules
 sudo udevadm trigger --name-match=kvm
 
@@ -36,7 +38,7 @@ sudo chown root:root /etc/hyper/config.toml
 sudo chmod 0644 /etc/hyper/config.toml
 
 sudo mkdir -p /sys/fs/cgroup/hyper
-echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control
+echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control >/dev/null
 
 # The BEAM (the `runner` user — Hyper refuses to run as root) owns work_dir.
 sudo mkdir -p /srv/hyper

From 39eb1c7a3c693843ea83231e4857cb612d83ced8 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:28:18 +0000
Subject: [PATCH 08/22] ci: run KVM integration + external tests on
 ubuntu-latest

---
 .github/workflows/ci.yml | 105 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 13e5895d..ace2d282 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -254,6 +254,111 @@ jobs:
           sudo -E env "PATH=$PATH" \
             cargo nextest run --features insecure_test_seams -E 'test(/_as_root$/)'
 
+  integration:
+    name: Integration (KVM E2E)
+    # ubuntu-latest x64 exposes /dev/kvm (all Linux runners with 2+ vCPUs,
+    # per GitHub's Apr 2024 changelog); arm64 runners do NOT — keep this x64.
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    env:
+      MIX_ENV: test
+    services:
+      postgres:
+        image: postgres:17-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v7
+
+      # Fail fast and loudly if GitHub ever withdraws nested virt, rather
+      # than timing out inside a VM boot.
+      - name: Assert KVM is available
+        run: '[ -e /dev/kvm ]'
+
+      - uses: erlef/setup-beam@v1
+        id: beam
+        with:
+          elixir-version: "1.20"
+          otp-version: "28"
+
+      # Separate cache key from the elixir job: this _build is test-env only
+      # and the two jobs would otherwise race the same cache entry.
+      - name: Cache deps and _build
+        uses: actions/cache@v6
+        with:
+          path: |
+            deps
+            _build
+          key: ${{ runner.os }}-mix-integration-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-${{ hashFiles('mix.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-mix-integration-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-
+
+      - name: Install dependencies
+        run: mix deps.get
+
+      # protoc + protoc-gen-elixir drive the :grpc_gen Mix compiler (same as
+      # the elixir job).
+      - name: Install protoc
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Install protoc-gen-elixir
+        run: mix escript.install hex protobuf 0.17.0 --force
+
+      - name: Install musl target for the guest-agent build
+        run: rustup target add x86_64-unknown-linux-musl
+
+      - name: Cache rust builds
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            native/guest-agent
+            native/suidhelper
+
+      - name: Compile (warnings as errors)
+        run: mix compile --warnings-as-errors
+
+      # After compile: firecracker.install / suidhelper.install are mix tasks,
+      # and the helper's checksum stamp must come from this build.
+      - name: Provision Firecracker host
+        run: .github/scripts/provision-kvm-host.sh
+
+      - name: Create and migrate test DB
+        run: |
+          mix ecto.create -r Hyper.Img.Db.Repo
+          mix ecto.migrate -r Hyper.Img.Db.Repo
+
+      # No --no-start: the app boots its real supervision tree, which runs
+      # Hyper.Node.test_system/0 host validation before any test executes.
+      # Multiple --only flags OR together: this runs ONLY the gated tags.
+      - name: Integration + external tests
+        run: mix test --only integration --only external --warnings-as-errors
+
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/codecov-action@v7
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: _build/test/junit.xml
+          flags: integration
+          report_type: test_results
+
+      # test-results.yml globs artifacts/**/*.xml, so this artifact joins the
+      # unified Test Results check automatically.
+      - name: Upload integration test results artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: integration-test-results
+          path: _build/test/junit.xml
+
   # Uploads the triggering event payload so the workflow_run publisher
   # (test-results.yml) can map results back to the originating PR -- required
   # for PR comments on fork PRs. Tiny job, always runs.

From 87e0a03c7616b25c652abe38f38b5b3c18136eca Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:36:51 +0000
Subject: [PATCH 09/22] ci: load dm modules with modprobe -a and surface target
 diagnostics

---
 .github/scripts/provision-kvm-host.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh
index 440b9da6..25e5fae4 100755
--- a/.github/scripts/provision-kvm-host.sh
+++ b/.github/scripts/provision-kvm-host.sh
@@ -19,8 +19,12 @@ sudo apt-get install -y \
   coreutils e2fsprogs libc-bin lvm2 skopeo util-linux \
   "linux-modules-extra-$(uname -r)"
 
-sudo modprobe dm_snapshot dm_thin_pool loop
-sudo dmsetup targets | grep -q thin-pool
+# -a is load-bearing: without it modprobe reads the 2nd+ names as module
+# PARAMETERS of the first, and the load fails.
+sudo modprobe -av dm_snapshot dm_thin_pool loop
+targets="$(sudo dmsetup targets)"
+echo "dmsetup targets: ${targets}"
+grep -q thin-pool <<<"${targets}" || { echo "ERROR: thin-pool dm target missing" >&2; exit 1; }
 
 sudo mkdir -p /etc/hyper
 sudo tee /etc/hyper/config.toml >/dev/null <<'EOF'

From 816f7279f9590413e9b7b66b41d38728efb59c60 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:42:21 +0000
Subject: [PATCH 10/22] ci: pre-create the layer store dir the boot validation
 requires

---
 .github/scripts/provision-kvm-host.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh
index 25e5fae4..88b75bfe 100755
--- a/.github/scripts/provision-kvm-host.sh
+++ b/.github/scripts/provision-kvm-host.sh
@@ -45,8 +45,11 @@ sudo mkdir -p /sys/fs/cgroup/hyper
 echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control >/dev/null
 
 # The BEAM (the `runner` user — Hyper refuses to run as root) owns work_dir.
+# layers/ must pre-exist: boot validation (Layer.Repo.test_system) checks it,
+# and the node only creates it lazily on first image load.
 sudo mkdir -p /srv/hyper
 sudo chown "$(id -u):$(id -g)" /srv/hyper
+mkdir -p /srv/hyper/layers
 
 # firecracker.install writes into /opt/firecracker as the invoking user, then
 # the helper requires both binaries root-owned and not group/world-writable.

From 5a7c6850e7b697a3a4da3d9ab9f6f65f808d9821 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:48:52 +0000
Subject: [PATCH 11/22] test(e2e): boot :micro instances so the default node
 budget admits them

---
 test/e2e/crash_teardown_test.exs | 5 ++++-
 test/e2e/vm_lifecycle_test.exs   | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/e2e/crash_teardown_test.exs b/test/e2e/crash_teardown_test.exs
index 4619c160..c40af6f8 100644
--- a/test/e2e/crash_teardown_test.exs
+++ b/test/e2e/crash_teardown_test.exs
@@ -20,7 +20,10 @@ defmodule Hyper.E2e.CrashTeardownTest do
 
   test "SIGKILL of firecracker reclaims the dm volume and the routing entry" do
     assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image)
-    assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id})
+    # :micro, not the :base default — :base asks for 32 GiB of disk budget,
+    # which the default node budget (4 GiB) refuses with :no_capacity on the
+    # small CI runner.
+    assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro})
     # A failed assertion before (or after) the SIGKILL must not leak a live VM
     # into other tests in the same run; stop_image_vm/1 treats :not_found as
     # :ok, so this is safe even after crash-reclaim already tore the VM down.
diff --git a/test/e2e/vm_lifecycle_test.exs b/test/e2e/vm_lifecycle_test.exs
index 6386914c..9b020d2f 100644
--- a/test/e2e/vm_lifecycle_test.exs
+++ b/test/e2e/vm_lifecycle_test.exs
@@ -25,7 +25,10 @@ defmodule Hyper.E2e.VmLifecycleTest do
   test "load -> create_vm -> exec -> stop reclaims the VM's dm volume" do
     assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image)
 
-    assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id})
+    # :micro, not the :base default — :base asks for 32 GiB of disk budget,
+    # which the default node budget (4 GiB) refuses with :no_capacity on the
+    # small CI runner.
+    assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro})
     # A failed assertion must not leak a live VM into the next E2E test in
     # the same run; stop_image_vm/1 is idempotent, so this is safe alongside
     # the explicit stop below (which is itself the behavior under test).

From 1fefb59eb1f5692e1d7e62960ff12fdb131f3900 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 00:55:08 +0000
Subject: [PATCH 12/22] test(e2e): stop the crash test's pkill matching its own
 sudo wrapper

---
 test/e2e/crash_teardown_test.exs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/test/e2e/crash_teardown_test.exs b/test/e2e/crash_teardown_test.exs
index c40af6f8..7b061360 100644
--- a/test/e2e/crash_teardown_test.exs
+++ b/test/e2e/crash_teardown_test.exs
@@ -37,9 +37,12 @@ defmodule Hyper.E2e.CrashTeardownTest do
     assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"])
     assert MapSet.member?(dm_devices(), rw_dev)
 
-    # The jailer/firecracker cmdline carries the vm_id (--id); the BEAM's
-    # own cmdline does not, so -f cannot match the test runner itself.
-    assert {_, 0} = System.cmd("sudo", ["pkill", "-9", "-f", vm_id])
+    # The jailer/firecracker cmdline carries the vm_id (--id). The [v]...
+    # bracket regex still matches it, but not this command's own sudo wrapper —
+    # whose cmdline contains the pattern text, not the raw id (a bare -f vm_id
+    # SIGKILLs its own sudo and System.cmd reports 137 instead of 0).
+    kill_pattern = "[" <> String.first(vm_id) <> "]" <> String.slice(vm_id, 1..-1//1)
+    assert {_, 0} = System.cmd("sudo", ["pkill", "-9", "-f", kill_pattern])
 
     # Monitor-driven teardown is immediate; the Reaper backstop is 60 s ticks
     # with two-strike confirmation, hence the generous deadline.

From 76b9f1bc02a97f07e327a6c4183e8972fbc5036a Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 01:01:07 +0000
Subject: [PATCH 13/22] test(e2e): retry exec on gRPC stream errors during
 guest boot

---
 test/support/e2e.ex | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/test/support/e2e.ex b/test/support/e2e.ex
index b3c08286..091c480e 100644
--- a/test/support/e2e.ex
+++ b/test/support/e2e.ex
@@ -37,9 +37,12 @@ defmodule Hyper.E2e do
   end
 
   @doc """
-  `Hyper.exec/3`, retried while the guest agent is still coming up
-  (`:agent_unavailable` / `:timeout` are the boot-race errors; anything else
-  returns immediately).
+  `Hyper.exec/3`, retried while the guest agent is still coming up.
+
+  Boot-race errors that get retried: `:agent_unavailable`, `:timeout`, and
+  `GRPC.RPCError` — the relay's vsock stream can connect and drop while the
+  in-guest agent is still starting, which surfaces as a gRPC stream error
+  rather than `:agent_unavailable`. Anything else returns immediately.
   """
   @spec await_exec(pid() | binary(), [String.t()], non_neg_integer()) ::
           {:ok, map()} | {:error, term()}
@@ -50,16 +53,23 @@ defmodule Hyper.E2e do
 
   defp do_await_exec(vm, argv, deadline) do
     case Hyper.exec(vm, argv) do
-      {:error, reason} when reason in [:agent_unavailable, :timeout] ->
-        if System.monotonic_time(:millisecond) > deadline do
-          {:error, reason}
-        else
-          Process.sleep(1_000)
-          do_await_exec(vm, argv, deadline)
-        end
+      {:error, reason} = err when reason in [:agent_unavailable, :timeout] ->
+        retry_or_return(err, vm, argv, deadline)
+
+      {:error, %GRPC.RPCError{}} = err ->
+        retry_or_return(err, vm, argv, deadline)
 
       other ->
         other
     end
   end
+
+  defp retry_or_return(err, vm, argv, deadline) do
+    if System.monotonic_time(:millisecond) > deadline do
+      err
+    else
+      Process.sleep(1_000)
+      do_await_exec(vm, argv, deadline)
+    end
+  end
 end

From 4d2126a9423162bf330f56e7a3da78979253a41e Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 01:08:50 +0000
Subject: [PATCH 14/22] test(e2e): give the crash test's liveness exec a 3 min
 deadline

---
 test/e2e/crash_teardown_test.exs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/e2e/crash_teardown_test.exs b/test/e2e/crash_teardown_test.exs
index 7b061360..a98f8577 100644
--- a/test/e2e/crash_teardown_test.exs
+++ b/test/e2e/crash_teardown_test.exs
@@ -33,8 +33,10 @@ defmodule Hyper.E2e.CrashTeardownTest do
     rw_dev = Hyper.Node.Img.Mutable.dm_name(vm_id)
 
     # Prove the VM is fully live first — otherwise the kill would race boot
-    # and the test could pass without exercising crash reclaim at all.
-    assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"])
+    # and the test could pass without exercising crash reclaim at all. The
+    # second guest of a run has come up slower than the first on nested-virt
+    # CI runners, hence the extended deadline.
+    assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"], :timer.minutes(3))
     assert MapSet.member?(dm_devices(), rw_dev)
 
     # The jailer/firecracker cmdline carries the vm_id (--id). The [v]...

From 03d58bee745a18ce4ee0ef319f6e005be441b062 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 01:14:49 +0000
Subject: [PATCH 15/22] ci: relax the CI host's cpu load budget so E2E VMs
 admit

---
 .github/scripts/provision-kvm-host.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh
index 88b75bfe..f99382bb 100755
--- a/.github/scripts/provision-kvm-host.sh
+++ b/.github/scripts/provision-kvm-host.sh
@@ -37,6 +37,12 @@ jailer = "/opt/firecracker/jailer"
 [jails]
 uid_gid_range = [900000, 999999]
 cgroup = "hyper"
+
+# The runner is a dedicated ephemeral host: the soft budget's default
+# cpu_max_load (0.8) refuses VMs with :no_capacity while compile/provision
+# load is still decaying, and nothing else runs here worth protecting.
+[budget]
+cpu_max_load = 4.0
 EOF
 sudo chown root:root /etc/hyper/config.toml
 sudo chmod 0644 /etc/hyper/config.toml

From 6c9cc0231e3ea2f9be29d65f41f51df534e229f2 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 02:58:51 +0000
Subject: [PATCH 16/22] test(e2e): pin the crash self-heal contract (cold boot,
 then clean stop)

The teardown-on-crash contract the plan specified contradicts Core's
documented design: firecracker death cold-boots the daemon/controller
pair on the same mutable rootfs (self-heal). Decision: self-heal is the
contract. The test now proves recovery (exec answers again on the same
dm volume) and that only the explicit stop reclaims the device.
---
 ...rdown_test.exs => crash_recovery_test.exs} | 46 ++++++++++++-------
 1 file changed, 30 insertions(+), 16 deletions(-)
 rename test/e2e/{crash_teardown_test.exs => crash_recovery_test.exs} (51%)

diff --git a/test/e2e/crash_teardown_test.exs b/test/e2e/crash_recovery_test.exs
similarity index 51%
rename from test/e2e/crash_teardown_test.exs
rename to test/e2e/crash_recovery_test.exs
index a98f8577..e48f97d5 100644
--- a/test/e2e/crash_teardown_test.exs
+++ b/test/e2e/crash_recovery_test.exs
@@ -1,10 +1,13 @@
-defmodule Hyper.E2e.CrashTeardownTest do
+defmodule Hyper.E2e.CrashRecoveryTest do
   @moduledoc """
-  Crash-reclaim contract: SIGKILLing the jailed Firecracker process behind a
-  running VM must tear the whole VM down and reclaim its writable dm volume —
-  monitors free the uid and mutable layer on :DOWN, and the Reaper (60 s
-  tick, two-strike confirm) is the backstop for anything they miss. Nothing
-  may resurrect the device (the idle-reaper-restart-resurrection regression).
+  Crash-recovery contract: SIGKILLing the jailed Firecracker process behind a
+  running VM must NOT kill the VM. `Core`'s `:one_for_all` deliberately
+  cold-boots the daemon + controller pair on the same mutable rootfs
+  (lib/hyper/node/fire_vmm/core.ex), so the guest comes back and answers
+  `exec` again on the SAME writable dm volume. Reclaim happens only on
+  explicit stop: `stop_image_vm/1` must then remove the volume — the
+  crash/recover cycle must not leak the device or the routing entry past the
+  stop.
 
   Runs only under `--only integration` on a provisioned host (see
   VmLifecycleTest for the environment contract).
@@ -18,22 +21,23 @@ defmodule Hyper.E2e.CrashTeardownTest do
 
   @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19")
 
-  test "SIGKILL of firecracker reclaims the dm volume and the routing entry" do
+  test "SIGKILL of firecracker cold-boots the VM; explicit stop reclaims the volume" do
     assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image)
+
     # :micro, not the :base default — :base asks for 32 GiB of disk budget,
     # which the default node budget (4 GiB) refuses with :no_capacity on the
     # small CI runner.
     assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro})
-    # A failed assertion before (or after) the SIGKILL must not leak a live VM
-    # into other tests in the same run; stop_image_vm/1 treats :not_found as
-    # :ok, so this is safe even after crash-reclaim already tore the VM down.
+    # A failed assertion anywhere below must not leak a live VM into other
+    # tests in the same run; stop_image_vm/1 treats :not_found as :ok, so
+    # this is safe alongside the explicit stop the test itself performs.
     on_exit(fn -> Hyper.Node.stop_image_vm(vm) end)
     vm_id = Hyper.id(vm)
     assert vm_id, "Hyper.id/1 returned nil for a freshly-created VM"
     rw_dev = Hyper.Node.Img.Mutable.dm_name(vm_id)
 
     # Prove the VM is fully live first — otherwise the kill would race boot
-    # and the test could pass without exercising crash reclaim at all. The
+    # and the test could pass without exercising crash recovery at all. The
     # second guest of a run has come up slower than the first on nested-virt
     # CI runners, hence the extended deadline.
     assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"], :timer.minutes(3))
@@ -46,12 +50,22 @@ defmodule Hyper.E2e.CrashTeardownTest do
     kill_pattern = "[" <> String.first(vm_id) <> "]" <> String.slice(vm_id, 1..-1//1)
     assert {_, 0} = System.cmd("sudo", ["pkill", "-9", "-f", kill_pattern])
 
-    # Monitor-driven teardown is immediate; the Reaper backstop is 60 s ticks
-    # with two-strike confirmation, hence the generous deadline.
-    assert poll_until(fn -> not MapSet.member?(dm_devices(), rw_dev) end, :timer.minutes(4)),
-           "writable dm volume #{rw_dev} survived firecracker SIGKILL"
+    # Self-heal: Core cold-boots the daemon/controller pair; the relay child
+    # is untouched, so exec reaches the rebooted guest once its agent is back.
+    assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"], :timer.minutes(3)),
+           "VM did not recover from a firecracker SIGKILL"
+
+    # The recovered VM still runs on ITS OWN volume — the crash must not have
+    # torn it down or swapped it.
+    assert MapSet.member?(dm_devices(), rw_dev),
+           "writable dm volume #{rw_dev} vanished across the crash/recovery cycle"
+
+    assert :ok = Hyper.Node.stop_image_vm(vm)
+
+    assert poll_until(fn -> not MapSet.member?(dm_devices(), rw_dev) end, :timer.seconds(90)),
+           "writable dm volume #{rw_dev} leaked after stop_image_vm"
 
     assert poll_until(fn -> Hyper.whereis(vm_id) == nil end, :timer.minutes(1)),
-           "routing entry for #{vm_id} survived firecracker SIGKILL"
+           "routing entry for #{vm_id} survived stop_image_vm"
   end
 end

From 44b2d62a2092601ee5bc8ec38c3946d6bedba44f Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 03:08:38 +0000
Subject: [PATCH 17/22] docs: document the integration/external test tiers and
 CI job

---
 AGENTS.md                | 11 +++++++++++
 CONTRIBUTING.md          |  2 ++
 docs/cookbook/install.md |  2 +-
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/AGENTS.md b/AGENTS.md
index ed34a468..ffb1020a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -23,12 +23,23 @@ mix check        # THE gate. Must pass before any PR. Runs, in order:
                  #   dialyzer                    (strict; @specs required)
 mix test                              # Elixir suite (needs Postgres for DB tests)
 mix test test/unit test/controls      # pure tests, no Postgres/Firecracker needed
+mix test --only integration --only external
+                 # live E2E: boots real Firecracker VMs. Needs a provisioned
+                 # KVM host (docs/cookbook/install.md) + passwordless sudo.
+                 # CI runs this in ci.yml's `integration` job on ubuntu-latest.
 cargo nextest run                     # Rust suite (run inside native/suidhelper/)
 ```
 
 Pure tests under `test/unit` and `test/controls` need neither Postgres nor
 Firecracker. DB-backed tests need `mix ecto.create && mix ecto.migrate` first.
 
+The `:integration` suite (`test/e2e/`) is the live Firecracker E2E: it starts
+the full supervision tree (never combine with `--no-start`) and drives real
+VMs. `:external` tests pull real OCI images. Both are excluded by default and
+run in CI's `integration` job, which provisions the GitHub-hosted runner via
+`.github/scripts/provision-kvm-host.sh` — keep that script in lockstep with
+`docs/cookbook/install.md`.
+
 ## Layout
 
 - `lib/` — Elixir source. Tests in `test/` mirror this tree.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 317bc586..06d31e88 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -113,6 +113,8 @@ happy path is often enough. Do **not** add mechanical, one-assertion-per-getter
 tests that exist only to inflate the count -- they will be asked to be removed.
 Match the style of the tests already in `test/`.
 
+The `:integration` and `:external` test suites are excluded by default. The `:integration` suite (`test/e2e/`) contains live Firecracker E2E tests that boot real VMs and require a provisioned KVM host (see `docs/cookbook/install.md`) and passwordless sudo. The `:external` suite pulls real OCI images. Both run automatically in CI's `integration` job on every PR, so you do not need to run them locally unless you are developing them -- `mix check` runs the default test set, which is sufficient for most contributions.
+
 ## Generated code
 
 The Firecracker API bindings under
diff --git a/docs/cookbook/install.md b/docs/cookbook/install.md
index 1116ea69..9aef986e 100644
--- a/docs/cookbook/install.md
+++ b/docs/cookbook/install.md
@@ -83,7 +83,7 @@ Hyper relies on `dm-snapshot` and `dm-thin` to build COW filesystems. Load the
 modules and confirm the targets are present:
 
 ```sh
-sudo modprobe dm_snapshot dm_thin_pool loop
+sudo modprobe -a dm_snapshot dm_thin_pool loop
 sudo dmsetup targets # must list snapshot, thin, and thin-pool
 ```
 

From e03ea1e1ec15667325065115cbbc909ba239831a Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 03:24:17 +0000
Subject: [PATCH 18/22] fix(cfg): let the [budget] TOML table override budget
 defaults
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

config/runtime.exs set `config :hyper, Hyper.Cfg.Budget, ...` for every
env, and app env is Cfg's highest-priority layer — so those "defaults"
permanently shadowed the [budget] TOML table for every operator and for
our own CI provisioning script's `[budget] cpu_max_load = 4.0`, which
was inert. Move the values into Hyper.Cfg.Budget as real default-layer
fallbacks passed through fetch_cfg, so the layering matches the
module's own documented contract: config.exs > TOML > default.

cpu_max_cap keeps its nilable optional_number path so an operator can
still explicitly set `cpu_max_cap: nil` to disable the cap.
---
 config/runtime.exs             | 19 ++-------
 lib/hyper/cfg/budget.ex        | 73 ++++++++++++++++++++++------------
 test/hyper/cfg/budget_test.exs | 58 +++++++++++++++++++++++++++
 3 files changed, 109 insertions(+), 41 deletions(-)
 create mode 100644 test/hyper/cfg/budget_test.exs

diff --git a/config/runtime.exs b/config/runtime.exs
index 6df3f4e9..93b48220 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -1,22 +1,11 @@
 import Config
 
-# Per-node resource budget. Lives in runtime config because it builds `Unit.*`
-# values, which are only loadable once the app's modules are on the code path.
-config :hyper, Hyper.Cfg.Budget,
-  mem_max: Unit.Information.gib(4),
-  disk_max: Unit.Information.gib(4),
-  cpu_max_load: 0.8,
-  cpu_max_cap: 4.0,
-  disk_bw_cap: Unit.Bandwidth.gibps(1),
-  disk_bw_max_load: 0.8,
-  net_bw_cap: Unit.Bandwidth.gibps(1),
-  net_bw_max_load: 0.8
-
 # Operator overrides from a well-known location. An optional Elixir config file
 # at /etc/hyper/config.exs (override the path with HYPER_CONFIG) is merged in
-# last, so its values win over every default set above. An absent file is a
-# no-op — the normal case in dev and CI. Skipped under :test so the suite never
-# reads host state.
+# as app env, which every `Hyper.Cfg.*` module treats as the highest-priority
+# layer — above the `[budget]`-style TOML tables and above built-in defaults.
+# An absent file is a no-op — the normal case in dev and CI. Skipped under
+# :test so the suite never reads host state.
 #
 # OpenTelemetry exporter wiring is resolved through Hyper.Cfg.Otel so that the
 # operator's `config :hyper, Hyper.Cfg.Otel, ...` stanza (if present) takes
diff --git a/lib/hyper/cfg/budget.ex b/lib/hyper/cfg/budget.ex
index 2d01799e..91dcb812 100644
--- a/lib/hyper/cfg/budget.ex
+++ b/lib/hyper/cfg/budget.ex
@@ -1,9 +1,11 @@
 defmodule Hyper.Cfg.Budget do
   @moduledoc """
   This node's resource budget. Each field reads from `config.exs`
-  (`config :hyper, Hyper.Cfg.Budget, ...`), then the `[budget]` table in
-  `/etc/hyper/config.toml`, then its default. `Unit.*` quantities may be given
-  as Elixir terms in `config.exs` or as strings (`"4GiB"`, `"1GiBps"`) in TOML.
+  (`config :hyper, Hyper.Cfg.Budget, ...`, typically set via the operator
+  override file `/etc/hyper/config.exs`), then the `[budget]` table in
+  `/etc/hyper/config.toml`, then its built-in default. `Unit.*` quantities may
+  be given as Elixir terms in `config.exs` or as strings (`"4GiB"`, `"1GiBps"`)
+  in TOML.
   """
 
   import Hyper.Cfg, only: [fetch_cfg: 1]
@@ -29,20 +31,33 @@ defmodule Hyper.Cfg.Budget do
     :net_bw_max_load
   ]
 
+  @default_mem_max Unit.Information.gib(4)
+  @default_disk_max Unit.Information.gib(4)
+  @default_cpu_max_load 0.8
+  @default_cpu_max_cap 4.0
+  @default_disk_bw_cap Unit.Bandwidth.gibps(1)
+  @default_disk_bw_max_load 0.8
+  @default_net_bw_cap Unit.Bandwidth.gibps(1)
+  @default_net_bw_max_load 0.8
+
   @spec load :: {:ok, t()} | {:error, term()}
   def load do
-    with {:ok, mem_max} <- information(:mem_max, "budget.mem_max"),
-         {:ok, disk_max} <- information(:disk_max, "budget.disk_max"),
-         {:ok, cpu_max_load} <- number(:cpu_max_load, "budget.cpu_max_load"),
-         {:ok, disk_bw_cap} <- bandwidth(:disk_bw_cap, "budget.disk_bw_cap"),
-         {:ok, disk_bw_max_load} <- number(:disk_bw_max_load, "budget.disk_bw_max_load"),
-         {:ok, net_bw_cap} <- bandwidth(:net_bw_cap, "budget.net_bw_cap"),
-         {:ok, net_bw_max_load} <- number(:net_bw_max_load, "budget.net_bw_max_load") do
+    with {:ok, mem_max} <- information(:mem_max, "budget.mem_max", @default_mem_max),
+         {:ok, disk_max} <- information(:disk_max, "budget.disk_max", @default_disk_max),
+         {:ok, cpu_max_load} <-
+           number(:cpu_max_load, "budget.cpu_max_load", @default_cpu_max_load),
+         {:ok, disk_bw_cap} <-
+           bandwidth(:disk_bw_cap, "budget.disk_bw_cap", @default_disk_bw_cap),
+         {:ok, disk_bw_max_load} <-
+           number(:disk_bw_max_load, "budget.disk_bw_max_load", @default_disk_bw_max_load),
+         {:ok, net_bw_cap} <- bandwidth(:net_bw_cap, "budget.net_bw_cap", @default_net_bw_cap),
+         {:ok, net_bw_max_load} <-
+           number(:net_bw_max_load, "budget.net_bw_max_load", @default_net_bw_max_load) do
       config = %__MODULE__{
         mem_max: mem_max,
         disk_max: disk_max,
         cpu_max_load: cpu_max_load,
-        cpu_max_cap: optional_number(:cpu_max_cap, "budget.cpu_max_cap"),
+        cpu_max_cap: optional_number(:cpu_max_cap, "budget.cpu_max_cap", @default_cpu_max_cap),
         disk_bw_cap: disk_bw_cap,
         disk_bw_max_load: disk_bw_max_load,
         net_bw_cap: net_bw_cap,
@@ -57,40 +72,46 @@ defmodule Hyper.Cfg.Budget do
   @spec get :: t()
   def get, do: :persistent_term.get(__MODULE__)
 
-  @spec information(atom(), String.t()) :: {:ok, Unit.Information.t()} | {:error, term()}
-  defp information(key, toml) do
-    with {:ok, v} <- required(key, toml) do
+  @spec information(atom(), String.t(), Unit.Information.t()) ::
+          {:ok, Unit.Information.t()} | {:error, term()}
+  defp information(key, toml, default) do
+    with {:ok, v} <- required(key, toml, default) do
       coerce(v, &Unit.Information.parse/1, Unit.Information, key)
     end
   end
 
-  @spec bandwidth(atom(), String.t()) :: {:ok, Unit.Bandwidth.t()} | {:error, term()}
-  defp bandwidth(key, toml) do
-    with {:ok, v} <- required(key, toml) do
+  @spec bandwidth(atom(), String.t(), Unit.Bandwidth.t()) ::
+          {:ok, Unit.Bandwidth.t()} | {:error, term()}
+  defp bandwidth(key, toml, default) do
+    with {:ok, v} <- required(key, toml, default) do
       coerce(v, &Unit.Bandwidth.parse/1, Unit.Bandwidth, key)
     end
   end
 
-  @spec number(atom(), String.t()) :: {:ok, number()} | {:error, term()}
-  defp number(key, toml) do
-    case required(key, toml) do
+  @spec number(atom(), String.t(), number()) :: {:ok, number()} | {:error, term()}
+  defp number(key, toml, default) do
+    case required(key, toml, default) do
       {:ok, n} when is_number(n) -> {:ok, n}
       {:ok, other} -> {:error, {:not_a_number, key, other}}
       {:error, _} = e -> e
     end
   end
 
-  @spec optional_number(atom(), String.t()) :: number() | nil
-  defp optional_number(key, toml) do
-    case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do
+  # cpu_max_cap stays nilable: an operator may explicitly set
+  # `cpu_max_cap: nil` in config.exs to disable the cap outright, so a
+  # non-number resolution (including that explicit nil) falls through to nil
+  # rather than failing the whole load.
+  @spec optional_number(atom(), String.t(), number()) :: number() | nil
+  defp optional_number(key, toml, default) do
+    case fetch_cfg(runtime: {__MODULE__, key}, toml: toml, default: default) do
       {:ok, n} when is_number(n) -> n
       _ -> nil
     end
   end
 
-  @spec required(atom(), String.t()) :: {:ok, term()} | {:error, term()}
-  defp required(key, toml) do
-    case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do
+  @spec required(atom(), String.t(), term()) :: {:ok, term()} | {:error, term()}
+  defp required(key, toml, default) do
+    case fetch_cfg(runtime: {__MODULE__, key}, toml: toml, default: default) do
       {:ok, v} -> {:ok, v}
       :error -> {:error, {:missing, key}}
     end
diff --git a/test/hyper/cfg/budget_test.exs b/test/hyper/cfg/budget_test.exs
new file mode 100644
index 00000000..94866e04
--- /dev/null
+++ b/test/hyper/cfg/budget_test.exs
@@ -0,0 +1,58 @@
+defmodule Hyper.Cfg.BudgetTest do
+  use ExUnit.Case, async: false
+
+  alias Hyper.Cfg.Budget
+  alias Hyper.Cfg.Toml
+
+  setup do
+    Application.delete_env(:hyper, Budget)
+    Toml.put_cache(%{})
+
+    on_exit(fn ->
+      Application.delete_env(:hyper, Budget)
+      Toml.reload()
+    end)
+
+    :ok
+  end
+
+  test "load/0 with no app env and no TOML table returns the documented built-in defaults" do
+    assert {:ok, config} = Budget.load()
+
+    assert config == %Budget{
+             mem_max: Unit.Information.gib(4),
+             disk_max: Unit.Information.gib(4),
+             cpu_max_load: 0.8,
+             cpu_max_cap: 4.0,
+             disk_bw_cap: Unit.Bandwidth.gibps(1),
+             disk_bw_max_load: 0.8,
+             net_bw_cap: Unit.Bandwidth.gibps(1),
+             net_bw_max_load: 0.8
+           }
+  end
+
+  test "a [budget] TOML table overrides the built-in default" do
+    Toml.put_cache(%{"budget" => %{"cpu_max_load" => 0.5, "mem_max" => "2GiB"}})
+
+    assert {:ok, config} = Budget.load()
+    assert config.cpu_max_load == 0.5
+    assert config.mem_max == Unit.Information.gib(2)
+    # Fields absent from the TOML table still fall back to their defaults.
+    assert config.disk_max == Unit.Information.gib(4)
+  end
+
+  test "an app-env override (config.exs) wins over a conflicting TOML value" do
+    Toml.put_cache(%{"budget" => %{"cpu_max_load" => 0.5}})
+    Application.put_env(:hyper, Budget, cpu_max_load: 0.9)
+
+    assert {:ok, config} = Budget.load()
+    assert config.cpu_max_load == 0.9
+  end
+
+  test "cpu_max_cap can be explicitly disabled via app env, overriding its default" do
+    Application.put_env(:hyper, Budget, cpu_max_cap: nil)
+
+    assert {:ok, config} = Budget.load()
+    assert config.cpu_max_cap == nil
+  end
+end

From 730e35c725f2fd112e485d0b80d030fe7025107a Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 03:24:43 +0000
Subject: [PATCH 19/22] docs: install guide must pre-create the layer store dir
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Boot validation (Hyper.Node.Layer.Repo.test_system/0) refuses with
:layer_dir_not_found unless <work_dir>/layers exists — the node only
creates it lazily on first image load. The install guide only had
operators create /srv/hyper itself, which would pass ownership setup
but fail the first boot.
---
 docs/cookbook/install.md | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/cookbook/install.md b/docs/cookbook/install.md
index 9aef986e..0acdaf1a 100644
--- a/docs/cookbook/install.md
+++ b/docs/cookbook/install.md
@@ -237,11 +237,14 @@ and the ones it deliberately does **not**.
 
 The node builds its entire on-disk tree (`jails`, `socks`, `scratch`, `layers`,
 `redist`) under `work_dir` (from `/etc/hyper/config.toml`, default `/srv/hyper`)
-**as this user**. It must therefore own that directory:
+**as this user**. It must therefore own that directory. Boot validation
+(`Hyper.Node.Layer.Repo.test_system/0`) refuses to start unless the `layers`
+subdirectory already exists — the node only creates it lazily on first image
+load, so pre-create it now:
 
 ```sh
-sudo mkdir -p /srv/hyper
-sudo chown hyper:hyper /srv/hyper
+sudo mkdir -p /srv/hyper/layers
+sudo chown -R hyper:hyper /srv/hyper
 ```
 
 ## Installation

From 36348f2b3d2e8e0d7afaa7a870f6530f57512536 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 03:25:39 +0000
Subject: [PATCH 20/22] test(e2e): prove crash recovery preserves the guest's
 volume state
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The dm device NAME surviving a crash/recover cycle doesn't prove it's
the SAME volume — a destroy+recreate under the same deterministic name
would pass too, silently losing guest state. Write a marker through the
guest and `sync` it (page cache dies with firecracker) before the
SIGKILL, then read it back after recovery instead of just re-running
/bin/true. The marker write doubles as the pre-crash liveness proof it
replaces; the dm-name check stays as a supplementary assertion.
---
 test/e2e/crash_recovery_test.exs | 35 ++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/test/e2e/crash_recovery_test.exs b/test/e2e/crash_recovery_test.exs
index e48f97d5..8713539d 100644
--- a/test/e2e/crash_recovery_test.exs
+++ b/test/e2e/crash_recovery_test.exs
@@ -4,10 +4,14 @@ defmodule Hyper.E2e.CrashRecoveryTest do
   running VM must NOT kill the VM. `Core`'s `:one_for_all` deliberately
   cold-boots the daemon + controller pair on the same mutable rootfs
   (lib/hyper/node/fire_vmm/core.ex), so the guest comes back and answers
-  `exec` again on the SAME writable dm volume. Reclaim happens only on
-  explicit stop: `stop_image_vm/1` must then remove the volume — the
-  crash/recover cycle must not leak the device or the routing entry past the
-  stop.
+  `exec` again on the SAME writable dm volume — proven not just by the dm
+  device name still existing post-recovery (a destroy+recreate under the same
+  deterministic name would also pass that check) but by a file written to the
+  guest's rootfs before the crash still being there after recovery, which only
+  survives if the underlying block device itself was preserved. Reclaim
+  happens only on explicit stop: `stop_image_vm/1` must then remove the
+  volume — the crash/recover cycle must not leak the device or the routing
+  entry past the stop.
 
   Runs only under `--only integration` on a provisioned host (see
   VmLifecycleTest for the environment contract).
@@ -39,8 +43,17 @@ defmodule Hyper.E2e.CrashRecoveryTest do
     # Prove the VM is fully live first — otherwise the kill would race boot
     # and the test could pass without exercising crash recovery at all. The
     # second guest of a run has come up slower than the first on nested-virt
-    # CI runners, hence the extended deadline.
-    assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"], :timer.minutes(3))
+    # CI runners, hence the extended deadline. The marker write doubles as the
+    # volume-identity proof: `sync` forces it past the page cache (which dies
+    # with firecracker) and onto the dm volume itself, so it can only survive
+    # the crash if that volume — not just a same-named replacement — does.
+    assert {:ok, %{exit_code: 0}} =
+             await_exec(
+               vm,
+               ["/bin/sh", "-c", "echo persisted > /marker && sync"],
+               :timer.minutes(3)
+             )
+
     assert MapSet.member?(dm_devices(), rw_dev)
 
     # The jailer/firecracker cmdline carries the vm_id (--id). The [v]...
@@ -52,11 +65,17 @@ defmodule Hyper.E2e.CrashRecoveryTest do
 
     # Self-heal: Core cold-boots the daemon/controller pair; the relay child
     # is untouched, so exec reaches the rebooted guest once its agent is back.
-    assert {:ok, %{exit_code: 0}} = await_exec(vm, ["/bin/true"], :timer.minutes(3)),
+    assert {:ok, %{stdout: marker, exit_code: 0}} =
+             await_exec(vm, ["/bin/cat", "/marker"], :timer.minutes(3)),
            "VM did not recover from a firecracker SIGKILL"
 
+    assert marker =~ "persisted",
+           "recovered VM lost pre-crash writes — not the same volume"
+
     # The recovered VM still runs on ITS OWN volume — the crash must not have
-    # torn it down or swapped it.
+    # torn it down or swapped it. Supplementary to the marker-file assertion
+    # above: this only proves the dm device name persisted, which a
+    # destroy+recreate under the same deterministic name would also satisfy.
     assert MapSet.member?(dm_devices(), rw_dev),
            "writable dm volume #{rw_dev} vanished across the crash/recovery cycle"
 

From 837e541480f6e09600f1a57cdea43c6cb824d224 Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 03:26:14 +0000
Subject: [PATCH 21/22] ci: harden the integration job's timeout and
 provisioning diagnostics

- Bump the integration job timeout 25 -> 35 min: worst case is two
  10-minute E2E tests plus ~4 min of provisioning/compile/setup, which
  could hit 25 and get cancelled mid-run; cancelled steps then skip
  `!cancelled()` result uploads exactly when flake history matters.
- provision-kvm-host.sh: give the suidhelper setuid-bit check a
  diagnostic on failure, matching the existing thin-pool check, instead
  of failing silently under `set -e`.
- provision-kvm-host.sh: enumerate the deliberate CI-only deltas from
  install.md in the header comment so a future sync pass doesn't
  remove them.
---
 .github/scripts/provision-kvm-host.sh | 16 +++++++++++++++-
 .github/workflows/ci.yml              |  6 +++++-
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/provision-kvm-host.sh b/.github/scripts/provision-kvm-host.sh
index f99382bb..74180026 100755
--- a/.github/scripts/provision-kvm-host.sh
+++ b/.github/scripts/provision-kvm-host.sh
@@ -3,6 +3,20 @@
 # for the `:integration` E2E suite. Mirrors docs/cookbook/install.md; keep
 # the two in sync. Assumes: passwordless sudo, repo compiled (the firecracker
 # and suidhelper install tasks are mix tasks), MIX_ENV matching the test run.
+#
+# Deliberate CI-only deltas from install.md — do NOT remove these in a future
+# sync pass:
+#   - the 0666 udev kvm rule (install.md assumes a real host with a `kvm`
+#     group an operator is added to; the ephemeral runner has neither)
+#   - the `[budget]` stanza raising cpu_max_load to 4.0 (this host is a
+#     dedicated ephemeral runner, so the default's contention guard against
+#     other workloads doesn't apply, and compile/provision load would
+#     otherwise trip :no_capacity)
+#   - pre-creating /srv/hyper/layers (install.md documents this too, but it's
+#     load-bearing here since nothing else primes it before the suite boots
+#     the node)
+#   - DEBIAN_FRONTEND=noninteractive (install.md is written for an interactive
+#     operator session; CI has no tty to prompt on)
 set -euo pipefail
 
 export DEBIAN_FRONTEND=noninteractive
@@ -67,6 +81,6 @@ sudo chmod 0755 /opt/firecracker/firecracker /opt/firecracker/jailer
 
 # With passwordless sudo the task installs setuid-root itself.
 mix suidhelper.install
-[ -u /usr/local/bin/hyper-suidhelper ]
+[ -u /usr/local/bin/hyper-suidhelper ] || { echo "ERROR: hyper-suidhelper missing or not setuid-root" >&2; exit 1; }
 
 echo "provisioned: kvm=$(ls -l /dev/kvm)"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ace2d282..69cf4a92 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -259,7 +259,11 @@ jobs:
     # ubuntu-latest x64 exposes /dev/kvm (all Linux runners with 2+ vCPUs,
     # per GitHub's Apr 2024 changelog); arm64 runners do NOT — keep this x64.
     runs-on: ubuntu-latest
-    timeout-minutes: 25
+    # Worst case: two 10-minute tests (vm_lifecycle + crash_recovery) plus
+    # ~4 minutes of provisioning/compile/setup. 25 could cancel the job right
+    # as it finishes, and `!cancelled()` steps then skip result uploads —
+    # exactly when flake history matters most.
+    timeout-minutes: 35
     env:
       MIX_ENV: test
     services:

From 833d20da7f38ca9687a2a6cac10193ecb718137b Mon Sep 17 00:00:00 2001
From: Marko Vejnovic <marko@harmont.dev>
Date: Tue, 7 Jul 2026 03:36:02 +0000
Subject: [PATCH 22/22] docs(cfg): record the budget defaults; erase cached
 budget between tests

---
 docs/cookbook/config.md        | 25 +++++++++++++------------
 test/hyper/cfg/budget_test.exs |  3 +++
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md
index 5521f5ca..84a91b03 100644
--- a/docs/cookbook/config.md
+++ b/docs/cookbook/config.md
@@ -185,18 +185,19 @@ headers = { "authorization" = "Bearer YOUR_TOKEN" }
 ## Budget Configuration
 
 Hyper allows you to control the absolute maximal budgets that are available to
-all VMs on a particular node. Every field is required except `cpu_max_cap`.
-
-| Config Key         | `config.exs`        | `config.toml`       | Default | Notes |
-| ------------------ | ------------------- | ------------------- | ------- | ----- |
-| `mem_max`          | `.mem_max`          | `.mem_max`          | -       | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of RAM the node offers VMs. Must not exceed available system memory. |
-| `disk_max`         | `.disk_max`         | `.disk_max`         | -       | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of disk the node offers VMs. Must not exceed available system disk. |
-| `cpu_max_load`     | `.cpu_max_load`     | `.cpu_max_load`     | -       | [$\beta$ (soft) budget](./architecture.md#budgets): instantaneous CPU load threshold, a fraction `0.0`–`1.0` of the cap. |
-| `cpu_max_cap`      | `.cpu_max_cap`      | `.cpu_max_cap`      | `nil`   | [$\beta$ (soft) budget](./architecture.md#budgets): soft CPU capacity in cores (a float). Optional. |
-| `disk_bw_cap`      | `.disk_bw_cap`      | `.disk_bw_cap`      | -       | [$\beta$ (soft) budget](./architecture.md#budgets): soft disk-bandwidth capacity ([unit](#unit)). |
-| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | -       | [$\beta$ (soft) budget](./architecture.md#budgets): disk-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. |
-| `net_bw_cap`       | `.net_bw_cap`       | `.net_bw_cap`       | -       | [$\beta$ (soft) budget](./architecture.md#budgets): soft network-bandwidth capacity ([unit](#unit)). |
-| `net_bw_max_load`  | `.net_bw_max_load`  | `.net_bw_max_load`  | -       | [$\beta$ (soft) budget](./architecture.md#budgets): network-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. |
+all VMs on a particular node. Every field has a built-in default; set only the
+keys you need to change.
+
+| Config Key         | `config.exs`        | `config.toml`       | Default   | Notes |
+| ------------------ | ------------------- | ------------------- | --------- | ----- |
+| `mem_max`          | `.mem_max`          | `.mem_max`          | `4 GiB`   | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of RAM the node offers VMs. Must not exceed available system memory. |
+| `disk_max`         | `.disk_max`         | `.disk_max`         | `4 GiB`   | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of disk the node offers VMs. Must not exceed available system disk. |
+| `cpu_max_load`     | `.cpu_max_load`     | `.cpu_max_load`     | `0.8`     | [$\beta$ (soft) budget](./architecture.md#budgets): instantaneous CPU load threshold, a fraction `0.0`–`1.0` of the cap. |
+| `cpu_max_cap`      | `.cpu_max_cap`      | `.cpu_max_cap`      | `4.0`     | [$\beta$ (soft) budget](./architecture.md#budgets): soft CPU capacity in cores (a float). Optional — an explicit `nil` in `config.exs` disables the cap. |
+| `disk_bw_cap`      | `.disk_bw_cap`      | `.disk_bw_cap`      | `1 GiBps` | [$\beta$ (soft) budget](./architecture.md#budgets): soft disk-bandwidth capacity ([unit](#unit)). |
+| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | `0.8`     | [$\beta$ (soft) budget](./architecture.md#budgets): disk-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. |
+| `net_bw_cap`       | `.net_bw_cap`       | `.net_bw_cap`       | `1 GiBps` | [$\beta$ (soft) budget](./architecture.md#budgets): soft network-bandwidth capacity ([unit](#unit)). |
+| `net_bw_max_load`  | `.net_bw_max_load`  | `.net_bw_max_load`  | `0.8`     | [$\beta$ (soft) budget](./architecture.md#budgets): network-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. |
 
 <!-- tabs open -->
 ### `config.exs`
diff --git a/test/hyper/cfg/budget_test.exs b/test/hyper/cfg/budget_test.exs
index 94866e04..881efa94 100644
--- a/test/hyper/cfg/budget_test.exs
+++ b/test/hyper/cfg/budget_test.exs
@@ -11,6 +11,9 @@ defmodule Hyper.Cfg.BudgetTest do
     on_exit(fn ->
       Application.delete_env(:hyper, Budget)
       Toml.reload()
+      # load/0 caches the struct; erase it so no later test observes this
+      # test's budget by accident.
+      :persistent_term.erase(Budget)
     end)
 
     :ok