feat: guest command exec over vsock (Hyper.exec/3)

.github/workflows/ci.yml

@@ -73,6 +73,18 @@ jobs:
       - name: Install protoc-gen-elixir
         run: mix escript.install hex protobuf 0.17.0 --force
 
+      # The `:guest_agent_build` Mix compiler builds the static musl guest agent
+      # (native/guest-agent) into priv/ on every `mix compile`; the host-arch
+      # build is a hard requirement, so the runner's default toolchain needs the
+      # musl std. Cross arches are best-effort and stay skipped here.
+      - name: Install musl target for the guest-agent build
+        run: rustup target add x86_64-unknown-linux-musl
+
+      - name: Cache guest-agent cargo build
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: native/guest-agent
+
       - name: Compile (warnings as errors)
         # --force matches the `mix check` alias: re-surfaces warnings even on a
         # cached _build, where an unchanged module would otherwise be skipped.

.gitignore

@@ -41,10 +41,12 @@ docs/superpowers/
 /lib/hyper/firecracker/api/operations/
 /lib/hyper/firecracker/api/schemas/
 
-# gRPC bindings are generated from proto/hyper/grpc/v0/hyper.proto before compile
+# gRPC bindings are generated from proto/**/*.proto before compile
 # -- see the :grpc_gen Mix compiler in mix.exs.
 /lib/hyper/grpc/v0/
+/lib/hyper/agent/v1/
 
 # The expected suidhelper build identity (version + checksum) is captured from
 # the stamped binary's `version` output -- see the :suidhelper_stamp Mix compiler.
 /lib/hyper/suid_helper/expected.ex
+/priv/guest-agent/

AGENTS.md

@@ -105,6 +105,31 @@ These will be asked to be removed (per `CONTRIBUTING.md`):
 
 Coverage is a side effect of good tests, never the target.
 
+### Answering coverage reports
+
+When codecov (or any coverage tool) flags uncovered lines, do NOT write tests
+to move the number. Triage each uncovered region first:
+
+- **Contract** — behavior a caller or operator relies on: a refusal, a
+  wire/decode boundary, a retry/advance classification, error propagation.
+  Write the behavioral test. Ask: "what realistic mutation of this code
+  should this test kill?" If you cannot name one, it is not a contract.
+- **Mechanical** — 1:1 mappings (`From` impls, delegation wrappers,
+  derive-equivalent code), CLI dispatch arms, `Display`/error strings nothing
+  parses. Decline: a test would restate the code and break on refactor
+  without catching bugs. Record the decline where the work is tracked.
+- **Privileged / integration-only** — paths needing root, a live guest, or a
+  second node. Decline at unit level and name the integration/E2E that
+  covers them.
+
+A patch may legitimately merge below 100%: uncovered mechanical lines cost
+less than tests that pin implementation detail.
+
+**Parametrize by data, not by assertion.** When two or more cases share one
+assertion shape and differ only in inputs/expected values, write one
+table-driven test (or a property) over rows — never copy-paste test bodies.
+If a case needs its own assertions, it is a separate test, not a row.
+
 ### Elixir specifics
 
 - `use ExUnit.Case, async: true` and `use ExUnitProperties` for property suites.

lib/hyper.ex

@@ -39,6 +39,59 @@ defmodule Hyper do
   defp resolve_arch(nil), do: Sys.Arch.current()
   defp resolve_arch(arch), do: {:ok, arch}
 
+  @doc """
+  Run `argv` inside VM `vm` — a pid (from `create_vm/1`) or a `Hyper.Vm.Id.t()`
+  binary — and return its captured output.
+
+  Cluster-routed: a pid resolves to its owning node directly (`node(pid)`); a
+  vm_id resolves via the routing registry (`whereis/1`). The command runs on
+  whichever node hosts the VM, via that node's `Hyper.Node.exec/3`.
+
+  Returns `{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}`, or
+  `{:error, reason}` — `:agent_unavailable` if the relay/agent is not yet
+  reachable, `:timeout` if the gRPC deadline expired, `:not_found` if the VM
+  cannot be resolved (unknown vm_id, or a pid whose id/owning node is gone),
+  `:node_unreachable` if the owning node cannot be reached over the cluster.
+  `opts`: `:env` (map `%{String.t() => String.t()}`), `:cwd` (string), `:timeout`
+  (ms, gRPC deadline forwarded to the relay; default 30 000).
+
+  `argv` is executed directly, not through a shell. Use an **absolute** `argv`
+  head (`["/bin/echo", "hi"]`): the guest runs as PID 1 with a near-empty
+  environment, and a non-empty `:env` *replaces* it entirely, so a bare command
+  name has no `PATH` to resolve against and returns exit code 127.
+  """
+  @spec exec(Hyper.Vm.t() | Hyper.Vm.Id.t(), [String.t()], keyword()) ::
+          {:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}
+          | {:error, term()}
+  def exec(vm, argv, opts \\ [])
+
+  def exec(vm, argv, opts) when is_pid(vm) and is_list(argv) do
+    case id(vm) do
+      nil -> {:error, :not_found}
+      vm_id -> route(node(vm), vm_id, argv, opts)
+    end
+  end
+
+  def exec(vm_id, argv, opts) when is_binary(vm_id) and is_list(argv) do
+    case whereis(vm_id) do
+      nil -> {:error, :not_found}
+      node -> route(node, vm_id, argv, opts)
+    end
+  end
+
+  # Run the node-local exec on the VM's owning node. A pid already carries its
+  # node; a vm_id was resolved through the routing registry. Transport failures
+  # (owning node down/partitioned) become `:node_unreachable` rather than a raise
+  # in the caller — the VM is unusable from here either way.
+  @spec route(node(), Hyper.Vm.Id.t(), [String.t()], keyword()) ::
+          {:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}
+          | {:error, term()}
+  defp route(node, vm_id, argv, opts) do
+    :erpc.call(node, Hyper.Node, :exec, [vm_id, argv, opts])
+  catch
+    :error, {:erpc, _} -> {:error, :node_unreachable}
+  end
+
   @doc "Cluster-wide: which node currently runs `vm_id`? `nil` if unknown."
   @spec whereis(Hyper.Vm.Id.t()) :: node() | nil
   def whereis(vm_id), do: Hyper.Cluster.Routing.whereis(vm_id)

lib/hyper/img/oci_loader.ex

@@ -21,6 +21,7 @@ defmodule Hyper.Img.OciLoader do
   use Unit.Operators
 
   alias Hyper.Img.OciLoader.Umoci
+  alias Hyper.Node.FireVMM.GuestAgent
   alias Unit.Information
 
   require Logger
@@ -62,6 +63,7 @@ defmodule Hyper.Img.OciLoader do
          {:ok, arch} <- Sys.Arch.current() do
       Sys.Tmp.with_tempdir("hyper-oci", fn tmp ->
         with {:ok, rootfs} <- pull_and_unpack(source, goarch(arch), tmp),
+             :ok <- stage_agent(rootfs, arch),
              {:ok, {content, files}} <- dir_usage(rootfs),
              params = ext4_params(content, files),
              {:ok, staged} <- build_ext4(rootfs, params) do
@@ -108,6 +110,17 @@ defmodule Hyper.Img.OciLoader do
   def goarch(:x86_64), do: "amd64"
   def goarch(:aarch64), do: "arm64"
 
+  @doc false
+  @spec stage_agent(Path.t(), Sys.Arch.t()) :: :ok | {:error, term()}
+  def stage_agent(rootfs, arch), do: stage_agent_from(rootfs, GuestAgent.path(arch))
+
+  @doc false
+  @spec stage_agent_from(Path.t(), Path.t()) :: :ok | {:error, term()}
+  def stage_agent_from(rootfs, agent_bin) do
+    dest = Path.join(rootfs, "hyper-init")
+    with :ok <- File.cp(agent_bin, dest), do: File.chmod(dest, 0o755)
+  end
+
   # `du` apparent bytes undercount ext4 block usage and the default inode ratio
   # starves file-dense trees, so the size carries the inode table plus slack and
   # the inode count is the file count with headroom.

lib/hyper/node.ex

@@ -105,6 +105,18 @@ defmodule Hyper.Node do
     end
   end
 
+  @doc """
+  Run `argv` in a VM hosted **on this node**, identified by `vm_id`, via its
+  relay/agent. The node-local half of `Hyper.exec/3`, which resolves the owning
+  node and `:erpc`-calls this function there.
+  """
+  @spec exec(Hyper.Vm.Id.t(), [String.t()], keyword()) ::
+          {:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}
+          | {:error, term()}
+  def exec(vm_id, argv, opts \\ []) do
+    Hyper.Node.FireVMM.Agent.exec(vm_id, argv, opts)
+  end
+
   @doc false
   @spec build_opts(Hyper.Vm.Id.t(), Hyper.Vm.Spec.t(), Users.id(), pid(), Path.t()) ::
           FireVMM.Opts.t()

lib/hyper/node/fire_vmm.ex

@@ -1,7 +1,7 @@
 defmodule Hyper.Node.FireVMM do
   @moduledoc """
-  Supervises a single Firecracker microVM, split into two independent subtrees so
-  no lifecycle invariant rides on the ordering of a flat child list:
+  Supervises a single Firecracker microVM, split into three independent children
+  so no lifecycle invariant rides on the ordering of a flat child list:
 
     1. `Hyper.Node.FireVMM.Core` - the daemon container + `:gen_statem`
        controller, coupled under `:one_for_all` (a controller crash also discards
@@ -10,14 +10,21 @@ defmodule Hyper.Node.FireVMM do
        (it derives the socket itself) and on nothing else in the tree, so it is
        an independent peer: its crashes don't disturb the core, and a core
        restart doesn't cycle it.
+    3. `Hyper.Node.FireVMM.Agent.Relay` - the host-side gRPC relay that bridges
+       inbound Unix-socket connections to the in-guest agent over vsock. Restart
+       is `:transient`: an abnormal crash (unexpected accept error) is restarted;
+       a clean stop (`:shutdown` from the supervisor) is not.
 
-  Strategy is `:one_for_one`: the two subtrees are restarted independently.
+  Strategy is `:one_for_one`: the three children are restarted independently.
   """
 
   use Supervisor
 
+  alias Hyper.Node.FireVMM.Agent
+  alias Hyper.Node.FireVMM.Agent.Relay
   alias Hyper.Node.FireVMM.Client
   alias Hyper.Node.FireVMM.Core
+  alias Hyper.Node.FireVMM.Jailer
 
   @doc "The scheduler period of each VM."
   @spec cpu_period() :: Unit.Time.t()
@@ -75,7 +82,13 @@ defmodule Hyper.Node.FireVMM do
           # which calls Client.run while waiting for the daemon's API. Client
           # depends only on vm_id (an independent peer), so no reverse dependency.
           {Client, %Client.Opts{vm_id: opts.vm_id}},
-          {Core, opts}
+          {Core, opts},
+          {Relay,
+           %{
+             vm_id: opts.vm_id,
+             vsock_uds: Jailer.host_vsock(opts.vm_id),
+             listen_path: Agent.relay_socket_path(opts.vm_id)
+           }}
         ]
 
         Supervisor.init(children, strategy: :one_for_one)
@@ -88,6 +101,6 @@ defmodule Hyper.Node.FireVMM do
   @doc "Test whether the system can run firecracker VMMs."
   @spec test_system() :: :ok | {:error, term()}
   def test_system do
-    Hyper.Node.FireVMM.Jailer.test_system()
+    Jailer.test_system()
   end
 end

lib/hyper/node/fire_vmm/agent.ex

@@ -0,0 +1,89 @@
+defmodule Hyper.Node.FireVMM.Agent do
+  @moduledoc """
+  gRPC client to the in-guest agent, dialed through the per-VM relay UDS.
+
+  `relay_socket_path/1` is the single source of truth for the host-side
+  Unix-domain socket the relay listens on. The relay GenServer (Task 7) will
+  call this function to derive its own listen path, so both sides agree
+  without a process-registry lookup.
+  """
+
+  alias Hyper.Agent.V1.{ExecRequest, GuestAgent.Stub}
+
+  # GRPC.Status codes stored as module attributes so they can be used in
+  # guard-equivalent pattern matches without calling functions at match time.
+  @unavailable GRPC.Status.unavailable()
+  @deadline_exceeded GRPC.Status.deadline_exceeded()
+
+  @default_timeout 30_000
+
+  @doc """
+  Deterministic host-side path for the per-VM relay Unix socket.
+
+  Derived from `Hyper.Cfg.Dirs.socket_dir/0` so both this client and the
+  relay GenServer agree without a registry lookup.
+
+  Task 7: the relay GenServer listens here.
+  """
+  @spec relay_socket_path(Hyper.Vm.Id.t()) :: Path.t()
+  def relay_socket_path(vm_id) do
+    # Vm.Id chars are [a-z2-7] plus a 'v' prefix — safe as a filename
+    # component without further sanitization.
+    Path.join(Hyper.Cfg.Dirs.socket_dir(), "grpc-#{vm_id}.sock")
+  end
+
+  @doc """
+  Run `argv` in the guest VM identified by `vm_id` via gRPC through the relay.
+
+  Returns `{:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}`,
+  or `{:error, term()}`. Error values: `:agent_unavailable` (relay unreachable —
+  gRPC UNAVAILABLE), `:timeout` (gRPC DEADLINE_EXCEEDED), or raw transport
+  errors for unexpected failures.
+
+  ## Options
+
+    - `:env` — environment map `%{String.t() => String.t()}` for the guest process
+    - `:cwd` — working directory inside the guest (`nil` → not set)
+    - `:timeout` — gRPC deadline in milliseconds (default: #{@default_timeout})
+  """
+  @spec exec(Hyper.Vm.Id.t(), [String.t()], keyword()) ::
+          {:ok, %{stdout: binary(), stderr: binary(), exit_code: integer()}}
+          | {:error, term()}
+  def exec(vm_id, argv, opts \\ []) do
+    path = relay_socket_path(vm_id)
+    timeout = Keyword.get(opts, :timeout, @default_timeout)
+
+    req = %ExecRequest{
+      argv: argv,
+      env: Map.new(Keyword.get(opts, :env, %{})),
+      cwd: opts[:cwd]
+    }
+
+    case GRPC.Stub.connect("unix://" <> path, adapter: GRPC.Client.Adapters.Gun) do
+      {:error, _} = err ->
+        map_grpc_error(err)
+
+      {:ok, ch} ->
+        result =
+          case Stub.exec(ch, req, timeout: timeout) do
+            {:ok, resp} ->
+              {:ok, %{exit_code: resp.exit_code, stdout: resp.stdout, stderr: resp.stderr}}
+
+            err ->
+              map_grpc_error(err)
+          end
+
+        _ = GRPC.Stub.disconnect(ch)
+        result
+    end
+  end
+
+  @spec map_grpc_error({:error, term()}) :: {:error, term()}
+  defp map_grpc_error({:error, %GRPC.RPCError{status: @unavailable}}),
+    do: {:error, :agent_unavailable}
+
+  defp map_grpc_error({:error, %GRPC.RPCError{status: @deadline_exceeded}}),
+    do: {:error, :timeout}
+
+  defp map_grpc_error({:error, _} = err), do: err
+end