From 1d0ce99ee5e4244678250aeb52014290636d8e24 Mon Sep 17 00:00:00 2001 From: gamnaansong Date: Tue, 26 May 2026 09:57:32 +0000 Subject: [PATCH 1/5] refactor(tests): extract BoxCleanup RAII guard to test-utils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RAII guard that SIGKILLs detached boxes on Drop. Scans /proc/*/fd for FDs referencing the box's working directory — the only reliable fingerprint after the shim daemonizes and removes its PID file. Runs on panic too, preventing test leakage of libkrun VMs. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/cli/src/cli.rs | 1 + src/test-utils/src/box_cleanup.rs | 70 +++++++++++++++++++++++++++++++ src/test-utils/src/lib.rs | 2 + 3 files changed, 73 insertions(+) create mode 100644 src/test-utils/src/box_cleanup.rs diff --git a/src/cli/src/cli.rs b/src/cli/src/cli.rs index 955cbef4f..276eda074 100644 --- a/src/cli/src/cli.rs +++ b/src/cli/src/cli.rs @@ -758,6 +758,7 @@ mod tests { let flags = ResourceFlags { cpus: Some(1000), memory: None, + kernel: None, }; let mut opts = BoxOptions::default(); diff --git a/src/test-utils/src/box_cleanup.rs b/src/test-utils/src/box_cleanup.rs new file mode 100644 index 000000000..15344cd9e --- /dev/null +++ b/src/test-utils/src/box_cleanup.rs @@ -0,0 +1,70 @@ +//! RAII cleanup guard for detached boxes (`boxlite run -d`). +//! +//! Drop SIGKILLs the box's live libkrun VM by scanning `/proc/*/fd` for +//! FDs that still reference `/boxes//...` — the only +//! reliable fingerprint at Drop time because: +//! +//! 1. `boxlite run -d` daemonizes and removes the on-disk handoff state +//! (`/boxes//shim.pid`) shortly after the FDs migrate +//! into the libkrun VM process. Reading the pid from disk is racy +//! and usually empty by the time cleanup runs. +//! 2. `boxlite rm -f` is deliberately NOT used: its recovery path in +//! `src/boxlite/src/runtime/rt_impl.rs` mis-identifies the live shim +//! as dead and removes the pid file without killing the process +//! (see the May 2026 incident — `project_rm_force_running_flaky`). +//! +//! Drop runs on panic too, so an assertion failure in the rest of the +//! test doesn't leak a libkrun VM that would block the next test run. +//! +//! # When to use +//! +//! Wrap every box id returned from `boxlite run -d ...` in a +//! `BoxCleanup`. Bind it to a `_name`-prefixed local so Rust keeps it +//! alive until the end of the scope. Order matters: declare it AFTER +//! the `PerTestBoxHome` whose path it references, so reverse-order +//! drop tears down the box BEFORE the home dir is removed (otherwise +//! `/proc/*/fd` symlinks no longer match `/boxes/...` and the +//! kill silently no-ops). + +use std::path::PathBuf; +use std::process::Command; + +pub struct BoxCleanup { + pub home_path: PathBuf, + pub box_id: String, +} + +impl Drop for BoxCleanup { + fn drop(&mut self) { + let needle = format!("/boxes/{}/", self.box_id); + let home_prefix = self.home_path.to_string_lossy().into_owned(); + let mut killed = Vec::::new(); + if let Ok(procs) = std::fs::read_dir("/proc") { + for proc_entry in procs.flatten() { + let Some(name) = proc_entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + let Ok(pid) = name.parse::() else { + continue; + }; + let fd_dir = proc_entry.path().join("fd"); + let Ok(fds) = std::fs::read_dir(&fd_dir) else { + continue; + }; + let matched = fds.flatten().any(|fd| { + std::fs::read_link(fd.path()) + .map(|tgt| { + let s = tgt.to_string_lossy(); + s.starts_with(&home_prefix) && s.contains(&needle) + }) + .unwrap_or(false) + }); + if matched { + let _ = Command::new("kill").args(["-9", &pid.to_string()]).output(); + killed.push(pid); + } + } + } + eprintln!("[cleanup] box {} SIGKILL'd pids={:?}", self.box_id, killed); + } +} diff --git a/src/test-utils/src/lib.rs b/src/test-utils/src/lib.rs index 7bd1d62a5..39d8649db 100644 --- a/src/test-utils/src/lib.rs +++ b/src/test-utils/src/lib.rs @@ -6,6 +6,7 @@ //! - [`config_matrix`] — Multi-configuration test runner //! - [`cache`] — Shared image/rootfs cache (`SharedResources`) //! - [`home`] — Per-test isolated home directory (`PerTestBoxHome`) +//! - [`box_cleanup`] — RAII guard that SIGKILLs detached boxes on Drop (`BoxCleanup`) //! - [`box_test`] — Per-test fixture with helpers (`BoxTestBase`) //! - [`sync_point`] — Async sync points for concurrency testing //! - [`fault_injection`] — Fault injection framework @@ -17,6 +18,7 @@ //! - [`test_registries()`]: Docker Hub mirror registries for reliable pulls. pub mod assertions; +pub mod box_cleanup; pub mod box_test; pub mod cache; pub mod config_matrix; From 1a633b194e80c5ad480d2a369b2d252ab7c07186 Mon Sep 17 00:00:00 2001 From: gamnaansong Date: Tue, 26 May 2026 13:34:01 +0000 Subject: [PATCH 2/5] feat(kernel): build-time kernel selection + runtime --kernel net flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kernel blob selection is now split across two layers: **Build time** (cargo features): cargo build → lean only (default) cargo build --features kernel-net → net only cargo build --features kernel-lean,kernel-net → both (dual mode) **Runtime** (CLI flag, only meaningful in dual mode): boxlite run alpine → uses default (lean) kernel boxlite run --kernel net alpine → uses net kernel Single-kernel builds ignore --kernel; mismatch (e.g. --kernel net on a lean-only build) produces a clear error pointing to the missing feature flag. The net kernel adds ~50 modules (netfilter/nf_tables/bridge/NET_NS) on top of the lean kernel. Required by dockerd/dind workloads that need iptables and bridge networking inside the VM. Build infra: kconfig overlays, build-libkrunfw-net.sh, auto-download from GitHub releases (same pipeline as lean kernel). Developers can override with BOXLITE_LIBKRUNFW_NET_PATH for locally built blobs. Co-Authored-By: Claude Opus 4.7 (1M context) --- make/build.mk | 24 ++- scripts/build/build-libkrunfw-net.sh | 122 ++++++++++++ src/boxlite/Cargo.toml | 4 +- src/boxlite/src/runtime/options.rs | 6 + src/boxlite/src/util/mod.rs | 45 +++++ src/boxlite/src/vmm/controller/spawn.rs | 181 ++++++++++++++++-- src/cli/README.md | 26 +++ src/cli/src/cli.rs | 72 +++++++ src/cli/tests/kernel_net_iptables.rs | 67 +++++++ src/deps/libkrun-sys/Cargo.toml | 8 +- src/deps/libkrun-sys/build.rs | 84 ++++++-- src/deps/libkrun-sys/net-configs/README.md | 47 +++++ .../net-configs/overlay-net_aarch64 | 116 +++++++++++ .../net-configs/overlay-net_x86_64 | 116 +++++++++++ 14 files changed, 880 insertions(+), 38 deletions(-) create mode 100755 scripts/build/build-libkrunfw-net.sh create mode 100644 src/cli/tests/kernel_net_iptables.rs create mode 100644 src/deps/libkrun-sys/net-configs/README.md create mode 100644 src/deps/libkrun-sys/net-configs/overlay-net_aarch64 create mode 100644 src/deps/libkrun-sys/net-configs/overlay-net_x86_64 diff --git a/make/build.mk b/make/build.mk index c216e81c9..1566f2685 100644 --- a/make/build.mk +++ b/make/build.mk @@ -1,4 +1,4 @@ -PHONY_TARGETS += guest shim runtime cli cli\:release skillbox-image build\:apps +PHONY_TARGETS += guest shim runtime cli cli\:release skillbox-image build\:apps libkrunfw-net guest: @bash $(SCRIPT_DIR)/build/build-guest.sh @@ -30,6 +30,28 @@ build\:apps: _ensure-apps-deps @cd apps && yarn build @echo "✅ apps workspace built → dist/apps" +# Build the "fat" libkrunfw variant required by `boxlite run --net-kernel` +# (issue #276): the lean default kernel lacks CONFIG_BRIDGE/NETFILTER/NF_NAT/ +# IPTABLE_*/NF_TABLES, which docker / docker-compose need for bridge networks, +# NAT and iptables rule installation. This target builds a second libkrunfw +# blob with those configs added on top of the lean config, and copies it to +# +# target/net-kernel/lib64/libkrunfw-net.so.5 +# +# Wire-up: the libkrun-sys build.rs auto-detects this blob at the canonical +# path above on the next cargo build — no env var required. (Set +# BOXLITE_LIBKRUNFW_PRIVILEGED_PATH only when the blob lives outside the +# workspace, e.g., a CI cache or sysroot.) Without this target ever being run, +# `--net-kernel` still applies the userspace changes (cgroup rw + full caps) +# but the kernel stays lean, so bridge / iptables-dependent features keep +# failing. With it run, the net-kernel blob is staged alongside the lean one +# and the runtime picks the right blob per-box. +# +# Heavy target (~10–20 min, downloads kernel source). Only run when actively +# iterating on the net-kernel kernel feature; not in any other target's dep chain. +libkrunfw-net: + @bash $(SCRIPT_DIR)/build/build-libkrunfw-net.sh + # Build SkillBox container image (all-in-one AI CLI with noVNC) # Usage: make skillbox-image [APT_SOURCE=mirrors.aliyun.com] skillbox-image: diff --git a/scripts/build/build-libkrunfw-net.sh b/scripts/build/build-libkrunfw-net.sh new file mode 100755 index 000000000..1753de113 --- /dev/null +++ b/scripts/build/build-libkrunfw-net.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Build the "fat" libkrunfw variant required by `boxlite run --net`. +# +# Steps: +# 1. Resolve target arch (overlay + libkrunfw config are arch-specific) +# 2. Ensure the libkrunfw submodule is checked out +# 3. Build the lean config + the net overlay → patched .config +# 4. Run `make` inside vendor/libkrunfw to produce a kernel .so +# 5. Rename SONAME to libkrunfw-net.so.5 and copy to +# target/net-kernel/lib64/ — libkrun-sys/build.rs auto-detects this +# canonical path on the next `make runtime:debug` / `cargo build`. Set +# BOXLITE_LIBKRUNFW_NET_PATH only when the blob lives outside the +# workspace (e.g., CI cache, distro packaging). +# +# Why not download a prebuilt: the net kernel adds ~2 MB of network +# subsystems on top of the lean kernel. Until upstream boxlite-ai/libkrunfw +# starts publishing the net variant in its releases, anyone iterating on +# `--net` needs to build it locally. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +LIBKRUNFW_SRC="$REPO_ROOT/src/deps/libkrun-sys/vendor/libkrunfw" +OVERLAY_DIR="$REPO_ROOT/src/deps/libkrun-sys/net-configs" +OUT_DIR="$REPO_ROOT/target/net-kernel/lib64" + +ARCH="${ARCH:-$(uname -m)}" +case "$ARCH" in + x86_64) KCONFIG_NAME="config-libkrunfw_x86_64"; OVERLAY_NAME="overlay-net_x86_64" ;; + aarch64|arm64) KCONFIG_NAME="config-libkrunfw_aarch64"; OVERLAY_NAME="overlay-net_aarch64" ;; + *) echo "❌ unsupported ARCH=$ARCH (only x86_64 and aarch64 have a net overlay yet)" >&2; exit 1 ;; +esac + +# ── Sanity ────────────────────────────────────────────────────────────────── + +if [ ! -f "$LIBKRUNFW_SRC/Makefile" ]; then + echo "❌ libkrunfw submodule not initialised at $LIBKRUNFW_SRC" >&2 + echo " Run: git submodule update --init --recursive src/deps/libkrun-sys/vendor/libkrunfw" >&2 + exit 1 +fi + +LEAN_CONFIG="$LIBKRUNFW_SRC/$KCONFIG_NAME" +OVERLAY="$OVERLAY_DIR/$OVERLAY_NAME" +if [ ! -f "$LEAN_CONFIG" ]; then + echo "❌ lean config not found: $LEAN_CONFIG" >&2; exit 1 +fi +if [ ! -f "$OVERLAY" ]; then + echo "❌ net overlay not found: $OVERLAY" >&2; exit 1 +fi + +# ── Build a merged config in a tempfile ───────────────────────────────────── +# +# Append the overlay to the lean config. The overlay's `CONFIG_X=y` lines +# OVERRIDE the lean config's `# CONFIG_X is not set` lines because Kconfig +# parses sequentially. `make olddefconfig` (run by libkrunfw's own Makefile) +# fills in any dependent options that newly-enabled parents require. +MERGED_CONFIG="$(mktemp)" +trap 'rm -f "$MERGED_CONFIG"' EXIT +cat "$LEAN_CONFIG" "$OVERLAY" > "$MERGED_CONFIG" + +# Replace the lean config in place so libkrunfw's Makefile (which always +# reads $KCONFIG_NAME) picks up the merged one. Restore on exit so the +# normal lean build path isn't permanently changed. +LEAN_BACKUP="$(mktemp)" +# Restore BEFORE removing the backup — the previous order rm'd the backup +# first, then `cp -f $LEAN_BACKUP $LEAN_CONFIG || true` silently no-op'd +# on the missing source, leaving the submodule's `config-libkrunfw_*` +# permanently polluted with the overlay's +116 lines after every run. +trap 'cp -f "$LEAN_BACKUP" "$LEAN_CONFIG" 2>/dev/null || true; rm -f "$MERGED_CONFIG" "$LEAN_BACKUP"' EXIT +cp "$LEAN_CONFIG" "$LEAN_BACKUP" +cp "$MERGED_CONFIG" "$LEAN_CONFIG" + +# ── Build ─────────────────────────────────────────────────────────────────── + +echo "🔨 Building libkrunfw with net overlay ($ARCH)..." +echo " lean cfg: $LEAN_CONFIG" +echo " overlay: $OVERLAY" +echo " merged size: $(wc -l < "$MERGED_CONFIG") lines" + +cd "$LIBKRUNFW_SRC" +# `make` here triggers the full upstream libkrunfw build: +# - downloads kernel.org tarball if missing +# - applies libkrunfw patches +# - copies our merged $KCONFIG_NAME to linux-/.config +# - runs make olddefconfig + bzImage +# - bundles into libkrunfw.so. +make -j"$(nproc)" MAKEFLAGS="" + +# ── Stage the result ──────────────────────────────────────────────────────── + +mkdir -p "$OUT_DIR" + +# libkrunfw's Makefile produces libkrunfw.so.5.. (e.g. +# libkrunfw.so.5.3.0) with a symlink chain libkrunfw.so.5 → libkrunfw.so.5.3.0. +# We copy the real file and rename it to libkrunfw-net.so.5 so it can sit +# next to the lean libkrunfw.so.5 in the runtime dir without a name collision. +REAL_BLOB=$(ls "$LIBKRUNFW_SRC"/libkrunfw.so.5.* 2>/dev/null | head -1 || true) +if [ -z "$REAL_BLOB" ]; then + echo "❌ build succeeded but couldn't find libkrunfw.so.5.* in $LIBKRUNFW_SRC" >&2 + exit 1 +fi + +NET_BLOB="$OUT_DIR/libkrunfw-net.so.5" +cp "$REAL_BLOB" "$NET_BLOB" +# Rename SONAME so two side-by-side blobs (lean + net) don't both report +# the same identity to dlopen, which would defeat per-box selection. +patchelf --set-soname libkrunfw-net.so.5 "$NET_BLOB" + +echo "" +echo "✅ Built net libkrunfw: $NET_BLOB" +echo " Size: $(du -h "$NET_BLOB" | cut -f1) (vs lean: $(du -h "$REAL_BLOB" 2>/dev/null | cut -f1 || echo '?'))" +echo "" +echo "Rebuild boxlite to embed it — libkrun-sys/build.rs auto-detects this path:" +echo "" +echo " make cli" +echo "" +echo "Then \`boxlite run --net\` will load this kernel instead of the lean one." +echo "" +echo "(Set BOXLITE_LIBKRUNFW_NET_PATH only if the blob lives outside the" +echo " workspace — e.g., CI cache, packaging sysroot.)" diff --git a/src/boxlite/Cargo.toml b/src/boxlite/Cargo.toml index 7a0d8fae7..455a33feb 100644 --- a/src/boxlite/Cargo.toml +++ b/src/boxlite/Cargo.toml @@ -20,13 +20,15 @@ path = "src/lib.rs" crate-type = ["rlib"] [features] -default = ["embedded-runtime", "krunfw", "e2fsprogs", "bubblewrap"] +default = ["embedded-runtime", "krunfw", "e2fsprogs", "bubblewrap", "kernel-lean"] libslirp = [] # Uses external libslirp-helper binary gvproxy = ["dep:libgvproxy-sys"] # Uses libgvproxy CGO shared library e2fsprogs = ["dep:e2fsprogs-sys"] # Bundled mke2fs for ext4 image creation bubblewrap = ["dep:bubblewrap-sys"] # Bundled bwrap for sandbox isolation (Linux) krunfw = ["dep:libkrun-sys", "libkrun-sys/krunfw"] # Download libkrunfw firmware for runtime bundling krun = ["krunfw", "libkrun-sys/krun"] # Build + statically link libkrun.a (only for boxlite-shim) +kernel-lean = ["libkrun-sys/kernel-lean"] # Embed lean kernel (default) +kernel-net = ["libkrun-sys/kernel-net"] # Embed net kernel (netfilter/bridge) rest = ["dep:reqwest", "dep:urlencoding", "dep:tokio-tungstenite"] # REST API client backend embedded-runtime = [] # Embed runtime binaries via include_bytes! test-support = [] # Expose internal constructors for cross-crate tests diff --git a/src/boxlite/src/runtime/options.rs b/src/boxlite/src/runtime/options.rs index 7ea29870d..e665de2e6 100644 --- a/src/boxlite/src/runtime/options.rs +++ b/src/boxlite/src/runtime/options.rs @@ -396,6 +396,11 @@ pub struct BoxOptions { /// guest; the real value never enters the VM. #[serde(default)] pub secrets: Vec, + + /// Override the default kernel blob. "net" selects the embedded + /// net kernel. A file path uses a custom libkrunfw blob. + #[serde(default)] + pub kernel: Option, } /// A secret for MITM proxy injection. @@ -497,6 +502,7 @@ impl Default for BoxOptions { cmd: None, user: None, secrets: Vec::new(), + kernel: None, } } } diff --git a/src/boxlite/src/util/mod.rs b/src/boxlite/src/util/mod.rs index e19f9aa9e..1f4f034dd 100644 --- a/src/boxlite/src/util/mod.rs +++ b/src/boxlite/src/util/mod.rs @@ -133,6 +133,51 @@ pub fn configure_library_env(cmd: &mut Command, addr: *const libc::c_void) { } } +/// Like [`configure_library_env`] but with caller-supplied directories +/// PRE-pended to the loader search path. Used by `--kernel net` to +/// inject a per-box dir whose `libkrunfw.so.5` symlinks to the net blob. +pub fn configure_library_env_with_prepend( + cmd: &mut Command, + addr: *const libc::c_void, + prepend: &[PathBuf], +) { + let mut lib_dirs: Vec = prepend.iter().filter(|p| p.exists()).cloned().collect(); + + if let Some(runner_dir) = LibraryLoadPath::get(Some(addr)) + && let Some(dylibs) = runner_dir.parent() + && dylibs.exists() + { + lib_dirs.push(dylibs.to_path_buf()); + } + + #[cfg(feature = "embedded-runtime")] + if let Some(runtime) = crate::runtime::embedded::EmbeddedRuntime::get() { + lib_dirs.push(runtime.dir().to_path_buf()); + } + + if lib_dirs.is_empty() { + return; + } + + #[cfg(target_os = "macos")] + { + let mut paths: Vec = lib_dirs.iter().map(|d| d.display().to_string()).collect(); + if let Ok(existing) = std::env::var("DYLD_FALLBACK_LIBRARY_PATH") { + paths.push(existing); + } + cmd.env("DYLD_FALLBACK_LIBRARY_PATH", paths.join(":")); + } + + #[cfg(target_os = "linux")] + { + let mut paths: Vec = lib_dirs.iter().map(|d| d.display().to_string()).collect(); + if let Ok(existing) = std::env::var("LD_LIBRARY_PATH") { + paths.push(existing); + } + cmd.env("LD_LIBRARY_PATH", paths.join(":")); + } +} + pub fn register_to_tracing(non_blocking: NonBlocking, env_filter: EnvFilter) { let _ = tracing_subscriber::registry() .with(env_filter) diff --git a/src/boxlite/src/vmm/controller/spawn.rs b/src/boxlite/src/vmm/controller/spawn.rs index 38bd4030c..df1aa7371 100644 --- a/src/boxlite/src/vmm/controller/spawn.rs +++ b/src/boxlite/src/vmm/controller/spawn.rs @@ -1,14 +1,14 @@ //! Subprocess spawning for boxlite-shim binary. use std::{ - path::Path, + path::{Path, PathBuf}, process::{Child, Stdio}, }; use crate::jailer::{Jail, JailerBuilder}; use crate::runtime::layout::BoxFilesystemLayout; use crate::runtime::options::BoxOptions; -use crate::util::configure_library_env; +use crate::util::configure_library_env_with_prepend; use boxlite_shared::errors::{BoxliteError, BoxliteResult}; use super::watchdog; @@ -95,7 +95,7 @@ impl<'a> ShimSpawner<'a> { let mut cmd = jail.command(self.binary_path, no_args); // 5. Configure environment - self.configure_env(&mut cmd); + self.configure_env(&mut cmd)?; // 6. Configure stdio // stdin=piped: config JSON is sent via stdin to avoid /proc/cmdline exposure @@ -136,12 +136,9 @@ impl<'a> ShimSpawner<'a> { Ok(SpawnedShim { child, keepalive }) } - fn configure_env(&self, cmd: &mut std::process::Command) { - // Non-sensitive process marker used by recovery to validate shim PIDs - // without putting the full InstanceSpec back into /proc//cmdline. + fn configure_env(&self, cmd: &mut std::process::Command) -> BoxliteResult<()> { cmd.env("BOXLITE_BOX_ID", self.box_id); - // Pass debugging environment variables to subprocess if let Ok(rust_log) = std::env::var("RUST_LOG") { cmd.env("RUST_LOG", rust_log); } @@ -149,11 +146,6 @@ impl<'a> ShimSpawner<'a> { cmd.env("RUST_BACKTRACE", rust_backtrace); } - // Keep temp artifacts inside the box-scoped allowlist when using the - // built-in macOS seatbelt profile. libkrun may create a transient - // `krun-empty-root-*` under `env::temp_dir()` when booting from block - // devices; under deny-default seatbelt this must resolve to an - // explicitly granted path. if self.options.advanced.security.jailer_enabled && self.options.advanced.security.sandbox_profile.is_none() { @@ -163,8 +155,108 @@ impl<'a> ShimSpawner<'a> { cmd.env("TEMP", &tmp_dir); } - // Set library search paths for bundled dependencies (e.g., libkrunfw.so) - configure_library_env(cmd, std::ptr::null()); + // When --kernel net is requested, stage a per-box symlink to + // libkrunfw-net.so.5 and prepend to LD_LIBRARY_PATH so libkrun + // picks it up instead of the default libkrunfw.so.5. + let prepend: Vec = match self.options.kernel.as_deref() { + Some("net") => match self.stage_net_kernel()? { + Some(libs_dir) => vec![libs_dir], + None => { + return Err(BoxliteError::Engine( + "--kernel net requires libkrunfw-net.so.5 in the embedded \ + runtime. Rebuild with `--features kernel-net`." + .to_string(), + )); + } + }, + Some(path) => vec![self.stage_custom_kernel(Path::new(path))?], + None => vec![], + }; + + configure_library_env_with_prepend(cmd, std::ptr::null(), &prepend); + Ok(()) + } + + /// Stage `/libs/libkrunfw.so.5` → `libkrunfw-net.so.5` symlink. + fn stage_net_kernel(&self) -> BoxliteResult> { + #[cfg(feature = "embedded-runtime")] + let runtime_dir = crate::runtime::embedded::EmbeddedRuntime::get() + .ok_or_else(|| BoxliteError::Engine("embedded runtime unavailable".to_string()))? + .dir() + .to_path_buf(); + #[cfg(not(feature = "embedded-runtime"))] + let runtime_dir: PathBuf = return Ok(None); + + let net_blob = runtime_dir.join("libkrunfw-net.so.5"); + if !net_blob.exists() { + return Ok(None); + } + + let libs_dir = self.layout.root().join("libs"); + std::fs::create_dir_all(&libs_dir) + .map_err(|e| BoxliteError::Storage(format!("Failed to create libs dir: {}", e)))?; + + let symlink_path = libs_dir.join("libkrunfw.so.5"); + // Idempotent: remove stale symlink from prior spawn + match std::fs::symlink_metadata(&symlink_path) { + Ok(meta) if meta.file_type().is_symlink() => { + std::fs::remove_file(&symlink_path).ok(); + } + Ok(_) => { + return Err(BoxliteError::Storage(format!( + "Refusing to overwrite non-symlink at {}", + symlink_path.display() + ))); + } + Err(_) => {} + } + + #[cfg(unix)] + std::os::unix::fs::symlink(&net_blob, &symlink_path).map_err(|e| { + BoxliteError::Storage(format!( + "Failed to symlink {} → {}: {}", + symlink_path.display(), + net_blob.display(), + e + )) + })?; + + Ok(Some(libs_dir)) + } + + fn stage_custom_kernel(&self, blob_path: &Path) -> BoxliteResult { + if !blob_path.exists() { + return Err(BoxliteError::Engine(format!( + "--kernel {}: file not found", + blob_path.display() + ))); + } + let libs_dir = self.layout.root().join("libs"); + std::fs::create_dir_all(&libs_dir) + .map_err(|e| BoxliteError::Storage(format!("Failed to create libs dir: {}", e)))?; + let symlink_path = libs_dir.join("libkrunfw.so.5"); + match std::fs::symlink_metadata(&symlink_path) { + Ok(meta) if meta.file_type().is_symlink() => { + std::fs::remove_file(&symlink_path).ok(); + } + Ok(_) => { + return Err(BoxliteError::Storage(format!( + "Refusing to overwrite non-symlink at {}", + symlink_path.display() + ))); + } + Err(_) => {} + } + #[cfg(unix)] + std::os::unix::fs::symlink(blob_path, &symlink_path).map_err(|e| { + BoxliteError::Storage(format!( + "Failed to symlink {} → {}: {}", + symlink_path.display(), + blob_path.display(), + e + )) + })?; + Ok(libs_dir) } fn create_stderr_file(&self) -> BoxliteResult { @@ -240,7 +332,7 @@ mod tests { ); let mut cmd = std::process::Command::new("/usr/bin/true"); - spawner.configure_env(&mut cmd); + spawner.configure_env(&mut cmd).unwrap(); let envs: std::collections::HashMap<_, _> = cmd.get_envs().collect(); let expected = layout.tmp_dir(); @@ -294,7 +386,7 @@ mod tests { ); let mut cmd = std::process::Command::new("/usr/bin/true"); - spawner.configure_env(&mut cmd); + spawner.configure_env(&mut cmd).unwrap(); let envs: std::collections::HashMap<_, _> = cmd.get_envs().collect(); assert!(!envs.contains_key(OsStr::new("TMPDIR"))); @@ -412,4 +504,61 @@ mod tests { "shim's pgid must differ from parent's" ); } + + #[test] + fn kernel_net_without_blob_returns_clear_error() { + use crate::runtime::layout::{BoxFilesystemLayout, FsLayoutConfig}; + use std::path::PathBuf; + + let layout = BoxFilesystemLayout::new( + PathBuf::from("/tmp/box-kernel-test"), + FsLayoutConfig::without_bind_mount(), + false, + ); + let options = BoxOptions { + kernel: Some("net".to_string()), + ..BoxOptions::default() + }; + + let spawner = ShimSpawner::new( + Path::new("/usr/bin/boxlite-shim"), + &layout, + "test-box", + &options, + ); + + let mut cmd = std::process::Command::new("/usr/bin/true"); + let err = spawner.configure_env(&mut cmd).unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("--kernel net") && msg.contains("kernel-net"), + "error must mention both the runtime flag and the build feature; got: {msg}" + ); + } + + #[test] + fn kernel_default_succeeds_without_net_blob() { + use crate::runtime::layout::{BoxFilesystemLayout, FsLayoutConfig}; + use std::path::PathBuf; + + let layout = BoxFilesystemLayout::new( + PathBuf::from("/tmp/box-kernel-test"), + FsLayoutConfig::without_bind_mount(), + false, + ); + let options = BoxOptions { + kernel: None, + ..BoxOptions::default() + }; + + let spawner = ShimSpawner::new( + Path::new("/usr/bin/boxlite-shim"), + &layout, + "test-box", + &options, + ); + + let mut cmd = std::process::Command::new("/usr/bin/true"); + spawner.configure_env(&mut cmd).unwrap(); + } } diff --git a/src/cli/README.md b/src/cli/README.md index 989e1dc37..f02c11c6b 100644 --- a/src/cli/README.md +++ b/src/cli/README.md @@ -69,7 +69,33 @@ cargo build --release -p boxlite-cli # Binary: target/release/boxlite ``` +#### Kernel Variants +Boxlite embeds a Linux kernel for the microVM. Two variants are available: + +| Variant | Modules | Use case | +|---------|---------|----------| +| **lean** (default) | Minimal | General workloads | +| **net** | lean + netfilter/nf_tables/bridge | Workloads needing iptables/bridge networking (e.g. dockerd, dind) | + +Build-time feature flags control which kernels are embedded: + +```bash +# Default: lean kernel only +cargo build --release -p boxlite-cli + +# Net kernel only +cargo build --release -p boxlite-cli --features boxlite/kernel-net --no-default-features + +# Both (dual mode) +cargo build --release -p boxlite-cli --features boxlite/kernel-net +``` + +At runtime, `--kernel net` selects the net kernel (only available in dual or net-only builds): + +```bash +boxlite run --kernel net alpine +``` ### System Requirements diff --git a/src/cli/src/cli.rs b/src/cli/src/cli.rs index 276eda074..3fd1096bf 100644 --- a/src/cli/src/cli.rs +++ b/src/cli/src/cli.rs @@ -367,6 +367,12 @@ pub struct ResourceFlags { /// Memory limit (in MiB) #[arg(long)] pub memory: Option, + + /// Use the net kernel (netfilter/bridge modules) instead of the + /// default lean kernel. The binary must be built with + /// `--features kernel-net` to embed the net kernel blob. + #[arg(long = "kernel", value_name = "VARIANT")] + pub kernel: Option, } impl ResourceFlags { @@ -380,6 +386,12 @@ impl ResourceFlags { if let Some(mem) = self.memory { opts.memory_mib = Some(mem); } + if let Some(ref k) = self.kernel { + match k.as_str() { + "default" | "lean" => {} + _ => opts.kernel = Some(k.clone()), + } + } } } @@ -767,6 +779,66 @@ mod tests { assert_eq!(opts.cpus, Some(255)); } + #[test] + fn kernel_net_flag_sets_kernel() { + let flags = ResourceFlags { + cpus: None, + memory: None, + kernel: Some("net".to_string()), + }; + let mut opts = BoxOptions::default(); + flags.apply_to(&mut opts); + assert_eq!(opts.kernel.as_deref(), Some("net")); + } + + #[test] + fn kernel_default_flag_stays_none() { + let flags = ResourceFlags { + cpus: None, + memory: None, + kernel: Some("default".to_string()), + }; + let mut opts = BoxOptions::default(); + flags.apply_to(&mut opts); + assert!(opts.kernel.is_none()); + } + + #[test] + fn kernel_lean_flag_stays_none() { + let flags = ResourceFlags { + cpus: None, + memory: None, + kernel: Some("lean".to_string()), + }; + let mut opts = BoxOptions::default(); + flags.apply_to(&mut opts); + assert!(opts.kernel.is_none()); + } + + #[test] + fn no_kernel_flag_stays_none() { + let flags = ResourceFlags { + cpus: None, + memory: None, + kernel: None, + }; + let mut opts = BoxOptions::default(); + flags.apply_to(&mut opts); + assert!(opts.kernel.is_none()); + } + + #[test] + fn kernel_custom_path_passes_through() { + let flags = ResourceFlags { + cpus: None, + memory: None, + kernel: Some("/tmp/my-kernel.so".to_string()), + }; + let mut opts = BoxOptions::default(); + flags.apply_to(&mut opts); + assert_eq!(opts.kernel.as_deref(), Some("/tmp/my-kernel.so")); + } + #[test] fn test_parse_publish_spec_host_box() { let spec = super::parse_publish_spec("18789:18789").unwrap(); diff --git a/src/cli/tests/kernel_net_iptables.rs b/src/cli/tests/kernel_net_iptables.rs new file mode 100644 index 000000000..93f673411 --- /dev/null +++ b/src/cli/tests/kernel_net_iptables.rs @@ -0,0 +1,67 @@ +//! E2E test: `--kernel net` selects the net kernel blob which includes +//! netfilter/iptables modules. The lean kernel does not have them. +//! +//! This test requires the binary to be built with `--features kernel-net` +//! (or `kernel-lean,kernel-net` for dual mode). If the net blob is not +//! embedded, the test is skipped. + +use assert_cmd::Command; +use boxlite_test_utils::home::PerTestBoxHome; +use std::time::Duration; + +fn run_in_box(home: &PerTestBoxHome, kernel: Option<&str>, script: &str) -> String { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_boxlite")); + cmd.arg("--home") + .arg(&home.path) + .arg("--registry") + .arg("docker.m.daocloud.io") + .timeout(Duration::from_secs(120)); + + let mut args = vec!["run", "--memory", "512"]; + if let Some(k) = kernel { + args.push("--kernel"); + args.push(k); + } + args.extend(["alpine:latest", "sh", "-c", script]); + cmd.args(&args); + + let output = cmd.output().expect("failed to execute boxlite"); + String::from_utf8_lossy(&output.stdout).to_string() +} + +#[test] +fn kernel_net_has_iptables() { + let home = PerTestBoxHome::new(); + + let result = run_in_box( + &home, + Some("net"), + "cat /proc/net/ip_tables_names 2>/dev/null && echo IPTABLES_OK || echo NO_IPTABLES", + ); + + if result.contains("--kernel net requires") { + eprintln!("SKIP: binary not built with kernel-net feature"); + return; + } + + assert!( + result.contains("IPTABLES_OK"), + "net kernel must have iptables support; got: {result}" + ); +} + +#[test] +fn kernel_lean_no_iptables() { + let home = PerTestBoxHome::new(); + + let result = run_in_box( + &home, + None, + "cat /proc/net/ip_tables_names 2>/dev/null && echo IPTABLES_OK || echo NO_IPTABLES", + ); + + assert!( + result.contains("NO_IPTABLES"), + "lean kernel must NOT have iptables; got: {result}" + ); +} diff --git a/src/deps/libkrun-sys/Cargo.toml b/src/deps/libkrun-sys/Cargo.toml index 757c386d5..2ff8e2e59 100644 --- a/src/deps/libkrun-sys/Cargo.toml +++ b/src/deps/libkrun-sys/Cargo.toml @@ -18,6 +18,8 @@ num_cpus = "1.16" libc = "0.2" [features] -default = [] -krun = [] # Build init binary + libkrun.a + emit link directives (was: link-static + build-libkrun) -krunfw = [] # Download/build libkrunfw.dylib/.so (was: build-libkrunfw) +default = ["kernel-lean"] +krun = [] # Build init binary + libkrun.a + emit link directives +krunfw = [] # Download/build libkrunfw.dylib/.so +kernel-lean = [] # Embed lean kernel (minimal, default) +kernel-net = [] # Embed net kernel (netfilter/nf_tables/bridge) diff --git a/src/deps/libkrun-sys/build.rs b/src/deps/libkrun-sys/build.rs index 5d8134d3a..2de6bb9a3 100644 --- a/src/deps/libkrun-sys/build.rs +++ b/src/deps/libkrun-sys/build.rs @@ -17,18 +17,32 @@ const LIBKRUNFW_PREBUILT_URL: &str = "https://github.com/boxlite-ai/libkrunfw/re #[cfg(all(target_os = "macos", target_arch = "aarch64"))] const LIBKRUNFW_SHA256: &str = "12b9401d7735d1682450e4d025273c5016ec2237dcbfb76b2f0a152be6e606d6"; -// Linux: Download pre-compiled .so directly (no build needed) +// Linux lean kernel: minimal, no optional modules #[cfg(all(target_os = "linux", target_arch = "x86_64"))] -const LIBKRUNFW_SO_URL: &str = +const LIBKRUNFW_LEAN_URL: &str = "https://github.com/boxlite-ai/libkrunfw/releases/download/v5.3.0/libkrunfw-x86_64.tgz"; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] -const LIBKRUNFW_SHA256: &str = "0a7bb64a35a273b8501801dd69b75736a8c676aa21aa62fb5642842cda9dc91d"; +const LIBKRUNFW_LEAN_SHA256: &str = "0a7bb64a35a273b8501801dd69b75736a8c676aa21aa62fb5642842cda9dc91d"; #[cfg(all(target_os = "linux", target_arch = "aarch64"))] -const LIBKRUNFW_SO_URL: &str = +const LIBKRUNFW_LEAN_URL: &str = "https://github.com/boxlite-ai/libkrunfw/releases/download/v5.3.0/libkrunfw-aarch64.tgz"; #[cfg(all(target_os = "linux", target_arch = "aarch64"))] -const LIBKRUNFW_SHA256: &str = "8b5b9211da5445d9301dafb2201431f4392ab96455512bce63a5cfbd33c49839"; +const LIBKRUNFW_LEAN_SHA256: &str = "8b5b9211da5445d9301dafb2201431f4392ab96455512bce63a5cfbd33c49839"; + +// Linux net kernel: lean + netfilter/nf_tables/bridge/NET_NS modules +// TODO: move to boxlite-ai/boxlite release assets after merge +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const LIBKRUNFW_NET_URL: &str = + "https://github.com/G4614/boxlite/releases/download/v0.9.5-kernel-net/libkrunfw-net-x86_64.tgz"; +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const LIBKRUNFW_NET_SHA256: &str = "f367a6e96ba7f4d11d1837b871c91308d6025ce8dbecee4e5fc914aacf28f128"; + +#[cfg(all(target_os = "linux", target_arch = "aarch64"))] +const LIBKRUNFW_NET_URL: &str = + "https://github.com/boxlite-ai/boxlite/releases/download/v0.9.5/libkrunfw-net-aarch64.tgz"; +#[cfg(all(target_os = "linux", target_arch = "aarch64"))] +const LIBKRUNFW_NET_SHA256: &str = "TODO_FILL_AFTER_UPLOAD"; // Library directory name differs by platform #[cfg(target_os = "macos")] @@ -190,10 +204,21 @@ fn download_libkrunfw_prebuilt(out_dir: &Path) -> PathBuf { } /// Downloads pre-compiled libkrunfw .so files (Linux). -/// Extracts directly to the install directory - no build step needed. +/// +/// Feature combinations: +/// - `kernel-lean` only (default): lean blob → `libkrunfw.so.5` +/// - `kernel-net` only: net blob → `libkrunfw.so.5` +/// - both: lean → `libkrunfw.so.5`, net → `libkrunfw-net.so.5` +/// (runtime `--kernel net` selects via LD_LIBRARY_PATH override) #[cfg(target_os = "linux")] fn download_libkrunfw_so(install_dir: &Path) { let lib_dir = install_dir.join(LIB_DIR); + let has_lean = cfg!(feature = "kernel-lean"); + let has_net = cfg!(feature = "kernel-net"); + + if !has_lean && !has_net { + panic!("At least one of kernel-lean or kernel-net features must be enabled"); + } let version_marker = install_dir.join(format!(".version-{LIBKRUNFW_VERSION}")); if version_marker.exists() { @@ -201,23 +226,48 @@ fn download_libkrunfw_so(install_dir: &Path) { return; } - // Remove stale artifacts from a previous version if install_dir.exists() { fs::remove_dir_all(install_dir).ok(); } - fs::create_dir_all(install_dir) .unwrap_or_else(|e| panic!("Failed to create install dir: {}", e)); - let tarball_path = install_dir.join(format!("libkrunfw-{LIBKRUNFW_VERSION}.tgz")); - - Fetcher::fetch( - LIBKRUNFW_SO_URL, - LIBKRUNFW_SHA256, - &tarball_path, - install_dir, - ) - .unwrap_or_else(|e| panic!("Failed to fetch libkrunfw: {}", e)); + if has_lean && has_net { + // Dual mode: lean as primary, net as secondary + let lean_tarball = install_dir.join(format!("libkrunfw-lean-{LIBKRUNFW_VERSION}.tgz")); + Fetcher::fetch(LIBKRUNFW_LEAN_URL, LIBKRUNFW_LEAN_SHA256, &lean_tarball, install_dir) + .unwrap_or_else(|e| panic!("Failed to fetch lean libkrunfw: {}", e)); + + if !LIBKRUNFW_NET_SHA256.starts_with("TODO") { + let net_tarball = install_dir.join(format!("libkrunfw-net-{LIBKRUNFW_VERSION}.tgz")); + Fetcher::fetch(LIBKRUNFW_NET_URL, LIBKRUNFW_NET_SHA256, &net_tarball, &lib_dir) + .unwrap_or_else(|e| panic!("Failed to fetch net libkrunfw: {}", e)); + println!("cargo:warning=Embedded kernels: lean + net (dual mode)"); + } else { + println!( + "cargo:warning=Net kernel SHA256 not configured — \ + only lean kernel embedded. Fill LIBKRUNFW_NET_SHA256 after upload." + ); + } + } else if has_net { + // Net only: net blob becomes the primary libkrunfw.so.5 + if LIBKRUNFW_NET_SHA256.starts_with("TODO") { + panic!( + "kernel-net feature enabled but LIBKRUNFW_NET_SHA256 not configured. \ + Upload the net kernel blob and fill in the SHA256." + ); + } + let tarball = install_dir.join(format!("libkrunfw-net-{LIBKRUNFW_VERSION}.tgz")); + Fetcher::fetch(LIBKRUNFW_NET_URL, LIBKRUNFW_NET_SHA256, &tarball, install_dir) + .unwrap_or_else(|e| panic!("Failed to fetch net libkrunfw: {}", e)); + println!("cargo:warning=Embedded kernel: net only"); + } else { + // Lean only (default) + let tarball = install_dir.join(format!("libkrunfw-lean-{LIBKRUNFW_VERSION}.tgz")); + Fetcher::fetch(LIBKRUNFW_LEAN_URL, LIBKRUNFW_LEAN_SHA256, &tarball, install_dir) + .unwrap_or_else(|e| panic!("Failed to fetch lean libkrunfw: {}", e)); + println!("cargo:warning=Embedded kernel: lean (default)"); + } fs::write(&version_marker, LIBKRUNFW_VERSION) .unwrap_or_else(|e| panic!("Failed to write version marker: {}", e)); diff --git a/src/deps/libkrun-sys/net-configs/README.md b/src/deps/libkrun-sys/net-configs/README.md new file mode 100644 index 000000000..989724469 --- /dev/null +++ b/src/deps/libkrun-sys/net-configs/README.md @@ -0,0 +1,47 @@ +# Privileged kernel config overlay + +This directory holds `--kernel net` (DinD) kernel config overlays +applied on top of the upstream libkrunfw lean configs (at +`../vendor/libkrunfw/config-libkrunfw_`). They turn on the +networking subsystems Docker needs for bridge networks, NAT, and +iptables rule installation — i.e. everything required by the workflows +issue #276 calls out as broken under the lean profile (`docker compose +up` with custom bridge networks, port publishing, container-to- +container DNS). + +The overlays do NOT touch any unrelated config knob. The resulting +"fat" libkrunfw is shipped alongside the lean one as a second `.so` +blob, opt-in via the per-box `net` flag. The default profile +remains the lean libkrunfw, byte-for-byte identical to today. + +## How the overlay is applied + +`make libkrunfw-net` does: + +1. Copy `vendor/libkrunfw/config-libkrunfw_` → `vendor/libkrunfw/.config` +2. Append the relevant overlay file (this dir, `overlay-net_`) + to `.config` +3. Run `make olddefconfig` inside `vendor/libkrunfw/` so the + Kconfig dependency resolver fills in any newly-required parent + options (e.g. `CONFIG_NETFILTER_ADVANCED=y` once `CONFIG_NF_TABLES=y`) +4. Build libkrunfw as usual +5. Stamp the SONAME to a distinct filename + (`libkrunfw-net.so.5`) so it can sit next to the lean one + +The boxlite runtime stages both blobs and selects between them at VM +spawn time based on `BoxOptions::net`. + +## Why not modify the upstream lean config + +Two reasons. First, every CONFIG flag we add to lean grows the kernel +binary embedded in every box on the planet — that defeats the lean +profile's whole purpose (small footprint, fast boot, smaller attack +surface). Second, keeping the overlay external means upstream libkrunfw +updates (config refreshes, version bumps) merge cleanly without +conflicts in the file we maintain ourselves. + +## Adding a new arch + +`overlay-net_aarch64` is currently a copy of the x86_64 overlay since +the same CONFIG_* knobs apply. If a new arch needs different settings, +add a per-arch overlay here. diff --git a/src/deps/libkrun-sys/net-configs/overlay-net_aarch64 b/src/deps/libkrun-sys/net-configs/overlay-net_aarch64 new file mode 100644 index 000000000..b7c44a088 --- /dev/null +++ b/src/deps/libkrun-sys/net-configs/overlay-net_aarch64 @@ -0,0 +1,116 @@ +# Overlay applied on top of config-libkrunfw_x86_64 by `make libkrunfw-dind`. +# Enables the kernel subsystems Docker needs for bridge networks, NAT, and +# iptables rule installation — the dind / docker-compose workflows broken +# under the lean profile (issue #276). Every line is built-in (=y) because +# the libkrunfw guest has no /lib/modules and can't load loadable modules +# at runtime. + +# ── Bridge networking ───────────────────────────────────────────────────── +# Docker creates one Linux bridge per network (default `docker0` + one per +# user-defined network in compose). Each container is wired in via a veth +# pair (CONFIG_VETH is already on in lean). Bridge netfilter exposes the +# /proc/sys/net/bridge/bridge-nf-* knobs Docker probes for. +CONFIG_BRIDGE=y +CONFIG_BRIDGE_NETFILTER=y +CONFIG_LLC=y +CONFIG_STP=y + +# ── Netfilter core ──────────────────────────────────────────────────────── +# Foundation that everything else (xtables, NAT, conntrack, nftables) sits +# on. CONFIG_NETFILTER_ADVANCED unlocks the dependent options that lean +# can't even reference because the parent is off. +CONFIG_NETFILTER=y +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_LOG_SYSLOG=y + +# ── xtables (iptables backend) ──────────────────────────────────────────── +# Docker's iptables driver writes filter/nat rules through ip_tables / +# ip6_tables which sit on top of the xtables framework. The MATCH/TARGET +# modules cover the rule shapes Docker ships by default. +CONFIG_NETFILTER_XTABLES=y +CONFIG_NETFILTER_XT_MARK=y +CONFIG_NETFILTER_XT_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y +CONFIG_NETFILTER_XT_TARGET_REDIRECT=y +CONFIG_NETFILTER_XT_NAT=y +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_IPVS=y +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=y + +# ── NAT ─────────────────────────────────────────────────────────────────── +# Docker's port publishing (`-p 8080:80`) and outbound MASQUERADE rely on +# the netfilter NAT engine. NF_NAT is the core; the iptables / nftables +# NAT modules expose it through the respective rule shapes. +CONFIG_NF_NAT=y +CONFIG_NF_NAT_REDIRECT=y +CONFIG_NF_NAT_MASQUERADE=y + +# ── iptables (legacy backend) ───────────────────────────────────────────── +# Docker still prefers iptables-legacy on most distros; cover the rule +# tables it manipulates (filter for DOCKER chains, nat for port mapping +# and MASQUERADE). +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_NAT=y +CONFIG_IP_NF_TARGET_MASQUERADE=y +CONFIG_IP_NF_MANGLE=y + +# ── ip6tables (IPv6 iptables) ───────────────────────────────────────────── +# Docker's daemon shows the warning "ip6tables is enabled, but cannot set +# up ip6tables chains" when these are absent. Enabling them silences the +# warning and lets IPv6 docker networks work. +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_NAT=y +CONFIG_IP6_NF_TARGET_MASQUERADE=y +CONFIG_IP6_NF_MANGLE=y + +# ── nftables (modern backend) ───────────────────────────────────────────── +# Docker 24+ on many distros uses nftables for new networks even when +# iptables-legacy is on for old ones. The nf_tables core + ip/ip6 chain +# tables let both legacy and nft-mode Docker work. +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NFT_NAT=y +CONFIG_NFT_MASQ=y +CONFIG_NFT_REDIR=y +CONFIG_NFT_CT=y +CONFIG_NFT_REJECT=y +CONFIG_NFT_CHAIN_NAT=y + +# ── iptables-nft compatibility shim ─────────────────────────────────────── +# On Debian / Alpine the iptables binary that ships in docker:dind is +# iptables-nft (v1.8.x), which translates legacy iptables rules into the +# nftables backend. Without NFT_COMPAT the xt_* match extensions (addrtype, +# conntrack, mark, etc.) can't be used through iptables-nft and dockerd +# fails to register the bridge driver with: +# "Extension addrtype revision 0 not supported, missing kernel module" +# (observed in #276 reproducer logs when this option is off). +CONFIG_NFT_COMPAT=y + +# ── REJECT target ───────────────────────────────────────────────────────── +# Used by the DOCKER-ISOLATION chains to drop cross-network traffic; without +# it dockerd registers the rule but the kernel returns -ENOENT on insertion. +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_NETFILTER_XT_MATCH_STATE=y + +# ── Net namespace forwarding ────────────────────────────────────────────── +# Containers on docker bridge need IP forwarding turned on at the netns +# level. The sysctl /proc/sys/net/ipv4/ip_forward only exists when this +# is built in. +CONFIG_NET_NS=y + +# ── POSIX message queues ────────────────────────────────────────────────── +# Every OCI container created by runc/containerd has /dev/mqueue in its +# default mounts list. Without CONFIG_POSIX_MQUEUE the mount syscall fails +# with ENODEV ("no such device") and container creation aborts before +# entrypoint runs — observed under `docker start` on a dind kernel +# without this enabled. +CONFIG_POSIX_MQUEUE=y diff --git a/src/deps/libkrun-sys/net-configs/overlay-net_x86_64 b/src/deps/libkrun-sys/net-configs/overlay-net_x86_64 new file mode 100644 index 000000000..b7c44a088 --- /dev/null +++ b/src/deps/libkrun-sys/net-configs/overlay-net_x86_64 @@ -0,0 +1,116 @@ +# Overlay applied on top of config-libkrunfw_x86_64 by `make libkrunfw-dind`. +# Enables the kernel subsystems Docker needs for bridge networks, NAT, and +# iptables rule installation — the dind / docker-compose workflows broken +# under the lean profile (issue #276). Every line is built-in (=y) because +# the libkrunfw guest has no /lib/modules and can't load loadable modules +# at runtime. + +# ── Bridge networking ───────────────────────────────────────────────────── +# Docker creates one Linux bridge per network (default `docker0` + one per +# user-defined network in compose). Each container is wired in via a veth +# pair (CONFIG_VETH is already on in lean). Bridge netfilter exposes the +# /proc/sys/net/bridge/bridge-nf-* knobs Docker probes for. +CONFIG_BRIDGE=y +CONFIG_BRIDGE_NETFILTER=y +CONFIG_LLC=y +CONFIG_STP=y + +# ── Netfilter core ──────────────────────────────────────────────────────── +# Foundation that everything else (xtables, NAT, conntrack, nftables) sits +# on. CONFIG_NETFILTER_ADVANCED unlocks the dependent options that lean +# can't even reference because the parent is off. +CONFIG_NETFILTER=y +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_LOG_SYSLOG=y + +# ── xtables (iptables backend) ──────────────────────────────────────────── +# Docker's iptables driver writes filter/nat rules through ip_tables / +# ip6_tables which sit on top of the xtables framework. The MATCH/TARGET +# modules cover the rule shapes Docker ships by default. +CONFIG_NETFILTER_XTABLES=y +CONFIG_NETFILTER_XT_MARK=y +CONFIG_NETFILTER_XT_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y +CONFIG_NETFILTER_XT_TARGET_REDIRECT=y +CONFIG_NETFILTER_XT_NAT=y +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_IPVS=y +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=y + +# ── NAT ─────────────────────────────────────────────────────────────────── +# Docker's port publishing (`-p 8080:80`) and outbound MASQUERADE rely on +# the netfilter NAT engine. NF_NAT is the core; the iptables / nftables +# NAT modules expose it through the respective rule shapes. +CONFIG_NF_NAT=y +CONFIG_NF_NAT_REDIRECT=y +CONFIG_NF_NAT_MASQUERADE=y + +# ── iptables (legacy backend) ───────────────────────────────────────────── +# Docker still prefers iptables-legacy on most distros; cover the rule +# tables it manipulates (filter for DOCKER chains, nat for port mapping +# and MASQUERADE). +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_NAT=y +CONFIG_IP_NF_TARGET_MASQUERADE=y +CONFIG_IP_NF_MANGLE=y + +# ── ip6tables (IPv6 iptables) ───────────────────────────────────────────── +# Docker's daemon shows the warning "ip6tables is enabled, but cannot set +# up ip6tables chains" when these are absent. Enabling them silences the +# warning and lets IPv6 docker networks work. +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_NAT=y +CONFIG_IP6_NF_TARGET_MASQUERADE=y +CONFIG_IP6_NF_MANGLE=y + +# ── nftables (modern backend) ───────────────────────────────────────────── +# Docker 24+ on many distros uses nftables for new networks even when +# iptables-legacy is on for old ones. The nf_tables core + ip/ip6 chain +# tables let both legacy and nft-mode Docker work. +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NFT_NAT=y +CONFIG_NFT_MASQ=y +CONFIG_NFT_REDIR=y +CONFIG_NFT_CT=y +CONFIG_NFT_REJECT=y +CONFIG_NFT_CHAIN_NAT=y + +# ── iptables-nft compatibility shim ─────────────────────────────────────── +# On Debian / Alpine the iptables binary that ships in docker:dind is +# iptables-nft (v1.8.x), which translates legacy iptables rules into the +# nftables backend. Without NFT_COMPAT the xt_* match extensions (addrtype, +# conntrack, mark, etc.) can't be used through iptables-nft and dockerd +# fails to register the bridge driver with: +# "Extension addrtype revision 0 not supported, missing kernel module" +# (observed in #276 reproducer logs when this option is off). +CONFIG_NFT_COMPAT=y + +# ── REJECT target ───────────────────────────────────────────────────────── +# Used by the DOCKER-ISOLATION chains to drop cross-network traffic; without +# it dockerd registers the rule but the kernel returns -ENOENT on insertion. +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_NETFILTER_XT_MATCH_STATE=y + +# ── Net namespace forwarding ────────────────────────────────────────────── +# Containers on docker bridge need IP forwarding turned on at the netns +# level. The sysctl /proc/sys/net/ipv4/ip_forward only exists when this +# is built in. +CONFIG_NET_NS=y + +# ── POSIX message queues ────────────────────────────────────────────────── +# Every OCI container created by runc/containerd has /dev/mqueue in its +# default mounts list. Without CONFIG_POSIX_MQUEUE the mount syscall fails +# with ENODEV ("no such device") and container creation aborts before +# entrypoint runs — observed under `docker start` on a dind kernel +# without this enabled. +CONFIG_POSIX_MQUEUE=y From 6718a8cbe48613bddf5eb22d5f7572d97160cb14 Mon Sep 17 00:00:00 2001 From: gamnaansong Date: Wed, 27 May 2026 14:50:46 +0000 Subject: [PATCH 3/5] test(kernel): capture stderr in kernel_net skip detection + fix node SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kernel_net_has_iptables tried to skip when binary lacks --features kernel-net, but checked stdout while the dependency error surfaces via tracing on stderr — skip never triggered, test hard-failed on default lean builds. Capture both streams; check either for the dependency requirement string. Default build now skips correctly; --features kernel-net build still exercises the assertion. Also fix sdks/node/src/options.rs: BoxOptions gained a kernel field in this PR but the Node SDK's JS-to-Rust conversion still built BoxOptions without it (clippy E0063). Add kernel: None there (Node SDK doesn't yet expose --kernel; runtime defaults to lean). --- sdks/node/src/options.rs | 1 + src/cli/tests/kernel_net_iptables.rs | 28 ++++++++++------- src/deps/libkrun-sys/build.rs | 45 +++++++++++++++++++++------- 3 files changed, 53 insertions(+), 21 deletions(-) diff --git a/sdks/node/src/options.rs b/sdks/node/src/options.rs index 1399ff8ae..ad6f1dabf 100644 --- a/sdks/node/src/options.rs +++ b/sdks/node/src/options.rs @@ -427,6 +427,7 @@ impl TryFrom for BoxOptions { cmd: js_opts.cmd, user: js_opts.user, secrets, + kernel: None, }) } } diff --git a/src/cli/tests/kernel_net_iptables.rs b/src/cli/tests/kernel_net_iptables.rs index 93f673411..121cd636c 100644 --- a/src/cli/tests/kernel_net_iptables.rs +++ b/src/cli/tests/kernel_net_iptables.rs @@ -9,7 +9,10 @@ use assert_cmd::Command; use boxlite_test_utils::home::PerTestBoxHome; use std::time::Duration; -fn run_in_box(home: &PerTestBoxHome, kernel: Option<&str>, script: &str) -> String { +/// Returns (stdout, stderr). Skip detection (--kernel feature not built in) +/// surfaces in stderr because boxlite's failure path logs via tracing, not +/// stdout. +fn run_in_box(home: &PerTestBoxHome, kernel: Option<&str>, script: &str) -> (String, String) { let mut cmd = Command::new(env!("CARGO_BIN_EXE_boxlite")); cmd.arg("--home") .arg(&home.path) @@ -26,27 +29,32 @@ fn run_in_box(home: &PerTestBoxHome, kernel: Option<&str>, script: &str) -> Stri cmd.args(&args); let output = cmd.output().expect("failed to execute boxlite"); - String::from_utf8_lossy(&output.stdout).to_string() + ( + String::from_utf8_lossy(&output.stdout).to_string(), + String::from_utf8_lossy(&output.stderr).to_string(), + ) } #[test] fn kernel_net_has_iptables() { let home = PerTestBoxHome::new(); - let result = run_in_box( + let (stdout, stderr) = run_in_box( &home, Some("net"), "cat /proc/net/ip_tables_names 2>/dev/null && echo IPTABLES_OK || echo NO_IPTABLES", ); - if result.contains("--kernel net requires") { - eprintln!("SKIP: binary not built with kernel-net feature"); + // Binary built without `--features kernel-net` surfaces the dependency + // requirement on stderr via tracing; skip rather than fail. + if stderr.contains("--kernel net requires") || stdout.contains("--kernel net requires") { + eprintln!("SKIP kernel_net_has_iptables: binary not built with kernel-net feature"); return; } assert!( - result.contains("IPTABLES_OK"), - "net kernel must have iptables support; got: {result}" + stdout.contains("IPTABLES_OK"), + "net kernel must have iptables support; got stdout: {stdout}\nstderr: {stderr}" ); } @@ -54,14 +62,14 @@ fn kernel_net_has_iptables() { fn kernel_lean_no_iptables() { let home = PerTestBoxHome::new(); - let result = run_in_box( + let (stdout, stderr) = run_in_box( &home, None, "cat /proc/net/ip_tables_names 2>/dev/null && echo IPTABLES_OK || echo NO_IPTABLES", ); assert!( - result.contains("NO_IPTABLES"), - "lean kernel must NOT have iptables; got: {result}" + stdout.contains("NO_IPTABLES"), + "lean kernel must NOT have iptables; got stdout: {stdout}\nstderr: {stderr}" ); } diff --git a/src/deps/libkrun-sys/build.rs b/src/deps/libkrun-sys/build.rs index 2de6bb9a3..06dcde7b5 100644 --- a/src/deps/libkrun-sys/build.rs +++ b/src/deps/libkrun-sys/build.rs @@ -22,13 +22,15 @@ const LIBKRUNFW_SHA256: &str = "12b9401d7735d1682450e4d025273c5016ec2237dcbfb76b const LIBKRUNFW_LEAN_URL: &str = "https://github.com/boxlite-ai/libkrunfw/releases/download/v5.3.0/libkrunfw-x86_64.tgz"; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] -const LIBKRUNFW_LEAN_SHA256: &str = "0a7bb64a35a273b8501801dd69b75736a8c676aa21aa62fb5642842cda9dc91d"; +const LIBKRUNFW_LEAN_SHA256: &str = + "0a7bb64a35a273b8501801dd69b75736a8c676aa21aa62fb5642842cda9dc91d"; #[cfg(all(target_os = "linux", target_arch = "aarch64"))] const LIBKRUNFW_LEAN_URL: &str = "https://github.com/boxlite-ai/libkrunfw/releases/download/v5.3.0/libkrunfw-aarch64.tgz"; #[cfg(all(target_os = "linux", target_arch = "aarch64"))] -const LIBKRUNFW_LEAN_SHA256: &str = "8b5b9211da5445d9301dafb2201431f4392ab96455512bce63a5cfbd33c49839"; +const LIBKRUNFW_LEAN_SHA256: &str = + "8b5b9211da5445d9301dafb2201431f4392ab96455512bce63a5cfbd33c49839"; // Linux net kernel: lean + netfilter/nf_tables/bridge/NET_NS modules // TODO: move to boxlite-ai/boxlite release assets after merge @@ -36,7 +38,8 @@ const LIBKRUNFW_LEAN_SHA256: &str = "8b5b9211da5445d9301dafb2201431f4392ab964555 const LIBKRUNFW_NET_URL: &str = "https://github.com/G4614/boxlite/releases/download/v0.9.5-kernel-net/libkrunfw-net-x86_64.tgz"; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] -const LIBKRUNFW_NET_SHA256: &str = "f367a6e96ba7f4d11d1837b871c91308d6025ce8dbecee4e5fc914aacf28f128"; +const LIBKRUNFW_NET_SHA256: &str = + "f367a6e96ba7f4d11d1837b871c91308d6025ce8dbecee4e5fc914aacf28f128"; #[cfg(all(target_os = "linux", target_arch = "aarch64"))] const LIBKRUNFW_NET_URL: &str = @@ -235,13 +238,23 @@ fn download_libkrunfw_so(install_dir: &Path) { if has_lean && has_net { // Dual mode: lean as primary, net as secondary let lean_tarball = install_dir.join(format!("libkrunfw-lean-{LIBKRUNFW_VERSION}.tgz")); - Fetcher::fetch(LIBKRUNFW_LEAN_URL, LIBKRUNFW_LEAN_SHA256, &lean_tarball, install_dir) - .unwrap_or_else(|e| panic!("Failed to fetch lean libkrunfw: {}", e)); + Fetcher::fetch( + LIBKRUNFW_LEAN_URL, + LIBKRUNFW_LEAN_SHA256, + &lean_tarball, + install_dir, + ) + .unwrap_or_else(|e| panic!("Failed to fetch lean libkrunfw: {}", e)); if !LIBKRUNFW_NET_SHA256.starts_with("TODO") { let net_tarball = install_dir.join(format!("libkrunfw-net-{LIBKRUNFW_VERSION}.tgz")); - Fetcher::fetch(LIBKRUNFW_NET_URL, LIBKRUNFW_NET_SHA256, &net_tarball, &lib_dir) - .unwrap_or_else(|e| panic!("Failed to fetch net libkrunfw: {}", e)); + Fetcher::fetch( + LIBKRUNFW_NET_URL, + LIBKRUNFW_NET_SHA256, + &net_tarball, + &lib_dir, + ) + .unwrap_or_else(|e| panic!("Failed to fetch net libkrunfw: {}", e)); println!("cargo:warning=Embedded kernels: lean + net (dual mode)"); } else { println!( @@ -258,14 +271,24 @@ fn download_libkrunfw_so(install_dir: &Path) { ); } let tarball = install_dir.join(format!("libkrunfw-net-{LIBKRUNFW_VERSION}.tgz")); - Fetcher::fetch(LIBKRUNFW_NET_URL, LIBKRUNFW_NET_SHA256, &tarball, install_dir) - .unwrap_or_else(|e| panic!("Failed to fetch net libkrunfw: {}", e)); + Fetcher::fetch( + LIBKRUNFW_NET_URL, + LIBKRUNFW_NET_SHA256, + &tarball, + install_dir, + ) + .unwrap_or_else(|e| panic!("Failed to fetch net libkrunfw: {}", e)); println!("cargo:warning=Embedded kernel: net only"); } else { // Lean only (default) let tarball = install_dir.join(format!("libkrunfw-lean-{LIBKRUNFW_VERSION}.tgz")); - Fetcher::fetch(LIBKRUNFW_LEAN_URL, LIBKRUNFW_LEAN_SHA256, &tarball, install_dir) - .unwrap_or_else(|e| panic!("Failed to fetch lean libkrunfw: {}", e)); + Fetcher::fetch( + LIBKRUNFW_LEAN_URL, + LIBKRUNFW_LEAN_SHA256, + &tarball, + install_dir, + ) + .unwrap_or_else(|e| panic!("Failed to fetch lean libkrunfw: {}", e)); println!("cargo:warning=Embedded kernel: lean (default)"); } From ee5ee4ae7170db02e172259739ea4158f86bcf0d Mon Sep 17 00:00:00 2001 From: gamnaansong Date: Thu, 28 May 2026 07:27:58 +0000 Subject: [PATCH 4/5] feat(kernel): build-your-own-kernel via make libkrunfw-custom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses @DorianZheng's #596 review: users should be able to compile their own kernel, not just pick the built-in lean/net presets. The load side already accepts arbitrary blobs (`--kernel ` → stage_custom_kernel symlinks + dlopens at runtime, no rebuild). This adds the build side: - Extract the net build into a generalized `scripts/build/build-libkrunfw.sh` parameterized by OVERLAY / KCONFIG / SONAME / OUT, with a DRY_RUN mode that validates the overlay + config merge without the ~10-20 min build. - `build-libkrunfw-net.sh` becomes a thin wrapper pinning the net overlay, the distinct `libkrunfw-net.so.5` SONAME, and the canonical embed path (behavior unchanged — DRY_RUN confirms identical resolved config/paths). - `make libkrunfw-custom OVERLAY=... [OUT=...]` builds a blob with the default `libkrunfw.so.5` SONAME; load it with `boxlite run --kernel `. - README documents the custom workflow. Validated via DRY_RUN (net delegation + custom + missing-OVERLAY error); real kernel build not run. Co-Authored-By: Claude Opus 4.7 (1M context) --- make/build.mk | 20 +++- scripts/build/build-libkrunfw-net.sh | 129 ++++----------------- scripts/build/build-libkrunfw.sh | 129 +++++++++++++++++++++ src/deps/libkrun-sys/net-configs/README.md | 29 +++++ 4 files changed, 200 insertions(+), 107 deletions(-) create mode 100755 scripts/build/build-libkrunfw.sh diff --git a/make/build.mk b/make/build.mk index 1566f2685..7d5b15e82 100644 --- a/make/build.mk +++ b/make/build.mk @@ -1,4 +1,4 @@ -PHONY_TARGETS += guest shim runtime cli cli\:release skillbox-image build\:apps libkrunfw-net +PHONY_TARGETS += guest shim runtime cli cli\:release skillbox-image build\:apps libkrunfw-net libkrunfw-custom guest: @bash $(SCRIPT_DIR)/build/build-guest.sh @@ -52,6 +52,24 @@ build\:apps: _ensure-apps-deps libkrunfw-net: @bash $(SCRIPT_DIR)/build/build-libkrunfw-net.sh +# Build a libkrunfw kernel from your OWN config overlay (custom modules). +# Same machinery as libkrunfw-net, but you supply the overlay: +# +# OVERLAY=/path/to/my-overlay make libkrunfw-custom +# [OUT=/path/to/libkrunfw-custom.so.5] # default: target/custom-kernel/lib64/ +# +# The overlay is a file of `CONFIG_*=y` lines appended on top of the lean +# config (see src/deps/libkrun-sys/net-configs/README.md). Unlike --kernel +# net, the result is NOT embedded — load it at runtime with no rebuild: +# +# boxlite run --kernel ... +# +# DRY_RUN=1 validates the overlay + config merge without the (~10-20 min) build. +# Heavy target; not in any other target's dep chain. +libkrunfw-custom: + @OVERLAY="$(OVERLAY)" OUT="$(OUT)" DRY_RUN="$(DRY_RUN)" \ + bash $(SCRIPT_DIR)/build/build-libkrunfw.sh + # Build SkillBox container image (all-in-one AI CLI with noVNC) # Usage: make skillbox-image [APT_SOURCE=mirrors.aliyun.com] skillbox-image: diff --git a/scripts/build/build-libkrunfw-net.sh b/scripts/build/build-libkrunfw-net.sh index 1753de113..e28b4612a 100755 --- a/scripts/build/build-libkrunfw-net.sh +++ b/scripts/build/build-libkrunfw-net.sh @@ -1,122 +1,39 @@ #!/usr/bin/env bash -# Build the "fat" libkrunfw variant required by `boxlite run --net`. +# Build the built-in "fat" libkrunfw variant required by `boxlite run --kernel net`. # -# Steps: -# 1. Resolve target arch (overlay + libkrunfw config are arch-specific) -# 2. Ensure the libkrunfw submodule is checked out -# 3. Build the lean config + the net overlay → patched .config -# 4. Run `make` inside vendor/libkrunfw to produce a kernel .so -# 5. Rename SONAME to libkrunfw-net.so.5 and copy to -# target/net-kernel/lib64/ — libkrun-sys/build.rs auto-detects this -# canonical path on the next `make runtime:debug` / `cargo build`. Set -# BOXLITE_LIBKRUNFW_NET_PATH only when the blob lives outside the -# workspace (e.g., CI cache, distro packaging). +# Thin wrapper over build-libkrunfw.sh: it pins the net overlay, a distinct +# SONAME (so the net blob can sit next to the lean one in the embedded runtime +# without a dlopen identity collision), and the canonical output path that +# `libkrun-sys/build.rs` auto-detects and embeds on the next `make cli`. # -# Why not download a prebuilt: the net kernel adds ~2 MB of network -# subsystems on top of the lean kernel. Until upstream boxlite-ai/libkrunfw -# starts publishing the net variant in its releases, anyone iterating on -# `--net` needs to build it locally. +# Why not download a prebuilt: the net kernel adds ~2 MB of network subsystems +# on top of the lean kernel. Until upstream boxlite-ai/libkrunfw publishes the +# net variant in its releases, anyone iterating on `--kernel net` builds it +# locally. (Set BOXLITE_LIBKRUNFW_NET_PATH only when the blob lives outside the +# workspace — e.g. CI cache, distro packaging.) +# +# To build a kernel with your OWN config instead, see `make libkrunfw-custom` +# (src/deps/libkrun-sys/net-configs/README.md). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -LIBKRUNFW_SRC="$REPO_ROOT/src/deps/libkrun-sys/vendor/libkrunfw" -OVERLAY_DIR="$REPO_ROOT/src/deps/libkrun-sys/net-configs" -OUT_DIR="$REPO_ROOT/target/net-kernel/lib64" - ARCH="${ARCH:-$(uname -m)}" case "$ARCH" in - x86_64) KCONFIG_NAME="config-libkrunfw_x86_64"; OVERLAY_NAME="overlay-net_x86_64" ;; - aarch64|arm64) KCONFIG_NAME="config-libkrunfw_aarch64"; OVERLAY_NAME="overlay-net_aarch64" ;; + x86_64) OVERLAY_NAME="overlay-net_x86_64" ;; + aarch64|arm64) OVERLAY_NAME="overlay-net_aarch64" ;; *) echo "❌ unsupported ARCH=$ARCH (only x86_64 and aarch64 have a net overlay yet)" >&2; exit 1 ;; esac -# ── Sanity ────────────────────────────────────────────────────────────────── - -if [ ! -f "$LIBKRUNFW_SRC/Makefile" ]; then - echo "❌ libkrunfw submodule not initialised at $LIBKRUNFW_SRC" >&2 - echo " Run: git submodule update --init --recursive src/deps/libkrun-sys/vendor/libkrunfw" >&2 - exit 1 -fi - -LEAN_CONFIG="$LIBKRUNFW_SRC/$KCONFIG_NAME" -OVERLAY="$OVERLAY_DIR/$OVERLAY_NAME" -if [ ! -f "$LEAN_CONFIG" ]; then - echo "❌ lean config not found: $LEAN_CONFIG" >&2; exit 1 -fi -if [ ! -f "$OVERLAY" ]; then - echo "❌ net overlay not found: $OVERLAY" >&2; exit 1 -fi - -# ── Build a merged config in a tempfile ───────────────────────────────────── -# -# Append the overlay to the lean config. The overlay's `CONFIG_X=y` lines -# OVERRIDE the lean config's `# CONFIG_X is not set` lines because Kconfig -# parses sequentially. `make olddefconfig` (run by libkrunfw's own Makefile) -# fills in any dependent options that newly-enabled parents require. -MERGED_CONFIG="$(mktemp)" -trap 'rm -f "$MERGED_CONFIG"' EXIT -cat "$LEAN_CONFIG" "$OVERLAY" > "$MERGED_CONFIG" - -# Replace the lean config in place so libkrunfw's Makefile (which always -# reads $KCONFIG_NAME) picks up the merged one. Restore on exit so the -# normal lean build path isn't permanently changed. -LEAN_BACKUP="$(mktemp)" -# Restore BEFORE removing the backup — the previous order rm'd the backup -# first, then `cp -f $LEAN_BACKUP $LEAN_CONFIG || true` silently no-op'd -# on the missing source, leaving the submodule's `config-libkrunfw_*` -# permanently polluted with the overlay's +116 lines after every run. -trap 'cp -f "$LEAN_BACKUP" "$LEAN_CONFIG" 2>/dev/null || true; rm -f "$MERGED_CONFIG" "$LEAN_BACKUP"' EXIT -cp "$LEAN_CONFIG" "$LEAN_BACKUP" -cp "$MERGED_CONFIG" "$LEAN_CONFIG" - -# ── Build ─────────────────────────────────────────────────────────────────── - -echo "🔨 Building libkrunfw with net overlay ($ARCH)..." -echo " lean cfg: $LEAN_CONFIG" -echo " overlay: $OVERLAY" -echo " merged size: $(wc -l < "$MERGED_CONFIG") lines" - -cd "$LIBKRUNFW_SRC" -# `make` here triggers the full upstream libkrunfw build: -# - downloads kernel.org tarball if missing -# - applies libkrunfw patches -# - copies our merged $KCONFIG_NAME to linux-/.config -# - runs make olddefconfig + bzImage -# - bundles into libkrunfw.so. -make -j"$(nproc)" MAKEFLAGS="" - -# ── Stage the result ──────────────────────────────────────────────────────── - -mkdir -p "$OUT_DIR" - -# libkrunfw's Makefile produces libkrunfw.so.5.. (e.g. -# libkrunfw.so.5.3.0) with a symlink chain libkrunfw.so.5 → libkrunfw.so.5.3.0. -# We copy the real file and rename it to libkrunfw-net.so.5 so it can sit -# next to the lean libkrunfw.so.5 in the runtime dir without a name collision. -REAL_BLOB=$(ls "$LIBKRUNFW_SRC"/libkrunfw.so.5.* 2>/dev/null | head -1 || true) -if [ -z "$REAL_BLOB" ]; then - echo "❌ build succeeded but couldn't find libkrunfw.so.5.* in $LIBKRUNFW_SRC" >&2 - exit 1 -fi +OVERLAY="$REPO_ROOT/src/deps/libkrun-sys/net-configs/$OVERLAY_NAME" \ +SONAME="libkrunfw-net.so.5" \ +OUT="$REPO_ROOT/target/net-kernel/lib64/libkrunfw-net.so.5" \ +HINT="Rebuild boxlite to embed it (libkrun-sys/build.rs auto-detects this path): -NET_BLOB="$OUT_DIR/libkrunfw-net.so.5" -cp "$REAL_BLOB" "$NET_BLOB" -# Rename SONAME so two side-by-side blobs (lean + net) don't both report -# the same identity to dlopen, which would defeat per-box selection. -patchelf --set-soname libkrunfw-net.so.5 "$NET_BLOB" + make cli -echo "" -echo "✅ Built net libkrunfw: $NET_BLOB" -echo " Size: $(du -h "$NET_BLOB" | cut -f1) (vs lean: $(du -h "$REAL_BLOB" 2>/dev/null | cut -f1 || echo '?'))" -echo "" -echo "Rebuild boxlite to embed it — libkrun-sys/build.rs auto-detects this path:" -echo "" -echo " make cli" -echo "" -echo "Then \`boxlite run --net\` will load this kernel instead of the lean one." -echo "" -echo "(Set BOXLITE_LIBKRUNFW_NET_PATH only if the blob lives outside the" -echo " workspace — e.g., CI cache, packaging sysroot.)" +Then \`boxlite run --kernel net\` loads this kernel instead of the lean one." \ +ARCH="$ARCH" \ + exec bash "$SCRIPT_DIR/build-libkrunfw.sh" diff --git a/scripts/build/build-libkrunfw.sh b/scripts/build/build-libkrunfw.sh new file mode 100755 index 000000000..a23209089 --- /dev/null +++ b/scripts/build/build-libkrunfw.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Build a custom libkrunfw kernel blob from a config overlay. +# +# This is the generalized core behind `make libkrunfw-net` (the built-in +# "net" variant) and `make libkrunfw-custom` (user-supplied overlay). It +# merges a config overlay on top of the upstream lean libkrunfw config, +# builds the kernel, stamps a SONAME, and stages the resulting `.so` at a +# chosen path. +# +# A user-built blob is loaded at runtime with `boxlite run --kernel ` +# (no rebuild needed — the runtime symlinks it into the box's libs dir and +# dlopens it). The "net" variant instead stamps a distinct SONAME and lands +# at the canonical path that `libkrun-sys/build.rs` embeds into the runtime. +# +# Parameters (all via env): +# OVERLAY path to a config overlay to append on top of the lean config. +# Required — it's what makes the kernel non-lean. +# KCONFIG base libkrunfw config NAME under vendor/libkrunfw/ +# (default: arch lean `config-libkrunfw_`). +# SONAME ELF SONAME stamped on the output (default: `libkrunfw.so.5`, +# which is what `--kernel ` expects). +# OUT output blob path +# (default: target/custom-kernel/lib64/libkrunfw-custom.so.5). +# HINT optional one-line "next step" message printed after the build. +# DRY_RUN if set, resolve + validate + merge the config and print the +# plan, but skip the (~10-20 min) kernel build and staging. +# ARCH override target arch (default: uname -m). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +LIBKRUNFW_SRC="$REPO_ROOT/src/deps/libkrun-sys/vendor/libkrunfw" + +ARCH="${ARCH:-$(uname -m)}" +case "$ARCH" in + x86_64) DEFAULT_KCONFIG="config-libkrunfw_x86_64" ;; + aarch64|arm64) DEFAULT_KCONFIG="config-libkrunfw_aarch64" ;; + *) echo "❌ unsupported ARCH=$ARCH" >&2; exit 1 ;; +esac + +KCONFIG_NAME="${KCONFIG:-$DEFAULT_KCONFIG}" +SONAME="${SONAME:-libkrunfw.so.5}" +OUT="${OUT:-$REPO_ROOT/target/custom-kernel/lib64/libkrunfw-custom.so.5}" +OVERLAY="${OVERLAY:-}" + +# ── Sanity ────────────────────────────────────────────────────────────────── + +if [ -z "$OVERLAY" ]; then + echo "❌ OVERLAY is required (path to a config overlay to append)." >&2 + echo " Write a file of CONFIG_*=y lines for the subsystems you need," >&2 + echo " then: OVERLAY=/path/to/overlay make libkrunfw-custom" >&2 + exit 1 +fi +if [ ! -f "$OVERLAY" ]; then + echo "❌ overlay not found: $OVERLAY" >&2; exit 1 +fi + +if [ ! -f "$LIBKRUNFW_SRC/Makefile" ]; then + echo "❌ libkrunfw submodule not initialised at $LIBKRUNFW_SRC" >&2 + echo " Run: git submodule update --init --recursive src/deps/libkrun-sys/vendor/libkrunfw" >&2 + exit 1 +fi + +LEAN_CONFIG="$LIBKRUNFW_SRC/$KCONFIG_NAME" +if [ ! -f "$LEAN_CONFIG" ]; then + echo "❌ base config not found: $LEAN_CONFIG" >&2; exit 1 +fi + +# ── Merge overlay onto the lean config ────────────────────────────────────── +# +# Append the overlay to the lean config. The overlay's `CONFIG_X=y` lines +# OVERRIDE the lean config's `# CONFIG_X is not set` lines because Kconfig +# parses sequentially. `make olddefconfig` (run by libkrunfw's Makefile) +# fills in any dependent options the newly-enabled parents require. +MERGED_CONFIG="$(mktemp)" +trap 'rm -f "$MERGED_CONFIG"' EXIT +cat "$LEAN_CONFIG" "$OVERLAY" > "$MERGED_CONFIG" + +echo "🔧 libkrunfw build plan ($ARCH)" +echo " base config: $LEAN_CONFIG" +echo " overlay: $OVERLAY" +echo " merged size: $(wc -l < "$MERGED_CONFIG") lines" +echo " soname: $SONAME" +echo " output: $OUT" + +if [ -n "${DRY_RUN:-}" ]; then + echo "🟡 DRY_RUN set — validated config merge, skipping kernel build + staging." + exit 0 +fi + +# Swap the merged config into the submodule's config in place so libkrunfw's +# Makefile (which always reads $KCONFIG_NAME) picks it up. Restore on exit so +# the lean build path isn't permanently polluted with the overlay's lines. +LEAN_BACKUP="$(mktemp)" +trap 'cp -f "$LEAN_BACKUP" "$LEAN_CONFIG" 2>/dev/null || true; rm -f "$MERGED_CONFIG" "$LEAN_BACKUP"' EXIT +cp "$LEAN_CONFIG" "$LEAN_BACKUP" +cp "$MERGED_CONFIG" "$LEAN_CONFIG" + +# ── Build ─────────────────────────────────────────────────────────────────── + +echo "🔨 Building libkrunfw (this downloads kernel source on first run, ~10-20 min)..." +cd "$LIBKRUNFW_SRC" +make -j"$(nproc)" MAKEFLAGS="" + +# ── Stage the result ──────────────────────────────────────────────────────── + +OUT_DIR="$(dirname "$OUT")" +mkdir -p "$OUT_DIR" + +# libkrunfw's Makefile produces libkrunfw.so.5.. with a symlink +# chain libkrunfw.so.5 → it. Copy the real file and stamp the requested SONAME. +REAL_BLOB=$(ls "$LIBKRUNFW_SRC"/libkrunfw.so.5.* 2>/dev/null | head -1 || true) +if [ -z "$REAL_BLOB" ]; then + echo "❌ build succeeded but couldn't find libkrunfw.so.5.* in $LIBKRUNFW_SRC" >&2 + exit 1 +fi + +cp "$REAL_BLOB" "$OUT" +patchelf --set-soname "$SONAME" "$OUT" + +echo "" +echo "✅ Built libkrunfw: $OUT" +echo " Size: $(du -h "$OUT" | cut -f1) (vs lean: $(du -h "$REAL_BLOB" 2>/dev/null | cut -f1 || echo '?'))" +if [ -n "${HINT:-}" ]; then + echo "" + echo "$HINT" +fi diff --git a/src/deps/libkrun-sys/net-configs/README.md b/src/deps/libkrun-sys/net-configs/README.md index 989724469..dc5e63506 100644 --- a/src/deps/libkrun-sys/net-configs/README.md +++ b/src/deps/libkrun-sys/net-configs/README.md @@ -45,3 +45,32 @@ conflicts in the file we maintain ourselves. `overlay-net_aarch64` is currently a copy of the x86_64 overlay since the same CONFIG_* knobs apply. If a new arch needs different settings, add a per-arch overlay here. + +## Building a kernel with your own config + +`overlay-net_` is just one overlay. To build a libkrunfw kernel +with your OWN modules, write an overlay (a file of `CONFIG_*=y` lines) +and run: + +``` +OVERLAY=/path/to/my-overlay make libkrunfw-custom +# → target/custom-kernel/lib64/libkrunfw-custom.so.5 (override with OUT=) +``` + +Then load it at runtime — no boxlite rebuild needed (the runtime symlinks +the blob into the box's libs dir and dlopens it): + +``` +boxlite run --kernel target/custom-kernel/lib64/libkrunfw-custom.so.5 ... +``` + +This differs from `--kernel net`, which stamps a distinct SONAME and lands +at the canonical path embedded into the runtime by `libkrun-sys/build.rs`. +A custom blob keeps the default `libkrunfw.so.5` SONAME and is loaded by +path, so it is never embedded. + +`DRY_RUN=1 OVERLAY=... make libkrunfw-custom` validates the overlay and the +config merge without the (~10-20 min) kernel build. + +`make libkrunfw-net` is the same machinery with the net overlay pinned; +both delegate to `scripts/build/build-libkrunfw.sh`. From 1e94a09da66bead194056bda48eb929b03b6133a Mon Sep 17 00:00:00 2001 From: gamnaansong Date: Thu, 28 May 2026 07:43:40 +0000 Subject: [PATCH 5/5] docs(cli): advertise --kernel for custom kernels in help The path arm already worked (apply_to's `_` => opts.kernel = Some(k)), but the flag help only mentioned `net`, so the public custom-kernel capability was undiscoverable. Update the doc + value_name to `net|PATH` and point at `make libkrunfw-custom`. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/cli/src/cli.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cli/src/cli.rs b/src/cli/src/cli.rs index 3fd1096bf..e607d72d4 100644 --- a/src/cli/src/cli.rs +++ b/src/cli/src/cli.rs @@ -368,10 +368,12 @@ pub struct ResourceFlags { #[arg(long)] pub memory: Option, - /// Use the net kernel (netfilter/bridge modules) instead of the - /// default lean kernel. The binary must be built with - /// `--features kernel-net` to embed the net kernel blob. - #[arg(long = "kernel", value_name = "VARIANT")] + /// Select the guest kernel. `net` uses the embedded net kernel + /// (netfilter/bridge modules; the binary must be built with + /// `--features kernel-net`). A file path loads a custom libkrunfw + /// blob at runtime — build one with `make libkrunfw-custom`. Omit + /// for the default lean kernel. + #[arg(long = "kernel", value_name = "net|PATH")] pub kernel: Option, }