From 5cb2ca8c29f1db6a46a17cb2de48d22671ba09a3 Mon Sep 17 00:00:00 2001 From: Sudipta Pandit Date: Thu, 4 Jun 2026 20:38:56 +0530 Subject: [PATCH] test(images): add container runtime test support Add Podman-backed pytest fixtures for resolving container images, creating per-test containers, and executing commands against them. Move existing image tests under static cases and add runtime coverage for container-base, including nginx custom image validation. Document running the container test suite and register the runtime capability in image metadata. --- base/images/images.toml | 27 +- base/images/tests/README.md | 163 +++++++-- base/images/tests/cases/__init__.py | 0 .../tests/cases/container-base/__init__.py | 0 .../runtime/container-base/test_basic.py | 27 ++ .../container-base/test_nginx/Dockerfile | 5 + .../container-base/test_nginx/nginx.conf | 27 ++ .../container-base/test_nginx/test_nginx.py | 45 +++ .../container-base/test_container.py | 0 .../cases/{ => static}/test_os_release.py | 0 .../tests/cases/{ => static}/test_packages.py | 0 .../cases/{ => static}/vm-base/test_kernel.py | 0 .../{ => static}/vm-base/test_partitions.py | 0 base/images/tests/cases/vm-base/__init__.py | 0 base/images/tests/conftest.py | 194 ++++++++++- base/images/tests/pyproject.toml | 1 + base/images/tests/utils/container_runtime.py | 317 ++++++++++++++++++ base/images/tests/utils/pytest_plugin.py | 97 ++++-- 18 files changed, 831 insertions(+), 72 deletions(-) delete mode 100644 base/images/tests/cases/__init__.py delete mode 100644 base/images/tests/cases/container-base/__init__.py create mode 100644 base/images/tests/cases/runtime/container-base/test_basic.py create mode 100644 base/images/tests/cases/runtime/container-base/test_nginx/Dockerfile create mode 100644 base/images/tests/cases/runtime/container-base/test_nginx/nginx.conf create mode 100644 base/images/tests/cases/runtime/container-base/test_nginx/test_nginx.py rename base/images/tests/cases/{ => static}/container-base/test_container.py (100%) rename base/images/tests/cases/{ => static}/test_os_release.py (100%) rename base/images/tests/cases/{ => static}/test_packages.py (100%) rename base/images/tests/cases/{ => static}/vm-base/test_kernel.py (100%) rename base/images/tests/cases/{ => static}/vm-base/test_partitions.py (100%) delete mode 100644 base/images/tests/cases/vm-base/__init__.py create mode 100644 base/images/tests/utils/container_runtime.py diff --git a/base/images/images.toml b/base/images/images.toml index b454c4237ec..e4ce97cb286 100644 --- a/base/images/images.toml +++ b/base/images/images.toml @@ -69,7 +69,10 @@ runtime-package-management = true [images.container-base] description = "Container Base Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "core" } -tests.test-suites = [{ name = "static-image-checks" }] +tests.test-suites = [ + { name = "static-image-checks" }, + { name = "runtime-container-tests" }, +] [images.container-base.capabilities] machine-bootable = false @@ -80,7 +83,10 @@ runtime-package-management = true [images.container-base-dev] description = "Container Base Image (dev)" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "core-dev" } -tests.test-suites = [{ name = "static-image-checks" }] +tests.test-suites = [ + { name = "static-image-checks" }, + { name = "runtime-container-tests" }, +] [images.container-base-dev.capabilities] machine-bootable = false @@ -160,7 +166,7 @@ description = "Offline image validation (shared + image-specific tests)" [test-suites.static-image-checks.pytest] working-dir = "tests" install = "pyproject" -test-paths = ["cases/"] +test-paths = ["cases/static/"] # {capabilities} is substituted by azldev as a comma-separated list of # the names of capabilities set to `true` for the image (e.g. # "machine-bootable,systemd,runtime-package-management"). Names use @@ -172,3 +178,18 @@ extra-args = [ "--image-name", "{image-name}", "--capabilities", "{capabilities}", ] + +# Runtime container tests — validate live container behavior via podman. +[test-suites.runtime-container-tests] +type = "pytest" +description = "Runtime container validation (exec into live containers)" + +[test-suites.runtime-container-tests.pytest] +working-dir = "tests" +install = "pyproject" +test-paths = ["cases/runtime/"] +extra-args = [ + "--image-path", "{image-path}", + "--image-name", "{image-name}", + "--capabilities", "{capabilities}", +] diff --git a/base/images/tests/README.md b/base/images/tests/README.md index ea1202d6e68..92c4489e1dc 100644 --- a/base/images/tests/README.md +++ b/base/images/tests/README.md @@ -1,13 +1,13 @@ # Azure Linux Image Tests -Static validation framework for built Azure Linux images (VM and container). -Mounts images read-only and runs pytest tests against the filesystem -without booting. +Validation framework for built Azure Linux images (VM and container). +Includes both static (offline filesystem) and runtime (live container) +tests, all driven by pytest. ## How it gets invoked -These tests are wired into `azldev` via the `[test-suites.static-image-checks]` -table in `base/images/images.toml`, and referenced by each image's +These tests are wired into `azldev` via the `[test-suites.*]` tables +in `base/images/images.toml`, and referenced by each image's `tests.test-suites`. The standard entry point is: ```bash @@ -30,22 +30,41 @@ where present — to validate the dev variant.) `pyproject.toml`, and invokes pytest with the right `--image-path`, `--image-name`, and `--capabilities` arguments. +## Test suites + +| Suite | Description | Runs for | +|-------|-------------|----------| +| `static-image-checks` | Offline filesystem validation — mounts images read-only | All images | +| `runtime-container-tests` | Live container tests via `podman exec` | Container images | + ## Direct (manual) invocation ```bash cd base/images/tests -# VM image — shared + VM-specific tests -uv run pytest cases/ \ +# Static tests — VM image +uv run pytest cases/static/ \ --image-path /path/to/image.raw \ --image-name vm-base \ --capabilities machine-bootable,systemd,runtime-package-management -# Container image — shared + container-specific tests -uv run pytest cases/ \ +# Static tests — Container image +uv run pytest cases/static/ \ + --image-path /path/to/image.oci.tar.xz \ + --image-name container-base \ + --capabilities container,runtime-package-management + +# Runtime tests — Container image (requires podman) +uv run pytest cases/runtime/ \ --image-path /path/to/image.oci.tar.xz \ --image-name container-base \ --capabilities container,runtime-package-management + +# Runtime tests — from a registry reference +uv run pytest cases/runtime/ \ + --image-ref mcr.microsoft.com/azurelinux/base/core:4.0 \ + --image-name container-base \ + --capabilities container,runtime-package-management ``` Test selection follows standard pytest positional arguments. Tests @@ -67,12 +86,16 @@ System packages (not pip-installable): - **`libguestfs-tools`** + **`guestfs-tools`** — `guestmount`, `guestunmount`, `virt-inspector` (VM images) -- **`skopeo`** — OCI archive conversion (container images) -- **`umoci`** — OCI image unpacking (container images) -- **`buildah`** — cleanup of rootless umoci extracts (container images) +- **`skopeo`** — OCI archive conversion (container images, static tests) +- **`umoci`** — OCI image unpacking (container images, static tests) +- **`buildah`** — cleanup of rootless umoci extracts (container images, static tests) +- **`podman`** — container runtime for live tests (container images, runtime tests) - **`rpm`** — for `rpm --root` package queries - **`uv`** — Python project/package manager +Runtime tests use `python-on-whales` to drive the Podman CLI directly; +the Podman REST API socket is not required. + `pytest_configure` does a preflight check and fails fast if any tool needed for the current `--image-type` is missing. @@ -82,30 +105,40 @@ needed for the current `--image-type` is missing. base/images/ ├── images.toml # Image registry + test-suite wiring └── tests/ - ├── pyproject.toml # uv project: pytest, plugin entry point - ├── conftest.py # Session fixtures + ├── pyproject.toml # uv project: pytest + python-on-whales deps + ├── conftest.py # Session fixtures (static + runtime) ├── utils/ # Helper package (not test-collected) │ ├── pytest_plugin.py # CLI options, markers, tool preflight + │ ├── container_runtime.py # python-on-whales based container orchestration │ ├── extract.py # Image mounting / extraction │ ├── disk.py # virt-inspector → DiskInfo │ ├── parsers.py # File content parsers │ ├── types.py # Dataclasses │ └── tools.py # Native-tool registry └── cases/ # Test cases - ├── test_os_release.py # Shared: /etc/os-release - ├── test_packages.py # Shared: rpm-db checks (capability-gated) - ├── vm-base/ # VM-specific tests (auto-restricted to the vm-base family — vm-base, vm-base-dev) - │ ├── test_kernel.py - │ └── test_partitions.py - └── container-base/ # Container-specific tests (auto-restricted to the container-base family) - └── test_container.py + ├── static/ # Offline filesystem tests + │ ├── test_os_release.py # Shared: /etc/os-release + │ ├── test_packages.py # Shared: rpm-db checks (capability-gated) + │ ├── vm-base/ # VM-specific static tests + │ │ ├── test_kernel.py + │ │ └── test_partitions.py + │ └── container-base/ # Container-specific static tests + │ └── test_container.py + └── runtime/ # Live container tests (via podman exec) + └── container-base/ + ├── test_basic.py # Basic: shell access, DNS resolution + └── test_nginx/ # Dockerfile test example + ├── test_nginx.py # Test logic + ├── Dockerfile # Custom image (ARG BASE_IMAGE) + └── nginx.conf # Supporting files ``` ## Available fixtures | Fixture | Scope | Type | Description | |---------|-------|------|-------------| -| `image_path` | session | `Path` | From `--image-path` | +| `image_path` | session | `Path \| None` | From `--image-path` (None when `--image-ref` used) | +| `image_ref` | session | `str \| None` | From `--image-ref` (None when `--image-path` used) | | `image_name` | session | `str \| None` | From `--image-name` | | `image_type` | session | `str` | `"vm"` or `"container"` (explicit / capabilities / extension) | | `capabilities` | session | `set[str]` | Parsed `--capabilities` | @@ -115,20 +148,84 @@ base/images/ | `installed_packages` | session | `set[str]` | Installed RPM names (`rpm --root`) | | `disk_info` | session | `DiskInfo \| None` | VM only | | `partition_table` | session | `list[PartitionInfo]` | VM only — auto-skips on containers | +| `podman_client` | session | `DockerClient \| None` | python-on-whales Podman client; None for non-container images | +| `container_image_ref` | session | `str \| None` | Loaded image ID (cached); None for non-container | +| `running_container` | function | `ContainerInstance` | Fresh container per test — auto-skips on VMs | +| `container_exec_shell` | function | callable | `(cmd, shell="bash") → ContainerExecResult` | +| `container_exec` | function | callable | `(args) → ContainerExecResult` | ## Adding tests -- **Shared (every image):** add a `cases/test_.py`. Use - `@pytest.mark.require_capability("…")` if the test only applies to - images with a given capability. -- **Image-specific:** add `cases//test_.py`. Tests - in such subdirectories are **automatically** restricted to that - image family (the plugin applies `@pytest.mark.image("")` - during collection — no boilerplate per file or per subdir). The - directory name is treated as a *family*: an `--image-name` matches - the family if it equals the family exactly OR has the form - `-` (so `cases/vm-base/` runs for both `vm-base` - and `vm-base-dev`). +- **Shared static (every image):** add a `cases/static/test_.py`. Use + `@pytest.mark.require_capability("…")` if the test only applies to + images with a given capability. +- **Image-specific static:** add `cases/static//test_.py`. + Tests in such subdirectories are **automatically** restricted to that + image family (the plugin applies `@pytest.mark.image("")` + during collection — no boilerplate per file or per subdir). The + directory name is treated as a *family*: an `--image-name` matches + the family if it equals the family exactly OR has the form + `-` (so `cases/static/vm-base/` runs for both + `vm-base` and `vm-base-dev`). +- **Shared runtime (every container):** add a `cases/runtime/test_.py`. + Use `container_exec_shell("...")` for normal runtime tests. Use + `container_exec([...])` only when the test must avoid a shell, such as + distroless or minimal images. Tests are auto-marked with + `@pytest.mark.runtime`. +- **Image-specific runtime:** add `cases/runtime//test_.py`. + +### Dockerfile-based runtime tests + +When a runtime test needs packages or config beyond what the base image +ships, give it its own directory with a `Dockerfile`: + +``` +cases/runtime/container-base/test_nginx/ + test_nginx.py # test logic + Dockerfile # builds on top of the image-under-test + nginx.conf # supporting files (COPY'd in Dockerfile) +``` + +The Dockerfile must use `ARG BASE_IMAGE` / `FROM ${BASE_IMAGE}` — the +framework injects the image-under-test automatically: + +```dockerfile +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +RUN dnf install -y nginx && dnf clean all +COPY nginx.conf /etc/nginx/nginx.conf +``` + +Mark tests with `@pytest.mark.dockerfile()` to trigger the build. The +marker optionally accepts a path relative to the test file's directory +(defaults to `Dockerfile` in the same directory): + +```python +# Auto-discovers Dockerfile in the same directory +@pytest.mark.dockerfile() +def test_nginx_config(container_exec_shell): + result = container_exec_shell("nginx -t") + assert result.exit_code == 0 + +# Explicit path to a different Dockerfile +@pytest.mark.dockerfile("alt/Dockerfile.debug") +def test_debug_variant(container_exec_shell): + ... +``` + +Built images are cached per session — multiple tests sharing the same +Dockerfile only trigger one build. + +> **Note:** All containers (plain and Dockerfile-based) run with +> `sleep infinity` as PID 1 — the Dockerfile's `CMD`/`ENTRYPOINT` is +> overridden. Tests that need a service should start it explicitly via +> `container_exec_shell("nginx")`. This keeps behaviour predictable and +> ensures each test controls exactly what runs. + +For service tests, poll readiness with a short bounded loop before asserting on +responses; do not assume the service binds synchronously. Foreground services +should be backgrounded explicitly, for example +`container_exec_shell("nohup my-service > /tmp/my-service.log 2>&1 &")`. ## Adding a native-tool dependency diff --git a/base/images/tests/cases/__init__.py b/base/images/tests/cases/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/base/images/tests/cases/container-base/__init__.py b/base/images/tests/cases/container-base/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/base/images/tests/cases/runtime/container-base/test_basic.py b/base/images/tests/cases/runtime/container-base/test_basic.py new file mode 100644 index 00000000000..098024a2e12 --- /dev/null +++ b/base/images/tests/cases/runtime/container-base/test_basic.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: MIT +"""Basic runtime tests for container-base images. + +Validate fundamental container behavior by exec-ing commands into a +running container. Each test gets a fresh container instance via the +``container_exec_shell`` fixture (see conftest.py). +""" + +from __future__ import annotations + + +def test_shell_accessible(container_exec_shell) -> None: + """Container shell must be functional via exec.""" + result = container_exec_shell("echo hello-from-container") + assert result.exit_code == 0, ( + f"Shell exec failed (exit_code={result.exit_code}): {result.output}" + ) + assert "hello-from-container" in result.output + + +def test_dns_resolution(container_exec_shell) -> None: + """Container must be able to resolve localhost via DNS.""" + result = container_exec_shell("getent hosts localhost") + assert result.exit_code == 0, ( + f"DNS resolution failed (exit_code={result.exit_code}): {result.output}" + ) + assert "localhost" in result.output diff --git a/base/images/tests/cases/runtime/container-base/test_nginx/Dockerfile b/base/images/tests/cases/runtime/container-base/test_nginx/Dockerfile new file mode 100644 index 00000000000..15bf2651231 --- /dev/null +++ b/base/images/tests/cases/runtime/container-base/test_nginx/Dockerfile @@ -0,0 +1,5 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +RUN dnf install -y nginx curl && dnf clean all +COPY nginx.conf /etc/nginx/nginx.conf +EXPOSE 80 diff --git a/base/images/tests/cases/runtime/container-base/test_nginx/nginx.conf b/base/images/tests/cases/runtime/container-base/test_nginx/nginx.conf new file mode 100644 index 00000000000..f72d519484a --- /dev/null +++ b/base/images/tests/cases/runtime/container-base/test_nginx/nginx.conf @@ -0,0 +1,27 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /run/nginx.pid; + +events { + worker_connections 128; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + server { + listen 80; + server_name localhost; + + location / { + return 200 "OK\n"; + add_header Content-Type text/plain; + } + + location /health { + return 200 "healthy\n"; + add_header Content-Type text/plain; + } + } +} diff --git a/base/images/tests/cases/runtime/container-base/test_nginx/test_nginx.py b/base/images/tests/cases/runtime/container-base/test_nginx/test_nginx.py new file mode 100644 index 00000000000..21efe837037 --- /dev/null +++ b/base/images/tests/cases/runtime/container-base/test_nginx/test_nginx.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: MIT +"""Validate nginx works on the container-base image. + +Uses ``@pytest.mark.dockerfile()`` to build a custom image with +nginx installed on top of the image-under-test. +""" + +from __future__ import annotations + +import time + +import pytest + + +def wait_for_http(container_exec_shell, url: str): + """Poll until an HTTP endpoint responds successfully.""" + result = None + for _ in range(5): + result = container_exec_shell(f"curl -sf {url}") + if result.exit_code == 0: + return result + time.sleep(1) + + assert result is not None + return result + + +@pytest.mark.dockerfile() +def test_nginx_config_valid(container_exec_shell) -> None: + """nginx configuration must pass validation.""" + result = container_exec_shell("nginx -t") + assert result.exit_code == 0, f"nginx -t failed: {result.output}" + assert "syntax is ok" in result.output + assert "test is successful" in result.output + + +@pytest.mark.dockerfile() +def test_nginx_health_endpoint(container_exec_shell) -> None: + """nginx /health endpoint must return 200.""" + start = container_exec_shell("nginx") + assert start.exit_code == 0, f"nginx failed to start: {start.output}" + + result = wait_for_http(container_exec_shell, "http://localhost:80/health") + assert result.exit_code == 0, f"health check failed: {result.output}" + assert "healthy" in result.output diff --git a/base/images/tests/cases/container-base/test_container.py b/base/images/tests/cases/static/container-base/test_container.py similarity index 100% rename from base/images/tests/cases/container-base/test_container.py rename to base/images/tests/cases/static/container-base/test_container.py diff --git a/base/images/tests/cases/test_os_release.py b/base/images/tests/cases/static/test_os_release.py similarity index 100% rename from base/images/tests/cases/test_os_release.py rename to base/images/tests/cases/static/test_os_release.py diff --git a/base/images/tests/cases/test_packages.py b/base/images/tests/cases/static/test_packages.py similarity index 100% rename from base/images/tests/cases/test_packages.py rename to base/images/tests/cases/static/test_packages.py diff --git a/base/images/tests/cases/vm-base/test_kernel.py b/base/images/tests/cases/static/vm-base/test_kernel.py similarity index 100% rename from base/images/tests/cases/vm-base/test_kernel.py rename to base/images/tests/cases/static/vm-base/test_kernel.py diff --git a/base/images/tests/cases/vm-base/test_partitions.py b/base/images/tests/cases/static/vm-base/test_partitions.py similarity index 100% rename from base/images/tests/cases/vm-base/test_partitions.py rename to base/images/tests/cases/static/vm-base/test_partitions.py diff --git a/base/images/tests/cases/vm-base/__init__.py b/base/images/tests/cases/vm-base/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/base/images/tests/conftest.py b/base/images/tests/conftest.py index 66f06ba29b3..699a749623f 100644 --- a/base/images/tests/conftest.py +++ b/base/images/tests/conftest.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: MIT """Root conftest — fixtures for image validation. -CLI options (``--image-path``, ``--image-name``, ``--image-type``, -``--capabilities``, ``--workdir``) are registered in -:mod:`utils.pytest_plugin` (loaded early via entry point). +CLI options (``--image-path``, ``--image-ref``, ``--image-name``, +``--image-type``, ``--capabilities``, ``--workdir``) are registered +in :mod:`utils.pytest_plugin` (loaded early via entry point). """ from __future__ import annotations @@ -23,6 +23,15 @@ unmount_container_image, unmount_vm_image, ) +from utils.container_runtime import ( + build_image, + cleanup_test_images, + create_container, + destroy_container, + exec_in_container, + get_podman_client, + resolve_image_reference, +) from utils.parsers import parse_os_release, query_rpm_packages from utils.pytest_plugin import ( derive_image_type_from_capabilities, @@ -57,8 +66,12 @@ def capabilities(request: pytest.FixtureRequest) -> set[str]: @pytest.fixture(scope="session") -def image_path(request: pytest.FixtureRequest) -> Path: - p = Path(request.config.getoption("--image-path")).resolve() +def image_path(request: pytest.FixtureRequest) -> Path | None: + """Path to image artifact from ``--image-path``, or None if ``--image-ref`` is used.""" + raw = request.config.getoption("--image-path") + if not raw: + return None + p = Path(raw).resolve() logger.info("Image path: %s", p) if not p.exists(): pytest.fail(f"Image file does not exist: {p}") @@ -66,11 +79,20 @@ def image_path(request: pytest.FixtureRequest) -> Path: return p +@pytest.fixture(scope="session") +def image_ref(request: pytest.FixtureRequest) -> str | None: + """Image reference from ``--image-ref``, or None if ``--image-path`` is used.""" + ref = request.config.getoption("--image-ref") + if ref: + logger.info("Image ref: %s", ref) + return ref + + @pytest.fixture(scope="session") def image_type( request: pytest.FixtureRequest, capabilities: set[str], - image_path: Path, + image_path: Path | None, image_ref: str | None, ) -> str: """``'vm'`` or ``'container'`` — from ``--image-type``, capabilities, or file extension.""" explicit = request.config.getoption("--image-type") @@ -83,14 +105,21 @@ def image_type( logger.info("Image type (from capabilities): %s", from_caps) return from_caps - detected = detect_image_type(str(image_path)) - if detected is None: - pytest.fail( - f"Cannot detect image type from extension of {image_path.name}. " - "Pass --image-type or --capabilities explicitly." - ) - logger.info("Image type (auto-detected from extension): %s", detected) - return detected + # --image-ref implies container. + if image_ref: + logger.info("Image type (from --image-ref): container") + return "container" + + if image_path: + detected = detect_image_type(str(image_path)) + if detected is not None: + logger.info("Image type (auto-detected from extension): %s", detected) + return detected + + pytest.fail( + "Cannot detect image type. " + "Pass --image-type or --capabilities explicitly." + ) @pytest.fixture(scope="session") @@ -140,8 +169,14 @@ def workdir(request: pytest.FixtureRequest) -> Path: @pytest.fixture(scope="session") -def rootfs(image_path: Path, image_type: str, workdir: Path) -> Path: - """Mounted rootfs — session yield-fixture with cleanup.""" +def rootfs(image_path: Path | None, image_type: str, workdir: Path) -> Path: + """Mounted rootfs — session yield-fixture with cleanup. + + Requires ``--image-path`` (not ``--image-ref``); skips otherwise. + """ + if image_path is None: + pytest.skip("rootfs requires --image-path (not available with --image-ref)") + if image_type == "vm": mountpoint = workdir / "vm-rootfs" mountpoint.mkdir(parents=True, exist_ok=True) @@ -161,11 +196,13 @@ def rootfs(image_path: Path, image_type: str, workdir: Path) -> Path: @pytest.fixture(scope="session") -def disk_info(image_path: Path, image_type: str) -> DiskInfo | None: +def disk_info(image_path: Path | None, image_type: str) -> DiskInfo | None: """Partition/filesystem info — ``None`` for container images.""" if image_type != "vm": logger.debug("Skipping disk inspection (not a VM image)") return None + if image_path is None: + pytest.skip("disk_info requires --image-path") logger.info("Inspecting disk: %s", image_path) return inspect_disk(image_path) @@ -214,3 +251,126 @@ def partition_table(disk_info: DiskInfo | None, image_type: str) -> list[Partiti p.size_bytes, ) return disk_info.partitions + + +# --------------------------------------------------------------------------- +# Container runtime fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def podman_client(image_type: str): + """Session-scoped python-on-whales Podman client; skips for non-container images.""" + if image_type != "container": + yield None + return + + client = get_podman_client() + try: + yield client + finally: + cleanup_test_images(client) + + +@pytest.fixture(scope="session") +def container_image_ref( + podman_client, image_path: Path | None, image_ref: str | None, + image_type: str, +) -> str | None: + """Resolve the container image once per session and cache the reference. + + - If ``--image-ref`` was given, returns it directly. + - If ``--image-path`` was given, loads the archive via ``podman load`` + and returns the resulting image ID. + - Returns ``None`` for non-container sessions. + """ + if image_type != "container": + return None + + return resolve_image_reference(podman_client, image_path=image_path, image_ref=image_ref) + + +@pytest.fixture +def running_container( + podman_client, image_type: str, + container_image_ref: str | None, request: pytest.FixtureRequest, +): + """Fresh container per test with guaranteed teardown. + + If marked with ``@pytest.mark.dockerfile()``, builds a custom + image from the specified Dockerfile first. Skips for non-container + images. + """ + if image_type != "container": + pytest.skip("running_container only applicable to container images") + if container_image_ref is None: + pytest.fail("container_image_ref was not resolved for a container image") + + # Check for @pytest.mark.dockerfile() marker. + effective_image = container_image_ref + dockerfile_marker = request.node.get_closest_marker("dockerfile") + if dockerfile_marker is not None: + test_dir = Path(request.fspath).parent + if dockerfile_marker.args: + dockerfile_path = (test_dir / dockerfile_marker.args[0]).resolve() + else: + dockerfile_path = (test_dir / "Dockerfile").resolve() + + if not dockerfile_path.exists(): + pytest.fail( + f"Dockerfile not found: {dockerfile_path} " + f"(from @pytest.mark.dockerfile on {request.node.name})" + ) + + effective_image = build_image( + podman_client, dockerfile_path, container_image_ref, + ) + + logger.info("Creating container for test %s", request.node.name) + instance = create_container(podman_client, effective_image) + + try: + yield instance + finally: + logger.info("Destroying container for test %s", request.node.name) + destroy_container(podman_client, instance.container_name) + + +@pytest.fixture +def container_exec(podman_client, running_container): + """Callable to execute commands in the running test container. + + Usage:: + + def test_example(container_exec): + result = container_exec(["echo", "hello"]) + assert result.exit_code == 0 + assert "hello" in result.output + """ + def _exec(command: list[str]): + return exec_in_container( + podman_client, + running_container.container_name, + command, + ) + return _exec + + +@pytest.fixture +def container_exec_shell(podman_client, running_container): + """Callable to execute shell commands in the running test container. + + Usage:: + + def test_example(container_exec_shell): + result = container_exec_shell("echo hello") + assert result.exit_code == 0 + assert "hello" in result.output + """ + def _exec_shell(command: str, *, shell: str = "bash"): + return exec_in_container( + podman_client, + running_container.container_name, + [shell, "-c", command], + ) + return _exec_shell diff --git a/base/images/tests/pyproject.toml b/base/images/tests/pyproject.toml index 212913a71b8..c29121ee15b 100644 --- a/base/images/tests/pyproject.toml +++ b/base/images/tests/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.0" requires-python = ">=3.12" dependencies = [ "pytest>=8.0", + "python-on-whales>=0.81.0", ] [build-system] diff --git a/base/images/tests/utils/container_runtime.py b/base/images/tests/utils/container_runtime.py new file mode 100644 index 00000000000..71f007665f1 --- /dev/null +++ b/base/images/tests/utils/container_runtime.py @@ -0,0 +1,317 @@ +# SPDX-License-Identifier: MIT +"""Container runtime orchestration using python-on-whales. + +Provides utilities for creating running containers with exec access +for runtime integration testing. Uses python-on-whales to drive the +Podman CLI directly. +""" + +from __future__ import annotations + +import logging +import uuid +from pathlib import Path +from typing import Iterator, NamedTuple, cast + +from python_on_whales import DockerClient +from python_on_whales.exceptions import DockerException, NoSuchContainer, NoSuchImage + +from .tools import NativeTool + + +logger = logging.getLogger(__name__) + +# Register podman as a required native tool for container testing. +PODMAN = NativeTool( + name="podman", + package_hint="podman", + reason="container runtime for live container tests", + when="container", +) + +_loaded_image_refs: set[str] = set() +_built_image_cache: dict[tuple[Path, str], str] = {} + + +class ContainerExecResult(NamedTuple): + """Result of executing a command inside a container. + + *stdout* and *stderr* hold the per-stream output. *output* is the + two streams concatenated, kept for convenience. + """ + exit_code: int + stdout: str + stderr: str + output: str + + +class ContainerInstance(NamedTuple): + """A running container managed by the test framework.""" + container_id: str + container_name: str + image_ref: str + + +class ContainerRuntimeError(Exception): + """Container runtime operation failed.""" + + +def get_podman_client() -> DockerClient: + """Create a python-on-whales client configured for the Podman CLI.""" + client = DockerClient(client_call=["podman"], client_type="podman") + try: + client.version() + except DockerException as exc: + raise ContainerRuntimeError( + f"Cannot run podman through python-on-whales: {exc}" + ) from exc + return client + + +def resolve_image_reference( + client: DockerClient, + *, + image_path: Path | None = None, + image_ref: str | None = None, +) -> str: + """Resolve to a Podman-usable image reference. + + Exactly one of *image_path* or *image_ref* must be provided. + + - **image_path**: a local archive file (``.tar``, ``.tar.xz``, etc.) + is loaded via ``podman load`` and the resulting image ID is returned. + - **image_ref**: an image reference (e.g. ``mcr.microsoft.com/azurelinux/base/core:4.0`` + or ``localhost/container-base:latest``). Pulled from the registry + if not already present locally. + + Returns: + Image ID (for archives) or the image reference string. + """ + if image_path and image_ref: + raise ContainerRuntimeError( + "image_path and image_ref are mutually exclusive" + ) + if not image_path and not image_ref: + raise ContainerRuntimeError( + "Either image_path or image_ref must be provided" + ) + + if image_ref: + # Ensure the image is available locally; pull if needed. + if client.image.exists(image_ref): + logger.info("Image already present locally: %s", image_ref) + else: + logger.info("Pulling image: %s", image_ref) + client.image.pull(image_ref) + return image_ref + + # Load from archive file. + assert image_path is not None + logger.info("Loading image archive: %s", image_path) + image_refs = client.image.load(image_path) + + if not image_refs: + raise ContainerRuntimeError( + f"podman load returned no images for {image_path}" + ) + + image_ref = image_refs[0] + if image_ref is None: + raise ContainerRuntimeError( + f"podman load returned an empty image reference for {image_path}" + ) + image_ref = str(image_ref) + logger.info("Loaded image: %s", image_ref) + _loaded_image_refs.add(image_ref) + return image_ref + + +def _build_cache_key(dockerfile_path: Path, base_image_ref: str) -> tuple[Path, str]: + return (dockerfile_path.resolve(), base_image_ref) + + +def build_image( + client: DockerClient, + dockerfile_path: Path, + base_image_ref: str, +) -> str: + """Build a container image from a Dockerfile. + + Injects the image-under-test as the ``BASE_IMAGE`` build arg. + Results are cached per session by Dockerfile path and base image. + """ + cache_key = _build_cache_key(dockerfile_path, base_image_ref) + if cache_key in _built_image_cache: + cached = _built_image_cache[cache_key] + logger.info("Using cached build for %s: %s", dockerfile_path.name, cached) + return cached + + context_dir = dockerfile_path.parent + image_tag = f"localhost/azl-test-{uuid.uuid4().hex[:12]}:latest" + logger.info( + "Building image from %s (base: %s, context: %s)", + dockerfile_path, base_image_ref, context_dir, + ) + + client.legacy_build( + context_path=context_dir, + file=dockerfile_path, + build_args={"BASE_IMAGE": base_image_ref}, + tags=image_tag, + quiet=True, + ) + + logger.info("Built image: %s", image_tag) + _built_image_cache[cache_key] = image_tag + return image_tag + + +def cleanup_test_images(client: DockerClient) -> None: + """Remove test-created images (best-effort, never raises).""" + image_ref_groups = ( + ("built", sorted(set(_built_image_cache.values()))), + ("loaded", sorted(_loaded_image_refs)), + ) + _built_image_cache.clear() + _loaded_image_refs.clear() + + for image_source, image_refs in image_ref_groups: + for image_ref in image_refs: + logger.info("Removing %s test image %s", image_source, image_ref) + try: + client.image.remove(image_ref, force=True) + except (NoSuchImage, DockerException) as exc: + logger.debug("Image remove skipped (%s): %s", image_ref, exc) + + +def create_container( + client: DockerClient, + image_ref: str, + container_name: str | None = None, +) -> ContainerInstance: + """Create and start a container with exec access. + + The container runs ``sleep infinity`` to stay alive for the duration + of the test, allowing repeated ``exec`` calls. + + Args: + client: Active python-on-whales Podman client. + image_ref: Image ID or reference to run. + container_name: Optional name; auto-generated if None. + + Returns: + A ContainerInstance with the container's ID, name, and image ref. + """ + if container_name is None: + container_name = f"azl-test-{uuid.uuid4().hex[:12]}" + + logger.info("Creating container %s from image %s", container_name, image_ref) + + container = client.container.run( + image_ref, + command=["sleep", "infinity"], + name=container_name, + detach=True, + ) + + # Verify the container is running and exec works. + try: + container.reload() + if container.state.status != "running": + raise ContainerRuntimeError( + f"Container {container_name} is not running " + f"(status: {container.state.status})" + ) + + result = exec_in_container(client, container_name, ["echo", "ready"]) + if result.exit_code != 0: + raise ContainerRuntimeError( + f"Container exec readiness check failed for {container_name} " + f"(exit_code={result.exit_code}, output={result.output!r})" + ) + except BaseException: + # Clean up on any failure (including KeyboardInterrupt). + logger.warning("Readiness check failed; removing container %s", container_name) + try: + container.remove(force=True) + except Exception as cleanup_exc: + logger.warning("Failed to clean up container %s: %s", container_name, cleanup_exc) + raise + + logger.info("Container ready: %s (ID: %s)", container_name, container.id[:12]) + return ContainerInstance( + container_id=container.id, + container_name=container_name, + image_ref=image_ref, + ) + + +def exec_in_container( + client: DockerClient, + container_name: str, + command: list[str], +) -> ContainerExecResult: + """Execute a command inside a running container. + + Args: + client: Active python-on-whales Podman client. + container_name: Name of the running container. + command: Command argv to execute. + + Returns: + ContainerExecResult with exit_code, per-stream stdout/stderr, and a + combined ``output`` for convenience. + """ + logger.debug("Container exec [%s]: %s", container_name, command) + stdout_parts: list[str] = [] + stderr_parts: list[str] = [] + + try: + # stream=True yields (stream_name, chunk) tuples so we can keep + # stdout and stderr separate; stream=False would merge them. + output = client.container.execute( + container_name, + command, + stream=True, + ) + if output is None: + raise ContainerRuntimeError( + f"podman exec returned no output stream for {container_name}" + ) + + for stream_name, chunk in cast("Iterator[tuple[str, bytes]]", output): + parts = stderr_parts if stream_name == "stderr" else stdout_parts + parts.append(chunk.decode("utf-8", errors="replace")) + return _make_exec_result(0, stdout_parts, stderr_parts) + except DockerException as exc: + # If the exception streamed nothing through, fall back to the + # output captured on the exception itself. + if not stdout_parts and exc.stdout: + stdout_parts.append(exc.stdout) + if not stderr_parts and exc.stderr: + stderr_parts.append(exc.stderr) + return _make_exec_result(exc.return_code, stdout_parts, stderr_parts) + + +def _make_exec_result( + exit_code: int, + stdout_parts: list[str], + stderr_parts: list[str], +) -> ContainerExecResult: + stdout = "".join(stdout_parts) + stderr = "".join(stderr_parts) + return ContainerExecResult( + exit_code=exit_code, + stdout=stdout, + stderr=stderr, + output=stdout + stderr, + ) + + +def destroy_container(client: DockerClient, container_name: str) -> None: + """Kill and remove a container (best-effort, never raises).""" + logger.info("Destroying container %s", container_name) + try: + client.container.remove(container_name, force=True) + except (NoSuchContainer, DockerException) as exc: + logger.debug("Container remove skipped (%s): %s", container_name, exc) diff --git a/base/images/tests/utils/pytest_plugin.py b/base/images/tests/utils/pytest_plugin.py index 01054828dcb..c7d5f55f434 100644 --- a/base/images/tests/utils/pytest_plugin.py +++ b/base/images/tests/utils/pytest_plugin.py @@ -60,8 +60,17 @@ def pytest_addoption(parser) -> None: # type: ignore[no-untyped-def] group = parser.getgroup("image", "Azure Linux image validation") group.addoption( "--image-path", - required=True, - help="Path to the built image artifact (VHD, raw, OCI tar.xz, etc.)", + default=None, + help="Path to the built image artifact (VHD, raw, OCI tar.xz, etc.). " + "Mutually exclusive with --image-ref.", + ) + group.addoption( + "--image-ref", + default=None, + help="Container image reference (e.g. 'mcr.microsoft.com/azurelinux/base/core:4.0' " + "or 'localhost/container-base:latest'). Podman will pull from the " + "registry if not already present locally. " + "Mutually exclusive with --image-path.", ) group.addoption( "--image-name", @@ -103,9 +112,35 @@ def pytest_configure(config) -> None: # type: ignore[no-untyped-def] "image(name): only run this test when --image-name matches the named image family " "(exact match, or a ``-`` image-name)", ) + config.addinivalue_line( + "markers", + "runtime: auto-applied to tests under cases/runtime/; " + "use -m 'not runtime' to exclude runtime tests", + ) + config.addinivalue_line( + "markers", + 'dockerfile(path=None): build a custom image from a Dockerfile before ' + 'running the test. With no args, auto-discovers "Dockerfile" in the ' + "test file's directory. With an arg, uses that path relative to the " + "test file's directory. The image-under-test is injected as the " + "BASE_IMAGE build arg.", + ) from utils.tools import check_tools + # Validate that exactly one of --image-path or --image-ref is provided. + image_path_raw = config.getoption("--image-path", default=None) + image_ref_raw = config.getoption("--image-ref", default=None) + if image_path_raw and image_ref_raw: + raise pytest.UsageError( + "--image-path and --image-ref are mutually exclusive. " + "Provide one or the other." + ) + if not image_path_raw and not image_ref_raw: + raise pytest.UsageError( + "Either --image-path or --image-ref is required." + ) + # Determine image type early (before fixtures) so we only check # the tools that are actually needed for this run. image_type = config.getoption("--image-type", default=None) @@ -113,6 +148,9 @@ def pytest_configure(config) -> None: # type: ignore[no-untyped-def] caps = parse_capabilities(config.getoption("--capabilities", default=None)) if caps: image_type = derive_image_type_from_capabilities(caps) + if image_type is None: + if config.getoption("--image-ref", default=None): + image_type = "container" if image_type is None: image_path = config.getoption("--image-path", default=None) if image_path: @@ -156,33 +194,54 @@ def pytest_runtest_setup(item: pytest.Item) -> None: def pytest_collection_modifyitems(config, items) -> None: # type: ignore[no-untyped-def] - """Auto-apply ``@pytest.mark.image("")`` to tests under ``cases//``. + """Auto-apply markers based on directory layout under ``cases/``. + + Layout convention (after restructure):: + + cases/ + static/ + test_*.py # shared static tests + /test_*.py # image-specific static + runtime/ + test_*.py # shared runtime tests + /test_*.py # image-specific runtime - Convention: any test file inside ``cases//`` (at any - depth) is automatically restricted to images whose name belongs to - that family. The directory name is the family; an ``--image-name`` - matches the family if it equals the family exactly OR if it has the - form ``-`` (e.g. ``vm-base-dev`` matches the - ``vm-base`` family). See :func:`pytest_runtest_setup`. + Auto-applied markers: - This keeps the routing rule co-located with the directory layout — - no per-subdir conftest, and no per-file ``pytestmark`` boilerplate - that contributors might forget to add. + - ``@pytest.mark.image("")`` on any test under + ``cases/static//`` or ``cases/runtime//`` so it + only runs when ``--image-name`` belongs to that family. + - ``@pytest.mark.runtime`` on any test under ``cases/runtime/`` so + static-only suites can exclude them with ``-m "not runtime"``. - Tests directly under ``cases/`` (no image subdir) get no marker - and run for every image. + Tests directly under ``cases/static/`` or ``cases/runtime/`` (no + image subdir) get no ``image`` marker and run for every image. """ from pathlib import Path for item in items: parts = Path(str(item.fspath)).parts - # Anchor on the right-most "cases" segment so the convention is - # robust against arbitrary parent directory names. + # Anchor on the right-most "cases" segment. try: cases_idx = len(parts) - 1 - parts[::-1].index("cases") except ValueError: continue - # Need at least cases//.py to derive a family name. - if cases_idx + 2 < len(parts): - image_dir = parts[cases_idx + 1] + + remaining = parts[cases_idx + 1:] + if not remaining: + continue + + # remaining[0] is "static" or "runtime"; remaining[1] (if present) + # is the image-family directory. + category = remaining[0] # "static" or "runtime" + + # Auto-apply runtime marker. + if category == "runtime": + item.add_marker(pytest.mark.runtime) + + # Auto-apply image() marker if there's an image-family subdir. + # e.g. cases/static/vm-base/test_kernel.py → image("vm-base") + # cases/runtime/container-base/test_foo.py → image("container-base") + if len(remaining) >= 3: # category + family_dir + file + image_dir = remaining[1] item.add_marker(pytest.mark.image(image_dir))