diff --git a/base/comps/bash/bash.comp.toml b/base/comps/bash/bash.comp.toml index 5ef0d2922be..f42faa8471a 100644 --- a/base/comps/bash/bash.comp.toml +++ b/base/comps/bash/bash.comp.toml @@ -2,3 +2,4 @@ # Pin to Fedora 44 dist-git head as of the AZL4 Public Preview cutoff. # F44 carries the build fixes we needed from rawhide; F43 does not. spec = { type = "upstream", upstream-distro = { name = "fedora", version = "44" }, upstream-commit = "a6bcc6767229199f4f02b781d1d39df0835d894b" } +tests.tests = [{ name = "bash-fedora-shell" }] diff --git a/base/comps/bash/bash.tests.toml b/base/comps/bash/bash.tests.toml new file mode 100644 index 00000000000..eac846cbef4 --- /dev/null +++ b/base/comps/bash/bash.tests.toml @@ -0,0 +1,22 @@ +# ============================================================================ +# bash — component-local test definitions +# ============================================================================ +# +# Fedora's bash dist-git ships a tmt plan at /plans/shell. We pin the tmt +# source to the *same* upstream commit that bash.comp.toml pins for the +# spec, so the test plan moves in lockstep with the version of bash we +# actually ship. (When the spec pin moves, this SHA moves with it.) +# ============================================================================ + +[tests.bash-fedora-shell] +type = "tmt" +description = "Fedora bash dist-git tmt smoke (plans/shell), lock-stepped with the bash spec pin" +kind = ["functional"] +# long-running omitted: virtual provision + a small shell plan finishes in minutes. + +[tests.bash-fedora-shell.tmt] +source = { git-url = "https://src.fedoraproject.org/rpms/bash.git", ref = "a6bcc6767229199f4f02b781d1d39df0835d894b" } +plan = "/plans/shell" + +[tests.bash-fedora-shell.tmt.provision] +how = "virtual" diff --git a/base/comps/components.toml b/base/comps/components.toml index a9f322d1324..9693790a118 100644 --- a/base/comps/components.toml +++ b/base/comps/components.toml @@ -1,4 +1,4 @@ -includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"] +includes = ["**/*.comp.toml", "**/*.tests.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"] # # Components imported from Fedora with no per-component modifications. @@ -256,7 +256,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.console-login-helper-messages] [components.console-setup] [components.container-selinux] -[components.containerd] [components.containers-common] [components.convmv] [components.corosync] @@ -272,14 +271,12 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.crash-gcore-command] [components.crash-trace-command] [components.createrepo_c] -[components.cri-o] [components.cri-tools] [components.'cri-tools1.35'] [components.criu] [components.cronie] [components.crontabs] [components.cross-binutils] -[components.crun] [components.cryptsetup] [components.cscope] [components.ctags] @@ -2108,7 +2105,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.mksh] [components.mm-common] [components.mobile-broadband-provider-info] -[components.moby-engine] [components.mock] [components.mockito] [components.mod_auth_gssapi] @@ -3299,7 +3295,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.pmix] [components.pngcrush] [components.pocketfft] -[components.podman] [components.poetry] [components.policycoreutils] [components.polkit] @@ -4718,7 +4713,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.rubygem-zeitwerk] [components.rubypick] [components.ruff] -[components.runc] [components.rust-actix-codec] [components.rust-actix-http] [components.rust-actix-macros] diff --git a/base/comps/container-runtime.tests.toml b/base/comps/container-runtime.tests.toml new file mode 100644 index 00000000000..4c199244cba --- /dev/null +++ b/base/comps/container-runtime.tests.toml @@ -0,0 +1,27 @@ +# ============================================================================ +# container-runtime — shared test-group spanning multiple runtime components +# ============================================================================ +# +# This file defines a single test-group that all container-runtime-shaped +# components (those that *run* containers) associate themselves with. Each +# member component carries `tests.tests = [{ group = "container-runtime" }]` +# in its own .comp.toml; the group itself is defined once here. +# +# Note on file placement: shared test-groups that span multiple components +# don't fit cleanly into either the image-level (base/images/images.tests.toml) +# or the per-component (/.tests.toml) homes. This flat root-level +# placement parallels how component-mingw-disablement.toml etc. live at the +# comps root. The pattern won't scale to many such files; revisit when we +# accumulate more cross-component test-groups. +# ============================================================================ + +[test-groups.container-runtime] +description = "Container runtime scenario tests: black-box exercise of whatever runtime is installed (containerd / runc / crun / podman / cri-o / moby-engine)" +# TODO: populate with a runtime-agnostic scenario test once authored. +# The rationale for the group is precisely to run a *scenario-level* test +# (e.g. pull → run → exec → stop a container) that exercises any of the +# member runtimes through their common interface — not the runtime-specific +# tests each upstream dist-git happens to carry. No such homegrown test +# exists in the tree yet, so the member list starts empty and the group +# acts purely as a stable identity for component associations. +tests = [] diff --git a/base/comps/containerd/containerd.comp.toml b/base/comps/containerd/containerd.comp.toml new file mode 100644 index 00000000000..ff244e2431c --- /dev/null +++ b/base/comps/containerd/containerd.comp.toml @@ -0,0 +1,2 @@ +[components.containerd] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/cri-o/cri-o.comp.toml b/base/comps/cri-o/cri-o.comp.toml new file mode 100644 index 00000000000..f9a5d8973d2 --- /dev/null +++ b/base/comps/cri-o/cri-o.comp.toml @@ -0,0 +1,2 @@ +[components.cri-o] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/crun/crun.comp.toml b/base/comps/crun/crun.comp.toml new file mode 100644 index 00000000000..eb51ac34400 --- /dev/null +++ b/base/comps/crun/crun.comp.toml @@ -0,0 +1,2 @@ +[components.crun] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/kernel-headers/kernel-headers.comp.toml b/base/comps/kernel-headers/kernel-headers.comp.toml index 1ff231aa549..9a62375502a 100644 --- a/base/comps/kernel-headers/kernel-headers.comp.toml +++ b/base/comps/kernel-headers/kernel-headers.comp.toml @@ -1,4 +1,5 @@ [components.kernel-headers] +tests.tests = [{ group = "kernel-tests" }] # Kernel-headers using Fedora spec with overlays for Azure Linux 6.18. # Fedora's spec expects a pre-built headers tarball. These overlays adapt it to # use Azure Linux kernel source and generate headers during build. diff --git a/base/comps/kernel.tests.toml b/base/comps/kernel.tests.toml new file mode 100644 index 00000000000..d8d4558dada --- /dev/null +++ b/base/comps/kernel.tests.toml @@ -0,0 +1,24 @@ +# ============================================================================ +# kernel-tests — shared test-group for kernel-shaped components +# ============================================================================ +# +# Bundles LISA kernel-oriented tests (LTP + XFS filesystem) into a stable +# group name that kernel-adjacent components (kernel, kernel-headers, ...) +# reference from their own .comp.toml via: +# +# tests.tests = [{ group = "kernel-tests" }] +# +# The underlying tests themselves are defined in +# base/images/images.tests.toml. This file only declares the group identity +# and its membership so component owners can opt in with a single line. +# +# File placement mirrors container-runtime.tests.toml: a flat root-level +# cross-component test-group lives at base/comps/.tests.toml. +# ============================================================================ + +[test-groups.kernel-tests] +description = "Kernel-oriented LISA coverage (LTP + XFS filesystem) for kernel-shaped components" +tests = [ + { name = "kernel_ltp" }, + { name = "xfs_filesystem" }, +] diff --git a/base/comps/kernel/kernel.comp.toml b/base/comps/kernel/kernel.comp.toml index 9f568dc9314..b77b71e8c61 100644 --- a/base/comps/kernel/kernel.comp.toml +++ b/base/comps/kernel/kernel.comp.toml @@ -1,4 +1,5 @@ [components.kernel] +tests.tests = [{ group = "kernel-tests" }] # Manual release calculation required because the spec uses %{specrelease} macro # Release: %{pkg_release} diff --git a/base/comps/moby-engine/moby-engine.comp.toml b/base/comps/moby-engine/moby-engine.comp.toml new file mode 100644 index 00000000000..e086b0864de --- /dev/null +++ b/base/comps/moby-engine/moby-engine.comp.toml @@ -0,0 +1,2 @@ +[components.moby-engine] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/podman/podman.comp.toml b/base/comps/podman/podman.comp.toml new file mode 100644 index 00000000000..b9c24deaf81 --- /dev/null +++ b/base/comps/podman/podman.comp.toml @@ -0,0 +1,2 @@ +[components.podman] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/runc/runc.comp.toml b/base/comps/runc/runc.comp.toml new file mode 100644 index 00000000000..eeb7c7cf64a --- /dev/null +++ b/base/comps/runc/runc.comp.toml @@ -0,0 +1,2 @@ +[components.runc] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/images/images.tests.toml b/base/images/images.tests.toml new file mode 100644 index 00000000000..87dee6b4fce --- /dev/null +++ b/base/images/images.tests.toml @@ -0,0 +1,159 @@ +# ============================================================================ +# Image test catalog +# ============================================================================ +# +# Single source of truth for all image-runnable test definitions and +# the named groups that bundle them. +# +# Schema: +# [tests.X] : a single unit of testing — one configuration of +# one runner/harness, with framework-specific +# options under [tests.X.]. +# [test-groups.X] : a named bundle of tests an image (or component) +# can reference instead of enumerating members. +# +# Image and component associations are NOT defined here. They live with +# the entity that runs them: +# - base/images/images.toml (tests.tests = [...]) +# - base/comps//.comp.toml or +# base/comps//.tests.toml (tests.tests = [...]) +# +# Both fields can mix per-test and per-group references: +# tests.tests = [{ name = "static-image-checks" }, { group = "x" }] +# +# Declarative metadata on [tests.X]: +# description : human-readable summary +# type : "pytest" | "tmt" | "lisa" +# kind : ["functional"] | ["performance"] | both +# long-running : boolean; omit unless the test may run for hours +# required-capabilities : capability tokens the image must declare +# +# ============================================================================ + +# ============================================================================= +# Static image checks (pytest) +# ============================================================================= + +# Single shared pytest suite for static (offline) image validation. +[tests.static-image-checks] +type = "pytest" +description = "Offline image validation (shared + image-specific tests)" +kind = ["functional"] + +[tests.static-image-checks.pytest] +working-dir = "tests" +install = "pyproject" +test-paths = ["cases/static/"] +# {capabilities} is substituted by azldev as a comma-separated list of +# the capabilities set to `true` for the image (e.g. +# "machine-bootable,systemd,runtime-package-management"). The pytest +# plugin parses this into a set; @pytest.mark.require_capability("...") +# tests that don't match are skipped. +extra-args = [ + "--image-path", "{image-path}", + "--image-name", "{image-name}", + "--capabilities", "{capabilities}", +] + +# Runtime container tests — validate live container behavior via podman. +[tests.runtime-container-tests] +type = "pytest" +description = "Runtime container validation (exec into live containers)" +kind = ["functional"] +required-capabilities = ["container"] + +[tests.runtime-container-tests.pytest] +working-dir = "tests" +install = "pyproject" +test-paths = ["cases/runtime/"] +extra-args = [ + "--image-path", "{image-path}", + "--image-name", "{image-name}", + "--capabilities", "{capabilities}", +] + +# ============================================================================= +# LISA tests +# ============================================================================= + +[tests.smoke_test] +type = "lisa" +description = "LISA smoke test — fastest PR gate (pinned)" +kind = ["functional"] +required-capabilities = ["machine-bootable"] +[tests.smoke_test.lisa] +source = { git-url = "https://github.com/microsoft/lisa.git", ref = "20260330.1" } +criteria = { testcaseNames = ["smoke_test"] } + +[tests.functional_core] +type = "lisa" +description = "Core functional coverage (LISA priority=1)" +kind = ["functional"] +required-capabilities = ["machine-bootable"] +[tests.functional_core.lisa] +criteria = { priority = [1] } + +[tests.functional_extended] +type = "lisa" +description = "Extended functional coverage (LISA priority=2)" +kind = ["functional"] +long-running = true +required-capabilities = ["machine-bootable"] +[tests.functional_extended.lisa] +criteria = { priority = [2] } + +[tests.kernel_ltp] +type = "lisa" +description = "Kernel LTP (Linux Test Project) suite" +kind = ["functional"] +long-running = true +required-capabilities = ["machine-bootable"] + +[tests.kernel_ltp.lisa] +criteria = { testcaseNames = ["verify_ltp_lite"] } + +[tests.xfs_filesystem] +type = "lisa" +description = "XFS data-disk filesystem tests (standard + NVMe)" +kind = ["functional"] +required-capabilities = ["machine-bootable"] + +[tests.xfs_filesystem.lisa] +criteria = { testcaseNames = ["verify_xfs_standard_datadisk", "verify_xfs_nvme_datadisk"] } + +[tests.lisa_perf] +type = "lisa" +description = "LISA performance test areas (network, nested, nvme, perf_tool, storage)" +kind = ["performance"] +long-running = true +required-capabilities = ["machine-bootable"] +[tests.lisa_perf.lisa] +criteria = [ + { area = "network", category = "performance" }, + { area = "nested", category = "performance" }, + { area = "nvme", category = "performance" }, + { area = "perf_tool", category = "performance" }, + { area = "storage", category = "performance" }, +] + +# ---- VM base test groups ------------------------------------------------- + +# Functional coverage applicable to a VM-shaped Azure Linux image. +[test-groups.vm-base-functional] +description = "Functional coverage for VM base images" +tests = [ + { name = "smoke_test" }, + { name = "functional_core" }, + { name = "functional_extended" }, + { name = "kernel_ltp" }, + { name = "xfs_filesystem" }, +] + +# Performance-oriented coverage. Kept as a separate group because +# performance runs typically have a different cadence and budget than +# functional runs, and consumers may want to opt in independently. +[test-groups.vm-base-performance] +description = "Performance coverage for VM base images" +tests = [ + { name = "lisa_perf" }, +] diff --git a/base/images/images.toml b/base/images/images.toml index e4ce97cb286..6b2e5d8fa22 100644 --- a/base/images/images.toml +++ b/base/images/images.toml @@ -22,7 +22,7 @@ # `` points at azl4-dev for local builds; koji overrides # this for distro builds). # -# Capabilities and test-suites are identical between a canonical / `-dev` +# Capabilities and tests are identical between a canonical / `-dev` # pair (the variants only differ in what `-repos` package ships and, # for `core`, the OCI tag). # ============================================================================ @@ -32,13 +32,10 @@ [images.vm-base] description = "VM Base Image" definition = { type = "kiwi", path = "vm-base/vm-base.kiwi", profile = "vm-base" } -tests.test-suites = [ - { name = "static-image-checks" }, - { name = "lisa-main" }, - { name = "lisa-xfs" }, - { name = "lisa-perf" }, - { name = "lisa-smoke" }, - { name = "lisa-kernel-ltp" }, +tests.tests = [ + { name = "static-image-checks" }, + { group = "vm-base-functional" }, + { group = "vm-base-performance" }, ] [images.vm-base.capabilities] @@ -50,13 +47,10 @@ runtime-package-management = true [images.vm-base-dev] description = "VM Base Image (dev)" definition = { type = "kiwi", path = "vm-base/vm-base.kiwi", profile = "vm-base-dev" } -tests.test-suites = [ - { name = "static-image-checks" }, - { name = "lisa-main" }, - { name = "lisa-xfs" }, - { name = "lisa-perf" }, - { name = "lisa-smoke" }, - { name = "lisa-kernel-ltp" }, +tests.tests = [ + { name = "static-image-checks" }, + { group = "vm-base-functional" }, + { group = "vm-base-performance" }, ] [images.vm-base-dev.capabilities] @@ -69,7 +63,7 @@ runtime-package-management = true [images.container-base] description = "Container Base Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "core" } -tests.test-suites = [ +tests.tests = [ { name = "static-image-checks" }, { name = "runtime-container-tests" }, ] @@ -83,7 +77,7 @@ runtime-package-management = true [images.container-base-dev] description = "Container Base Image (dev)" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "core-dev" } -tests.test-suites = [ +tests.tests = [ { name = "static-image-checks" }, { name = "runtime-container-tests" }, ] @@ -129,67 +123,3 @@ definition = { type = "kiwi", path = "vm-iso-installer/vm-iso-installer.kiwi", p [images.vm-iso-installer-dev] description = "VM ISO Installer (dev)" definition = { type = "kiwi", path = "vm-iso-installer/vm-iso-installer.kiwi", profile = "vm-iso-installer-dev" } - -# ============================================================================ -# Test suites -# ============================================================================ - -[test-suites.lisa-main] -type = "lisa" -description = "Lisa tests which are run by default" - -[test-suites.lisa-xfs] -type = "lisa" -description = "Lisa xfs tests" - -[test-suites.lisa-perf] -type = "lisa" -description = "Lisa perf tests" - -[test-suites.lisa-smoke] -type = "lisa" -description = "Lisa smoke tests" - -[test-suites.lisa-cvm] -type = "lisa" -description = "Lisa cvm tests" - -[test-suites.lisa-kernel-ltp] -type = "lisa" -description = "Lisa kernel LTP tests" - -# Single shared pytest suite for static (offline) image validation. -[test-suites.static-image-checks] -type = "pytest" -description = "Offline image validation (shared + image-specific tests)" - -[test-suites.static-image-checks.pytest] -working-dir = "tests" -install = "pyproject" -test-paths = ["cases/static/"] -# {capabilities} is substituted by azldev as a comma-separated list of -# the names of capabilities set to `true` for the image (e.g. -# "machine-bootable,systemd,runtime-package-management"). Names use -# the same dash-case form declared above. The pytest plugin parses -# this string into a set; @pytest.mark.require_capability("...") tests -# that don't match are skipped. -extra-args = [ - "--image-path", "{image-path}", - "--image-name", "{image-name}", - "--capabilities", "{capabilities}", -] - -# Runtime container tests — validate live container behavior via podman. -[test-suites.runtime-container-tests] -type = "pytest" -description = "Runtime container validation (exec into live containers)" - -[test-suites.runtime-container-tests.pytest] -working-dir = "tests" -install = "pyproject" -test-paths = ["cases/runtime/"] -extra-args = [ - "--image-path", "{image-path}", - "--image-name", "{image-name}", - "--capabilities", "{capabilities}", -] diff --git a/base/project.toml b/base/project.toml index 1db4bb60f9c..a6abd0ed546 100644 --- a/base/project.toml +++ b/base/project.toml @@ -2,7 +2,7 @@ # Base project # -includes = ["comps/components.toml", "images/images.toml"] +includes = ["comps/components.toml", "images/images.toml", "images/images.tests.toml"] [project] description = 'azurelinux-base'