test: add framework label catalog and component/image test mappings

base/comps/bash/bash.comp.toml

@@ -2,3 +2,4 @@
 # Pin to Fedora 44 dist-git head as of the AZL4 Public Preview cutoff.
 # F44 carries the build fixes we needed from rawhide; F43 does not.
 spec = { type = "upstream", upstream-distro = { name = "fedora", version = "44" }, upstream-commit = "a6bcc6767229199f4f02b781d1d39df0835d894b" }
+tests.tests = [{ name = "bash-fedora-shell" }]

base/comps/bash/bash.tests.toml

@@ -0,0 +1,22 @@
+# ============================================================================
+# bash — component-local test definitions
+# ============================================================================
+#
+# Fedora's bash dist-git ships a tmt plan at /plans/shell. We pin the tmt
+# source to the *same* upstream commit that bash.comp.toml pins for the
+# spec, so the test plan moves in lockstep with the version of bash we
+# actually ship. (When the spec pin moves, this SHA moves with it.)
+# ============================================================================
+
+[tests.bash-fedora-shell]
+type        = "tmt"
+description = "Fedora bash dist-git tmt smoke (plans/shell), lock-stepped with the bash spec pin"
+kind        = ["functional"]
+# long-running omitted: virtual provision + a small shell plan finishes in minutes.
+
+[tests.bash-fedora-shell.tmt]
+source = { git-url = "https://src.fedoraproject.org/rpms/bash.git", ref = "a6bcc6767229199f4f02b781d1d39df0835d894b" }
+plan   = "/plans/shell"
+
+[tests.bash-fedora-shell.tmt.provision]
+how = "virtual"

base/comps/components.toml

@@ -1,4 +1,4 @@
-includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"]
+includes = ["**/*.comp.toml", "**/*.tests.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"]
 
 #
 # Components imported from Fedora with no per-component modifications.
@@ -256,7 +256,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min
 [components.console-login-helper-messages]
 [components.console-setup]
 [components.container-selinux]
-[components.containerd]
 [components.containers-common]
 [components.convmv]
 [components.corosync]
@@ -272,14 +271,12 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min
 [components.crash-gcore-command]
 [components.crash-trace-command]
 [components.createrepo_c]
-[components.cri-o]
 [components.cri-tools]
 [components.'cri-tools1.35']
 [components.criu]
 [components.cronie]
 [components.crontabs]
 [components.cross-binutils]
-[components.crun]
 [components.cryptsetup]
 [components.cscope]
 [components.ctags]
@@ -2108,7 +2105,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min
 [components.mksh]
 [components.mm-common]
 [components.mobile-broadband-provider-info]
-[components.moby-engine]
 [components.mock]
 [components.mockito]
 [components.mod_auth_gssapi]
@@ -3299,7 +3295,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min
 [components.pmix]
 [components.pngcrush]
 [components.pocketfft]
-[components.podman]
 [components.poetry]
 [components.policycoreutils]
 [components.polkit]
@@ -4718,7 +4713,6 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min
 [components.rubygem-zeitwerk]
 [components.rubypick]
 [components.ruff]
-[components.runc]
 [components.rust-actix-codec]
 [components.rust-actix-http]
 [components.rust-actix-macros]

base/comps/container-runtime.tests.toml

@@ -0,0 +1,27 @@
+# ============================================================================
+# container-runtime — shared test-group spanning multiple runtime components
+# ============================================================================
+#
+# This file defines a single test-group that all container-runtime-shaped
+# components (those that *run* containers) associate themselves with. Each
+# member component carries `tests.tests = [{ group = "container-runtime" }]`
+# in its own <name>.comp.toml; the group itself is defined once here.
+#
+# Note on file placement: shared test-groups that span multiple components
+# don't fit cleanly into either the image-level (base/images/images.tests.toml)
+# or the per-component (<comp>/<comp>.tests.toml) homes. This flat root-level
+# placement parallels how component-mingw-disablement.toml etc. live at the
+# comps root. The pattern won't scale to many such files; revisit when we
+# accumulate more cross-component test-groups.
+# ============================================================================
+
+[test-groups.container-runtime]
+description = "Container runtime scenario tests: black-box exercise of whatever runtime is installed (containerd / runc / crun / podman / cri-o / moby-engine)"
+# TODO: populate with a runtime-agnostic scenario test once authored.
+# The rationale for the group is precisely to run a *scenario-level* test
+# (e.g. pull → run → exec → stop a container) that exercises any of the
+# member runtimes through their common interface — not the runtime-specific
+# tests each upstream dist-git happens to carry. No such homegrown test
+# exists in the tree yet, so the member list starts empty and the group
+# acts purely as a stable identity for component associations.
+tests = []

base/comps/containerd/containerd.comp.toml

@@ -0,0 +1,2 @@
+[components.containerd]
+tests.tests = [{ group = "container-runtime" }]

base/comps/cri-o/cri-o.comp.toml

@@ -0,0 +1,2 @@
+[components.cri-o]
+tests.tests = [{ group = "container-runtime" }]

base/comps/crun/crun.comp.toml

@@ -0,0 +1,2 @@
+[components.crun]
+tests.tests = [{ group = "container-runtime" }]

base/comps/kernel-headers/kernel-headers.comp.toml

@@ -1,4 +1,5 @@
 [components.kernel-headers]
+tests.tests = [{ group = "kernel-tests" }]
 # Kernel-headers using Fedora spec with overlays for Azure Linux 6.18.
 # Fedora's spec expects a pre-built headers tarball. These overlays adapt it to
 # use Azure Linux kernel source and generate headers during build.

base/comps/kernel.tests.toml

@@ -0,0 +1,24 @@
+# ============================================================================
+# kernel-tests — shared test-group for kernel-shaped components
+# ============================================================================
+#
+# Bundles LISA kernel-oriented tests (LTP + XFS filesystem) into a stable
+# group name that kernel-adjacent components (kernel, kernel-headers, ...)
+# reference from their own <name>.comp.toml via:
+#
+#   tests.tests = [{ group = "kernel-tests" }]
+#
+# The underlying tests themselves are defined in
+# base/images/images.tests.toml. This file only declares the group identity
+# and its membership so component owners can opt in with a single line.
+#
+# File placement mirrors container-runtime.tests.toml: a flat root-level
+# cross-component test-group lives at base/comps/<topic>.tests.toml.
+# ============================================================================
+
+[test-groups.kernel-tests]
+description = "Kernel-oriented LISA coverage (LTP + XFS filesystem) for kernel-shaped components"
+tests = [
+    { name = "kernel_ltp" },
+    { name = "xfs_filesystem" },
+]

base/comps/kernel/kernel.comp.toml

@@ -1,4 +1,5 @@
 [components.kernel]
+tests.tests = [{ group = "kernel-tests" }]
 
 # Manual release calculation required because the spec uses %{specrelease} macro
 # Release: %{pkg_release}

base/comps/moby-engine/moby-engine.comp.toml

@@ -0,0 +1,2 @@
+[components.moby-engine]
+tests.tests = [{ group = "container-runtime" }]

base/comps/podman/podman.comp.toml

@@ -0,0 +1,2 @@
+[components.podman]
+tests.tests = [{ group = "container-runtime" }]

base/comps/runc/runc.comp.toml

@@ -0,0 +1,2 @@
+[components.runc]
+tests.tests = [{ group = "container-runtime" }]

base/images/images.tests.toml

@@ -0,0 +1,159 @@
+# ============================================================================
+# Image test catalog
+# ============================================================================
+#
+# Single source of truth for all image-runnable test definitions and
+# the named groups that bundle them.
+#
+# Schema:
+#   [tests.X]            : a single unit of testing — one configuration of
+#                          one runner/harness, with framework-specific
+#                          options under [tests.X.<framework>].
+#   [test-groups.X]      : a named bundle of tests an image (or component)
+#                          can reference instead of enumerating members.
+#
+# Image and component associations are NOT defined here. They live with
+# the entity that runs them:
+#   - base/images/images.toml          (tests.tests = [...])
+#   - base/comps/<name>/<name>.comp.toml or
+#     base/comps/<name>/<name>.tests.toml (tests.tests = [...])
+#
+# Both fields can mix per-test and per-group references:
+#   tests.tests = [{ name = "static-image-checks" }, { group = "x" }]
+#
+# Declarative metadata on [tests.X]:
+#   description           : human-readable summary
+#   type                  : "pytest" | "tmt" | "lisa"
+#   kind                  : ["functional"] | ["performance"] | both
+#   long-running          : boolean; omit unless the test may run for hours
+#   required-capabilities : capability tokens the image must declare
+#
+# ============================================================================
+
+# =============================================================================
+# Static image checks (pytest)
+# =============================================================================
+
+# Single shared pytest suite for static (offline) image validation.
+[tests.static-image-checks]
+type        = "pytest"
+description = "Offline image validation (shared + image-specific tests)"
+kind        = ["functional"]
+
+[tests.static-image-checks.pytest]
+working-dir = "tests"
+install     = "pyproject"
+test-paths  = ["cases/static/"]
+# {capabilities} is substituted by azldev as a comma-separated list of
+# the capabilities set to `true` for the image (e.g.
+# "machine-bootable,systemd,runtime-package-management"). The pytest
+# plugin parses this into a set; @pytest.mark.require_capability("...")
+# tests that don't match are skipped.
+extra-args = [
+    "--image-path", "{image-path}",
+    "--image-name", "{image-name}",
+    "--capabilities", "{capabilities}",
+]
+
+# Runtime container tests — validate live container behavior via podman.
+[tests.runtime-container-tests]
+type        = "pytest"
+description = "Runtime container validation (exec into live containers)"
+kind        = ["functional"]
+required-capabilities = ["container"]
+
+[tests.runtime-container-tests.pytest]
+working-dir = "tests"
+install     = "pyproject"
+test-paths  = ["cases/runtime/"]
+extra-args = [
+    "--image-path", "{image-path}",
+    "--image-name", "{image-name}",
+    "--capabilities", "{capabilities}",
+]
+
+# =============================================================================
+# LISA tests
+# =============================================================================
+
+[tests.smoke_test]
+type                  = "lisa"
+description           = "LISA smoke test — fastest PR gate (pinned)"
+kind                  = ["functional"]
+required-capabilities = ["machine-bootable"]
+[tests.smoke_test.lisa]
+source   = { git-url = "https://github.com/microsoft/lisa.git", ref = "20260330.1" }
+criteria = { testcaseNames = ["smoke_test"] }
+
+[tests.functional_core]
+type                  = "lisa"
+description           = "Core functional coverage (LISA priority=1)"
+kind                  = ["functional"]
+required-capabilities = ["machine-bootable"]
+[tests.functional_core.lisa]
+criteria = { priority = [1] }
+
+[tests.functional_extended]
+type                  = "lisa"
+description           = "Extended functional coverage (LISA priority=2)"
+kind                  = ["functional"]
+long-running          = true
+required-capabilities = ["machine-bootable"]
+[tests.functional_extended.lisa]
+criteria = { priority = [2] }
+
+[tests.kernel_ltp]
+type                  = "lisa"
+description           = "Kernel LTP (Linux Test Project) suite"
+kind                  = ["functional"]
+long-running          = true
+required-capabilities = ["machine-bootable"]
+
+[tests.kernel_ltp.lisa]
+criteria = { testcaseNames = ["verify_ltp_lite"] }
+
+[tests.xfs_filesystem]
+type                  = "lisa"
+description           = "XFS data-disk filesystem tests (standard + NVMe)"
+kind                  = ["functional"]
+required-capabilities = ["machine-bootable"]
+
+[tests.xfs_filesystem.lisa]
+criteria = { testcaseNames = ["verify_xfs_standard_datadisk", "verify_xfs_nvme_datadisk"] }
+
+[tests.lisa_perf]
+type                  = "lisa"
+description           = "LISA performance test areas (network, nested, nvme, perf_tool, storage)"
+kind                  = ["performance"]
+long-running          = true
+required-capabilities = ["machine-bootable"]
+[tests.lisa_perf.lisa]
+criteria = [
+  { area = "network",   category = "performance" },
+  { area = "nested",    category = "performance" },
+  { area = "nvme",      category = "performance" },
+  { area = "perf_tool", category = "performance" },
+  { area = "storage",   category = "performance" },
+]
+
+# ---- VM base test groups -------------------------------------------------
+
+# Functional coverage applicable to a VM-shaped Azure Linux image.
+[test-groups.vm-base-functional]
+description = "Functional coverage for VM base images"
+tests = [
+  { name = "smoke_test" },
+  { name = "functional_core" },
+  { name = "functional_extended" },
+  { name = "kernel_ltp" },
+  { name = "xfs_filesystem" },
+]
+
+# Performance-oriented coverage. Kept as a separate group because
+# performance runs typically have a different cadence and budget than
+# functional runs, and consumers may want to opt in independently.
+[test-groups.vm-base-performance]
+description = "Performance coverage for VM base images"
+tests = [
+  { name = "lisa_perf" },
+]