From 7cf239a0baa9080b7fc0bd9ea658ca631b517f93 Mon Sep 17 00:00:00 2001 From: Bhagyashri Date: Wed, 27 May 2026 01:53:39 +0530 Subject: [PATCH 1/3] test: add framework label catalog and component/image test mappings --- base/comps/components.toml | 8 + base/comps/grub2/grub2.comp.toml | 1 + .../kernel-headers/kernel-headers.comp.toml | 1 + base/comps/kernel/kernel.comp.toml | 1 + base/comps/systemd/systemd.comp.toml | 1 + base/images/images.toml | 33 ++ base/tests/framework-labels.toml | 400 ++++++++++++++++++ 7 files changed, 445 insertions(+) create mode 100644 base/tests/framework-labels.toml diff --git a/base/comps/components.toml b/base/comps/components.toml index a9f322d1324..d0722ffbc0b 100644 --- a/base/comps/components.toml +++ b/base/comps/components.toml @@ -7,28 +7,35 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.389-ds-base] [components.7zip] +test_labels = ["archive_operations", "compression_performance", "backup_functionality"] [components.AMF] [components.CGAL] [components.CUnit] +test_labels = ["test_execution", "test_framework_integration", "test_reporting"] [components.CharLS] [components.CppMicroServices] [components.Cython] [components.DevIL] [components.FP16] [components.GitPython] +test_labels = ["vcs_operations", "vcs_performance", "vcs_integration"] [components.HdrHistogram_c] [components.JUnitParams] +test_labels = ["test_execution", "test_framework_integration", "test_reporting"] [components.Judy] [components.LibRaw] [components.Lmod] [components.MUMPS] [components.ModemManager] [components.NetworkManager] +test_labels = ["network_service_integration", "connectivity_validation", "service_lifecycle"] [components.OpenCL-ICD-Loader] [components.OpenIPMI] [components.PEGTL] [components.PackageKit] +test_labels = ["package_operations", "repository_management", "package_dependency_resolution"] [components.PackageKit-Qt] +test_labels = ["package_operations", "repository_management", "package_dependency_resolution"] [components.PyQt-builder] [components.PyQt4] [components.PyYAML] @@ -4719,6 +4726,7 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.rubypick] [components.ruff] [components.runc] +test_labels = ["container_lifecycle", "container_networking", "container_security", "oci_runtime_tests"] [components.rust-actix-codec] [components.rust-actix-http] [components.rust-actix-macros] diff --git a/base/comps/grub2/grub2.comp.toml b/base/comps/grub2/grub2.comp.toml index 3c388c8d574..a9b299731d4 100644 --- a/base/comps/grub2/grub2.comp.toml +++ b/base/comps/grub2/grub2.comp.toml @@ -1,4 +1,5 @@ [components.grub2] +test_labels = ["boot_integration", "uefi_compatibility", "secure_boot_validation"] # Disable Xen module builds — not applicable to Azure Linux (Hyper-V/KVM). # grub.macros enables with_xen_arch and with_xen_pvh_arch on x86_64 by default, diff --git a/base/comps/kernel-headers/kernel-headers.comp.toml b/base/comps/kernel-headers/kernel-headers.comp.toml index 1ff231aa549..f237196a31d 100644 --- a/base/comps/kernel-headers/kernel-headers.comp.toml +++ b/base/comps/kernel-headers/kernel-headers.comp.toml @@ -1,4 +1,5 @@ [components.kernel-headers] +test_labels = ["kmod_load", "kernel_integration", "boot_validation", "syscall_tests"] # Kernel-headers using Fedora spec with overlays for Azure Linux 6.18. # Fedora's spec expects a pre-built headers tarball. These overlays adapt it to # use Azure Linux kernel source and generate headers during build. diff --git a/base/comps/kernel/kernel.comp.toml b/base/comps/kernel/kernel.comp.toml index 9f568dc9314..ff79332936d 100644 --- a/base/comps/kernel/kernel.comp.toml +++ b/base/comps/kernel/kernel.comp.toml @@ -1,4 +1,5 @@ [components.kernel] +test_labels = ["kmod_load", "kernel_integration", "boot_validation", "syscall_tests"] # Manual release calculation required because the spec uses %{specrelease} macro # Release: %{pkg_release} diff --git a/base/comps/systemd/systemd.comp.toml b/base/comps/systemd/systemd.comp.toml index 25259e34c6d..e3ec7991bc5 100644 --- a/base/comps/systemd/systemd.comp.toml +++ b/base/comps/systemd/systemd.comp.toml @@ -1,4 +1,5 @@ [components.systemd] +test_labels = ["systemd_lifecycle", "service_management", "systemd_integration", "journal_validation"] # AZL's setup package is 2.15.0-1, but upstream Fedora 41+ systemd requires >= 2.15.0-3. # Lower the version threshold to match what's available in AZL. diff --git a/base/images/images.toml b/base/images/images.toml index e4ce97cb286..f1d5967ef1a 100644 --- a/base/images/images.toml +++ b/base/images/images.toml @@ -47,6 +47,15 @@ container = false systemd = true runtime-package-management = true +[images.vm-base.test-workflows] +# Test labels selected per validation tier. Labels are declared in +# base/tests/framework-labels.toml. Capability gating is applied by the +# resolver: labels whose `requires_capabilities` are not satisfied by +# this image's `[images.vm-base.capabilities]` block are skipped. +pr_validation = ["openqa_install_smoke", "tmt_tier0", "lisa_priority0", "lisa_smoke"] +nightly_validation = ["lisa_priority1", "lisa_priority2", "lisa_ltp", "lisa_xfs", "lisa_perf"] +release_validation = ["tmt_tier2", "lisa_priority2", "openqa_install_comprehensive", "security_tests", "compatibility_tests"] + [images.vm-base-dev] description = "VM Base Image (dev)" definition = { type = "kiwi", path = "vm-base/vm-base.kiwi", profile = "vm-base-dev" } @@ -65,6 +74,11 @@ container = false systemd = true runtime-package-management = true +[images.vm-base-dev.test-workflows] +pr_validation = ["openqa_install_smoke", "tmt_tier0", "lisa_priority0", "lisa_smoke"] +nightly_validation = ["lisa_priority1", "lisa_priority2", "lisa_ltp", "lisa_xfs", "lisa_perf"] +release_validation = ["tmt_tier2", "lisa_priority2", "openqa_install_comprehensive", "security_tests", "compatibility_tests"] + # ---- container-base (core profile) ------------------------------------- [images.container-base] description = "Container Base Image" @@ -80,6 +94,11 @@ container = true systemd = false runtime-package-management = true +[images.container-base.test-workflows] +pr_validation = ["container_smoke", "base_container_validation", "pytest_container_static"] +nightly_validation = ["container_runtime_full", "oci_compliance", "registry_integration", "pytest_container_static", "pytest_container_runtime"] +release_validation = ["container_security_scan", "oci_security", "distroless_validation", "pytest_container_release"] + [images.container-base-dev] description = "Container Base Image (dev)" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "core-dev" } @@ -94,6 +113,11 @@ container = true systemd = false runtime-package-management = true +[images.container-base-dev.test-workflows] +pr_validation = ["container_smoke", "base_container_validation", "pytest_container_static"] +nightly_validation = ["container_runtime_full", "oci_compliance", "registry_integration", "pytest_container_static", "pytest_container_runtime"] +release_validation = ["container_security_scan", "oci_security", "distroless_validation", "pytest_container_release"] + # ---- distroless containers (single-variant only) ----------------------- # No -dev sibling: with a single build-time repo and no runtime package # management, a -dev distroless build would be byte-identical. @@ -102,14 +126,23 @@ runtime-package-management = true description = "Container Distroless Minimal Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "distroless-minimal" } +[images.container-distroless-minimal.test-workflows] +release_validation = ["minimal_surface_tests", "security_surface", "static_analysis"] + [images.container-distroless-base] description = "Container Distroless Base Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "distroless-base" } +[images.container-distroless-base.test-workflows] +release_validation = ["minimal_surface_tests", "distroless_validation", "static_analysis"] + [images.container-distroless-debug] description = "Container Distroless Debug Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "distroless-debug" } +[images.container-distroless-debug.test-workflows] +release_validation = ["minimal_surface_tests", "distroless_validation", "static_analysis"] + # ---- wsl --------------------------------------------------------------- [images.wsl] diff --git a/base/tests/framework-labels.toml b/base/tests/framework-labels.toml new file mode 100644 index 00000000000..7059a6a88f2 --- /dev/null +++ b/base/tests/framework-labels.toml @@ -0,0 +1,400 @@ +# Azure Linux Framework Labels Catalog +# +# Declares the test labels referenced by: +# - base/images/images.toml ([images..test-workflows] tier maps) +# - base/comps/components.toml ([components.] `test_labels`) +# +# A label is a named, reusable test-selection record. Each label maps to +# a framework-native filter (`fmf_filter` / `lisa_criteria` / +# `openqa_suites` / `pytest_markers`) or is `pending = true` and emits +# no filter. +# +# Execution policy lives on the label, not on the gate: +# retry_count — integer; retries-on-failure for this label's runner +# timeout_min — integer; per-label wall-clock ceiling in minutes +# +# Capability tokens used by `requires_capabilities` here are produced by +# image definitions in base/images/images.toml (one bool per capability +# under each `[images..capabilities]` table). +# +# Ownership +# --------- +# - This file is owned by the Azure Linux team and lives in this repo +# so test selection is reviewable here. +# - azldev consumes this file as a generic resolver — it does NOT +# hard-code label semantics in Go. +# - The resolver dispatches on which filter field is present, not on +# the label name. Prefixes (tmt_, lisa_, ...) are descriptive only. + +[metadata] +version = "1.0.0" +schema_version = 1 +author = "Azure Linux Team" +description = "Azure Linux framework label catalog" + +# ============================================================================= +# Framework enablement +# +# Distro-wide kill switch for label dispatch. A label whose framework is +# disabled here is skipped (with a clear reason in --format explain) and +# its filter is not emitted. +# +# Defaults to enabled if a framework is not listed. +# ============================================================================= + +[frameworks] +tmt = { enabled = false } +lisa = { enabled = true } +openqa = { enabled = false } +pytest = { enabled = true } +image = { enabled = true } + +# ============================================================================= +# TMT labels (Test Management Tool / fmf filter syntax) +# ============================================================================= + +[tmt_labels.tmt_tier0] +description = "TMT smoke tier — fastest, most critical" +type = "tmt" +fmf_filter = "tier:0" +estimated_minutes = 15 +retry_count = 2 +timeout_min = 30 + +[tmt_labels.tmt_tier1] +description = "TMT integration tier — medium coverage" +type = "tmt" +fmf_filter = "tier:1" +estimated_minutes = 45 +retry_count = 1 +timeout_min = 90 + +[tmt_labels.tmt_tier2] +description = "TMT full tier — comprehensive" +type = "tmt" +fmf_filter = "tier:2" +estimated_minutes = 120 +retry_count = 1 +timeout_min = 240 + + +# ============================================================================= +# LISA labels (Linux Integration Services Automation) +# ============================================================================= + +[lisa_labels.lisa_priority0] +description = "LISA PR-gate priority — critical path" +type = "lisa" +lisa_criteria = { priority = [0] } +estimated_minutes = 20 +retry_count = 2 +timeout_min = 40 + +[lisa_labels.lisa_smoke] +description = "LISA smoke test — fastest PR gate" +type = "lisa" +lisa_criteria = { testcaseNames = ["smoke_test"] } +estimated_minutes = 10 +retry_count = 2 +timeout_min = 20 + +[lisa_labels.lisa_priority1] +description = "LISA important functional tests" +type = "lisa" +lisa_criteria = { priority = [1] } +estimated_minutes = 60 +retry_count = 1 +timeout_min = 120 + +[lisa_labels.lisa_priority2] +description = "LISA extended coverage" +type = "lisa" +lisa_criteria = { priority = [2] } +estimated_minutes = 180 +retry_count = 1 +timeout_min = 360 + +[lisa_labels.lisa_ltp] +description = "LISA kernel LTP (Linux Test Project) suite" +type = "lisa" +lisa_criteria = { testcaseNames = ["verify_ltp_lite"] } +estimated_minutes = 90 +retry_count = 1 +timeout_min = 180 + +[lisa_labels.lisa_xfs] +description = "LISA XFS data-disk tests (standard + NVMe)" +type = "lisa" +lisa_criteria = { testcaseNames = ["verify_xfs_standard_datadisk", "verify_xfs_nvme_datadisk"] } +requires_capabilities = ["storage"] +estimated_minutes = 60 +retry_count = 1 +timeout_min = 120 + +[lisa_labels.lisa_perf] +description = "LISA performance test areas (network, nested, nvme, perf_tool, storage)" +type = "lisa" +lisa_criteria = { criteria = [ + { area = "network", category = "performance" }, + { area = "nested", category = "performance" }, + { area = "nvme", category = "performance" }, + { area = "perf_tool", category = "performance" }, + { area = "storage", category = "performance" }, +] } +estimated_minutes = 240 +retry_count = 1 +timeout_min = 480 + + +# ============================================================================= +# OpenQA labels (suite-list selection) +# ============================================================================= + +[openqa_labels.openqa_install_smoke] +description = "OpenQA basic installation validation" +type = "openqa" +openqa_suites = ["install_default", "install_package_set_minimal"] +estimated_minutes = 30 +retry_count = 2 +timeout_min = 60 + +[openqa_labels.openqa_install_comprehensive] +description = "OpenQA full installation matrix" +type = "openqa" +openqa_suites = [ + "install_default", + "install_delete_pata", + "install_sata", + "install_multi", + "install_software_raid", + "install_lvmthin", +] +estimated_minutes = 120 +retry_count = 1 +timeout_min = 240 + + +# ============================================================================= +# pytest labels (in-tree, filesystem-based image tests) +# +# Pytest labels target image tests that live under base/images/tests/cases/ +# and are dispatched via `pytest`. Each label declares: +# - `pytest_markers` : -m expressions to select tests +# - `pytest_files` : optional path globs to limit collection +# - `pytest_args` : optional extra args passed verbatim +# +# The resolver groups all pytest labels in a workflow into a single +# combined invocation (union of markers/files, ordered args) so the +# runner can fire one pytest process per workflow. +# ============================================================================= + +[pytest_labels.pytest_container_static] +description = "Static container validation (image-tree pytest)" +type = "pytest" +pytest_markers = ["static_container_test"] +pytest_files = ["cases/**/*.py"] +requires_capabilities = ["container"] +estimated_minutes = 25 +retry_count = 2 +timeout_min = 50 + +[pytest_labels.pytest_container_runtime] +description = "Runtime container behavior (image-tree pytest)" +type = "pytest" +pytest_markers = ["runtime_container_tests"] +pytest_files = ["cases/**/*.py"] +requires_capabilities = ["container"] +estimated_minutes = 45 +retry_count = 1 +timeout_min = 90 + +[pytest_labels.pytest_vm_basic] +description = "In-VM pytest smoke tests" +type = "pytest" +pytest_markers = ["vm_basic"] +pytest_files = ["cases/**/*.py"] +requires_capabilities = ["machine-bootable"] +estimated_minutes = 30 +retry_count = 2 +timeout_min = 60 + +[pytest_labels.pytest_container_release] +description = "Release-gate container pytest with coverage collection" +type = "pytest" +pytest_markers = ["static_container_test"] +pytest_files = ["cases/**/*.py"] +pytest_args = [ + "--strict-markers", + "--durations=20", + "--cov=base/images/tests/cases", + "--cov-report=term-missing", +] +requires_capabilities = ["container"] +estimated_minutes = 60 +retry_count = 1 +timeout_min = 120 + + + +# Each label is `pending = true` until a concrete framework binding (pytest +# marker, lisa criteria, etc.) is provided. +[image_labels] +vm_smoke = { description = "Lightweight VM smoke validation", requires_capabilities = ["machine-bootable"], estimated_minutes = 20, pending = true } +vm_basic_tests = { description = "Basic VM functional coverage", requires_capabilities = ["machine-bootable"], estimated_minutes = 40, pending = true } +container_smoke = { description = "Lightweight container smoke validation", requires_capabilities = ["container"], estimated_minutes = 15, pending = true } +base_container_validation = { description = "Base-image container validation", requires_capabilities = ["container"], estimated_minutes = 25, pending = true } +distroless_validation = { description = "Distroless / minimal image validation", requires_capabilities = ["container"], estimated_minutes = 30, pending = true } +minimal_surface_tests = { description = "Validation of minimal image surface", estimated_minutes = 20, pending = true } +oci_compliance = { description = "OCI specification compliance", requires_capabilities = ["container"], estimated_minutes = 25, pending = true } +oci_security = { description = "OCI security validation", requires_capabilities = ["container"], estimated_minutes = 30, pending = true } +registry_integration = { description = "Container registry integration", requires_capabilities = ["container"], estimated_minutes = 20, pending = true } +container_security_scan = { description = "Container security scanning", requires_capabilities = ["container"], estimated_minutes = 40, pending = true } + +# ============================================================================= +# Component dynamic labels +# +# Referenced from `test_labels = [...]` on individual +# `[components.]` entries in base/comps/components.toml. All are +# currently `pending = true` — the references are aspirational metadata. +# Concrete filters should be added incrementally as each component's +# dynamic test infrastructure lands. +# ============================================================================= + +[component_labels] +# Kernel & boot +kmod_load = { description = "Kernel module load/unload", pending = true } +kernel_integration = { description = "Kernel runtime integration", pending = true } +boot_validation = { description = "Boot path validation", pending = true } +boot_integration = { description = "Boot process integration", pending = true } +syscall_tests = { description = "Syscall surface validation", pending = true } +uefi_compatibility = { description = "UEFI compatibility", pending = true } +secure_boot_validation = { description = "Secure boot validation", pending = true } +firmware_loading = { description = "Firmware loading", pending = true } +hardware_compatibility = { description = "Hardware compatibility", pending = true } +microcode_update = { description = "Microcode update flow", pending = true } + +# System services +systemd_lifecycle = { description = "systemd lifecycle", pending = true } +systemd_integration = { description = "systemd integration", pending = true } +service_management = { description = "Service management", pending = true } +service_lifecycle = { description = "Service lifecycle", pending = true } +journal_validation = { description = "journald validation", pending = true } +network_service_integration = { description = "Network service integration", pending = true } +connectivity_validation = { description = "Connectivity validation", pending = true } +selinux_enforcement = { description = "SELinux enforcement", pending = true } +tpm_functionality = { description = "TPM functionality", pending = true } +crypto_operations = { description = "Crypto operations", pending = true } +security_integration = { description = "Security stack integration", pending = true } +storage_integration = { description = "Storage integration", pending = true } +filesystem_operations = { description = "Filesystem operations", pending = true } +device_management = { description = "Device management", pending = true } + +# Toolchains & languages +compiler_runtime_tests = { description = "Compiler runtime tests", pending = true } +cross_compile_validation = { description = "Cross-compilation validation", pending = true } +glibc_functionality = { description = "glibc functionality", pending = true } +python_runtime_tests = { description = "Python runtime tests", pending = true } +python_module_integration = { description = "Python module integration", pending = true } +python_package_functionality = { description = "Python package functionality", pending = true } +rust_runtime_tests = { description = "Rust runtime tests", pending = true } +rust_integration = { description = "Rust integration", pending = true } +rust_performance = { description = "Rust performance", pending = true } +cargo_test = { description = "Cargo test execution", pending = true } +go_test = { description = "Go test execution", pending = true } +go_integration = { description = "Go integration", pending = true } +go_performance = { description = "Go performance", pending = true } +go_race_detection = { description = "Go race detection", pending = true } +nodejs_runtime_tests = { description = "Node.js runtime tests", pending = true } +nodejs_performance = { description = "Node.js performance", pending = true } +npm_integration = { description = "npm integration", pending = true } +java_runtime_tests = { description = "Java runtime tests", pending = true } +java_application_tests = { description = "Java application tests", pending = true } +maven_integration = { description = "Maven integration", pending = true } + +# Container & virtualization +container_lifecycle = { description = "Container lifecycle", pending = true } +container_networking = { description = "Container networking", pending = true } +container_security = { description = "Container security", pending = true } +oci_runtime_tests = { description = "OCI runtime tests", pending = true } +vm_lifecycle = { description = "VM lifecycle", pending = true } +vm_migration = { description = "VM migration", pending = true } +libvirt_integration = { description = "libvirt integration", pending = true } +hypervisor_performance = { description = "Hypervisor performance", pending = true } + +# Databases & data +mariadb_functionality = { description = "MariaDB functionality", pending = true } +mariadb_replication = { description = "MariaDB replication", pending = true } +mysql_functionality = { description = "MySQL functionality", pending = true } +mysql_clustering = { description = "MySQL clustering", pending = true } +mysql_performance = { description = "MySQL performance", pending = true } +postgresql_integration = { description = "PostgreSQL integration", pending = true } +postgresql_extension_tests = { description = "PostgreSQL extension tests", pending = true } +database_performance = { description = "Database performance", pending = true } + +# Networking & protocols +network_integration = { description = "Network integration", pending = true } +network_performance = { description = "Network performance", pending = true } +network_protocol_tests = { description = "Network protocol tests", pending = true } +network_tool_functionality = { description = "Network tool functionality", pending = true } +protocol_tests = { description = "Protocol tests", pending = true } +http_load_tests = { description = "HTTP load tests", pending = true } +web_server_functionality = { description = "Web server functionality", pending = true } +web_performance = { description = "Web performance", pending = true } + +# Security & auth +auth_integration = { description = "Authentication integration", pending = true } +kerberos_functionality = { description = "Kerberos functionality", pending = true } +ldap_operations = { description = "LDAP operations", pending = true } +crypto_functionality = { description = "Crypto functionality", pending = true } +encryption_tests = { description = "Encryption tests", pending = true } +compliance_validation = { description = "Compliance validation", pending = true } +security_monitoring = { description = "Security monitoring", pending = true } + +# Filesystem & storage +fs_integrity_tests = { description = "Filesystem integrity tests", pending = true } +fs_performance = { description = "Filesystem performance", pending = true } +archive_operations = { description = "Archive operations", pending = true } +compression_performance = { description = "Compression performance", pending = true } +backup_functionality = { description = "Backup functionality", pending = true } + +# Build & dev tools +build_system_tests = { description = "Build system tests", pending = true } +build_performance = { description = "Build performance", pending = true } +cross_platform_build = { description = "Cross-platform build", pending = true } +debugger_functionality = { description = "Debugger functionality", pending = true } +profiler_tests = { description = "Profiler tests", pending = true } +symbol_resolution = { description = "Symbol resolution", pending = true } +library_integration = { description = "Library integration", pending = true } +library_runtime_tests = { description = "Library runtime tests", pending = true } +package_operations = { description = "Package operations", pending = true } +package_dependency_resolution = { description = "Package dependency resolution", pending = true } +repository_management = { description = "Repository management", pending = true } +vcs_operations = { description = "VCS operations", pending = true } +vcs_integration = { description = "VCS integration", pending = true } +vcs_performance = { description = "VCS performance", pending = true } +test_execution = { description = "Test execution", pending = true } +test_framework_integration = { description = "Test framework integration", pending = true } +test_reporting = { description = "Test reporting", pending = true } +monitoring_functionality = { description = "Monitoring functionality", pending = true } +log_processing = { description = "Log processing", pending = true } +performance_tuning = { description = "Performance tuning", pending = true } + +# Desktop / GUI / multimedia +gui_interaction = { description = "GUI interaction", pending = true } +gui_rendering = { description = "GUI rendering", pending = true } +gui_performance = { description = "GUI performance", pending = true } +graphics_rendering = { description = "Graphics rendering", pending = true } +graphics_performance = { description = "Graphics performance", pending = true } +font_rendering = { description = "Font rendering", pending = true } +multimedia_playback = { description = "Multimedia playback", pending = true } +multimedia_processing = { description = "Multimedia processing", pending = true } +multimedia_performance = { description = "Multimedia performance", pending = true } +audio_processing = { description = "Audio processing", pending = true } +text_editor_functionality = { description = "Text editor functionality", pending = true } +text_processing = { description = "Text processing", pending = true } +text_processing_functionality = { description = "Text processing functionality", pending = true } +file_manager_tests = { description = "File manager tests", pending = true } +spell_check_tests = { description = "Spell check tests", pending = true } +productivity_integration = { description = "Productivity integration", pending = true } +language_processing = { description = "Language processing", pending = true } +internationalization_tests = { description = "Internationalization tests", pending = true } From 19f3aa48a2eed3b3d3b995d34c98159378dcc6a3 Mon Sep 17 00:00:00 2001 From: Bhagyashri Date: Wed, 3 Jun 2026 23:21:27 +0530 Subject: [PATCH 2/3] Adopt schmea plus merge conflicts --- base/comps/bash/bash.comp.toml | 1 + base/comps/bash/bash.tests.toml | 22 + base/comps/components.toml | 16 +- base/comps/container-runtime.tests.toml | 27 ++ base/comps/containerd/containerd.comp.toml | 2 + base/comps/cri-o/cri-o.comp.toml | 2 + base/comps/crun/crun.comp.toml | 2 + base/comps/grub2/grub2.comp.toml | 1 - .../kernel-headers/kernel-headers.comp.toml | 2 +- base/comps/kernel-tests.toml | 24 ++ base/comps/kernel/kernel.comp.toml | 2 +- base/comps/moby-engine/moby-engine.comp.toml | 2 + base/comps/podman/podman.comp.toml | 2 + base/comps/runc/runc.comp.toml | 2 + base/comps/systemd/systemd.comp.toml | 1 - base/images/images.tests.toml | 158 +++++++ base/images/images.toml | 125 +----- base/project.toml | 2 +- base/tests/framework-labels.toml | 400 ------------------ 19 files changed, 259 insertions(+), 534 deletions(-) create mode 100644 base/comps/bash/bash.tests.toml create mode 100644 base/comps/container-runtime.tests.toml create mode 100644 base/comps/containerd/containerd.comp.toml create mode 100644 base/comps/cri-o/cri-o.comp.toml create mode 100644 base/comps/crun/crun.comp.toml create mode 100644 base/comps/kernel-tests.toml create mode 100644 base/comps/moby-engine/moby-engine.comp.toml create mode 100644 base/comps/podman/podman.comp.toml create mode 100644 base/comps/runc/runc.comp.toml create mode 100644 base/images/images.tests.toml delete mode 100644 base/tests/framework-labels.toml diff --git a/base/comps/bash/bash.comp.toml b/base/comps/bash/bash.comp.toml index 5ef0d2922be..f42faa8471a 100644 --- a/base/comps/bash/bash.comp.toml +++ b/base/comps/bash/bash.comp.toml @@ -2,3 +2,4 @@ # Pin to Fedora 44 dist-git head as of the AZL4 Public Preview cutoff. # F44 carries the build fixes we needed from rawhide; F43 does not. spec = { type = "upstream", upstream-distro = { name = "fedora", version = "44" }, upstream-commit = "a6bcc6767229199f4f02b781d1d39df0835d894b" } +tests.tests = [{ name = "bash-fedora-shell" }] diff --git a/base/comps/bash/bash.tests.toml b/base/comps/bash/bash.tests.toml new file mode 100644 index 00000000000..eac846cbef4 --- /dev/null +++ b/base/comps/bash/bash.tests.toml @@ -0,0 +1,22 @@ +# ============================================================================ +# bash — component-local test definitions +# ============================================================================ +# +# Fedora's bash dist-git ships a tmt plan at /plans/shell. We pin the tmt +# source to the *same* upstream commit that bash.comp.toml pins for the +# spec, so the test plan moves in lockstep with the version of bash we +# actually ship. (When the spec pin moves, this SHA moves with it.) +# ============================================================================ + +[tests.bash-fedora-shell] +type = "tmt" +description = "Fedora bash dist-git tmt smoke (plans/shell), lock-stepped with the bash spec pin" +kind = ["functional"] +# long-running omitted: virtual provision + a small shell plan finishes in minutes. + +[tests.bash-fedora-shell.tmt] +source = { git-url = "https://src.fedoraproject.org/rpms/bash.git", ref = "a6bcc6767229199f4f02b781d1d39df0835d894b" } +plan = "/plans/shell" + +[tests.bash-fedora-shell.tmt.provision] +how = "virtual" diff --git a/base/comps/components.toml b/base/comps/components.toml index d0722ffbc0b..bef1e8fdf52 100644 --- a/base/comps/components.toml +++ b/base/comps/components.toml @@ -1,4 +1,4 @@ -includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"] +includes = ["**/*.comp.toml", "**/*.tests.toml", "container-runtime.tests.toml", "kernel-tests.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"] # # Components imported from Fedora with no per-component modifications. @@ -7,35 +7,28 @@ includes = ["**/*.comp.toml", "component-check-disablement.toml", "component-min [components.389-ds-base] [components.7zip] -test_labels = ["archive_operations", "compression_performance", "backup_functionality"] [components.AMF] [components.CGAL] [components.CUnit] -test_labels = ["test_execution", "test_framework_integration", "test_reporting"] [components.CharLS] [components.CppMicroServices] [components.Cython] [components.DevIL] [components.FP16] [components.GitPython] -test_labels = ["vcs_operations", "vcs_performance", "vcs_integration"] [components.HdrHistogram_c] [components.JUnitParams] -test_labels = ["test_execution", "test_framework_integration", "test_reporting"] [components.Judy] [components.LibRaw] [components.Lmod] [components.MUMPS] [components.ModemManager] [components.NetworkManager] -test_labels = ["network_service_integration", "connectivity_validation", "service_lifecycle"] [components.OpenCL-ICD-Loader] [components.OpenIPMI] [components.PEGTL] [components.PackageKit] -test_labels = ["package_operations", "repository_management", "package_dependency_resolution"] [components.PackageKit-Qt] -test_labels = ["package_operations", "repository_management", "package_dependency_resolution"] [components.PyQt-builder] [components.PyQt4] [components.PyYAML] @@ -263,7 +256,6 @@ test_labels = ["package_operations", "repository_management", "package_dependenc [components.console-login-helper-messages] [components.console-setup] [components.container-selinux] -[components.containerd] [components.containers-common] [components.convmv] [components.corosync] @@ -279,14 +271,12 @@ test_labels = ["package_operations", "repository_management", "package_dependenc [components.crash-gcore-command] [components.crash-trace-command] [components.createrepo_c] -[components.cri-o] [components.cri-tools] [components.'cri-tools1.35'] [components.criu] [components.cronie] [components.crontabs] [components.cross-binutils] -[components.crun] [components.cryptsetup] [components.cscope] [components.ctags] @@ -2115,7 +2105,6 @@ test_labels = ["package_operations", "repository_management", "package_dependenc [components.mksh] [components.mm-common] [components.mobile-broadband-provider-info] -[components.moby-engine] [components.mock] [components.mockito] [components.mod_auth_gssapi] @@ -3306,7 +3295,6 @@ test_labels = ["package_operations", "repository_management", "package_dependenc [components.pmix] [components.pngcrush] [components.pocketfft] -[components.podman] [components.poetry] [components.policycoreutils] [components.polkit] @@ -4725,8 +4713,6 @@ test_labels = ["package_operations", "repository_management", "package_dependenc [components.rubygem-zeitwerk] [components.rubypick] [components.ruff] -[components.runc] -test_labels = ["container_lifecycle", "container_networking", "container_security", "oci_runtime_tests"] [components.rust-actix-codec] [components.rust-actix-http] [components.rust-actix-macros] diff --git a/base/comps/container-runtime.tests.toml b/base/comps/container-runtime.tests.toml new file mode 100644 index 00000000000..4c199244cba --- /dev/null +++ b/base/comps/container-runtime.tests.toml @@ -0,0 +1,27 @@ +# ============================================================================ +# container-runtime — shared test-group spanning multiple runtime components +# ============================================================================ +# +# This file defines a single test-group that all container-runtime-shaped +# components (those that *run* containers) associate themselves with. Each +# member component carries `tests.tests = [{ group = "container-runtime" }]` +# in its own .comp.toml; the group itself is defined once here. +# +# Note on file placement: shared test-groups that span multiple components +# don't fit cleanly into either the image-level (base/images/images.tests.toml) +# or the per-component (/.tests.toml) homes. This flat root-level +# placement parallels how component-mingw-disablement.toml etc. live at the +# comps root. The pattern won't scale to many such files; revisit when we +# accumulate more cross-component test-groups. +# ============================================================================ + +[test-groups.container-runtime] +description = "Container runtime scenario tests: black-box exercise of whatever runtime is installed (containerd / runc / crun / podman / cri-o / moby-engine)" +# TODO: populate with a runtime-agnostic scenario test once authored. +# The rationale for the group is precisely to run a *scenario-level* test +# (e.g. pull → run → exec → stop a container) that exercises any of the +# member runtimes through their common interface — not the runtime-specific +# tests each upstream dist-git happens to carry. No such homegrown test +# exists in the tree yet, so the member list starts empty and the group +# acts purely as a stable identity for component associations. +tests = [] diff --git a/base/comps/containerd/containerd.comp.toml b/base/comps/containerd/containerd.comp.toml new file mode 100644 index 00000000000..ff244e2431c --- /dev/null +++ b/base/comps/containerd/containerd.comp.toml @@ -0,0 +1,2 @@ +[components.containerd] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/cri-o/cri-o.comp.toml b/base/comps/cri-o/cri-o.comp.toml new file mode 100644 index 00000000000..f9a5d8973d2 --- /dev/null +++ b/base/comps/cri-o/cri-o.comp.toml @@ -0,0 +1,2 @@ +[components.cri-o] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/crun/crun.comp.toml b/base/comps/crun/crun.comp.toml new file mode 100644 index 00000000000..eb51ac34400 --- /dev/null +++ b/base/comps/crun/crun.comp.toml @@ -0,0 +1,2 @@ +[components.crun] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/grub2/grub2.comp.toml b/base/comps/grub2/grub2.comp.toml index a9b299731d4..3c388c8d574 100644 --- a/base/comps/grub2/grub2.comp.toml +++ b/base/comps/grub2/grub2.comp.toml @@ -1,5 +1,4 @@ [components.grub2] -test_labels = ["boot_integration", "uefi_compatibility", "secure_boot_validation"] # Disable Xen module builds — not applicable to Azure Linux (Hyper-V/KVM). # grub.macros enables with_xen_arch and with_xen_pvh_arch on x86_64 by default, diff --git a/base/comps/kernel-headers/kernel-headers.comp.toml b/base/comps/kernel-headers/kernel-headers.comp.toml index f237196a31d..9a62375502a 100644 --- a/base/comps/kernel-headers/kernel-headers.comp.toml +++ b/base/comps/kernel-headers/kernel-headers.comp.toml @@ -1,5 +1,5 @@ [components.kernel-headers] -test_labels = ["kmod_load", "kernel_integration", "boot_validation", "syscall_tests"] +tests.tests = [{ group = "kernel-tests" }] # Kernel-headers using Fedora spec with overlays for Azure Linux 6.18. # Fedora's spec expects a pre-built headers tarball. These overlays adapt it to # use Azure Linux kernel source and generate headers during build. diff --git a/base/comps/kernel-tests.toml b/base/comps/kernel-tests.toml new file mode 100644 index 00000000000..d8d4558dada --- /dev/null +++ b/base/comps/kernel-tests.toml @@ -0,0 +1,24 @@ +# ============================================================================ +# kernel-tests — shared test-group for kernel-shaped components +# ============================================================================ +# +# Bundles LISA kernel-oriented tests (LTP + XFS filesystem) into a stable +# group name that kernel-adjacent components (kernel, kernel-headers, ...) +# reference from their own .comp.toml via: +# +# tests.tests = [{ group = "kernel-tests" }] +# +# The underlying tests themselves are defined in +# base/images/images.tests.toml. This file only declares the group identity +# and its membership so component owners can opt in with a single line. +# +# File placement mirrors container-runtime.tests.toml: a flat root-level +# cross-component test-group lives at base/comps/.tests.toml. +# ============================================================================ + +[test-groups.kernel-tests] +description = "Kernel-oriented LISA coverage (LTP + XFS filesystem) for kernel-shaped components" +tests = [ + { name = "kernel_ltp" }, + { name = "xfs_filesystem" }, +] diff --git a/base/comps/kernel/kernel.comp.toml b/base/comps/kernel/kernel.comp.toml index ff79332936d..b77b71e8c61 100644 --- a/base/comps/kernel/kernel.comp.toml +++ b/base/comps/kernel/kernel.comp.toml @@ -1,5 +1,5 @@ [components.kernel] -test_labels = ["kmod_load", "kernel_integration", "boot_validation", "syscall_tests"] +tests.tests = [{ group = "kernel-tests" }] # Manual release calculation required because the spec uses %{specrelease} macro # Release: %{pkg_release} diff --git a/base/comps/moby-engine/moby-engine.comp.toml b/base/comps/moby-engine/moby-engine.comp.toml new file mode 100644 index 00000000000..e086b0864de --- /dev/null +++ b/base/comps/moby-engine/moby-engine.comp.toml @@ -0,0 +1,2 @@ +[components.moby-engine] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/podman/podman.comp.toml b/base/comps/podman/podman.comp.toml new file mode 100644 index 00000000000..b9c24deaf81 --- /dev/null +++ b/base/comps/podman/podman.comp.toml @@ -0,0 +1,2 @@ +[components.podman] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/runc/runc.comp.toml b/base/comps/runc/runc.comp.toml new file mode 100644 index 00000000000..eeb7c7cf64a --- /dev/null +++ b/base/comps/runc/runc.comp.toml @@ -0,0 +1,2 @@ +[components.runc] +tests.tests = [{ group = "container-runtime" }] diff --git a/base/comps/systemd/systemd.comp.toml b/base/comps/systemd/systemd.comp.toml index e3ec7991bc5..25259e34c6d 100644 --- a/base/comps/systemd/systemd.comp.toml +++ b/base/comps/systemd/systemd.comp.toml @@ -1,5 +1,4 @@ [components.systemd] -test_labels = ["systemd_lifecycle", "service_management", "systemd_integration", "journal_validation"] # AZL's setup package is 2.15.0-1, but upstream Fedora 41+ systemd requires >= 2.15.0-3. # Lower the version threshold to match what's available in AZL. diff --git a/base/images/images.tests.toml b/base/images/images.tests.toml new file mode 100644 index 00000000000..71e922a2e76 --- /dev/null +++ b/base/images/images.tests.toml @@ -0,0 +1,158 @@ +# ============================================================================ +# Image test catalog +# ============================================================================ +# +# Single source of truth for all image-runnable test definitions and +# the named groups that bundle them. +# +# Schema: +# [tests.X] : a single unit of testing — one configuration of +# one runner/harness, with framework-specific +# options under [tests.X.]. +# [test-groups.X] : a named bundle of tests an image (or component) +# can reference instead of enumerating members. +# +# Image and component associations are NOT defined here. They live with +# the entity that runs them: +# - base/images/images.toml (tests.tests = [...]) +# - base/comps//.comp.toml or +# base/comps//.tests.toml (tests.tests = [...]) +# +# Both fields can mix per-test and per-group references: +# tests.tests = [{ name = "static-image-checks" }, { group = "x" }] +# +# Declarative metadata on [tests.X]: +# description : human-readable summary +# type : "pytest" | "tmt" | "lisa" | "openqa" | "pending" +# kind : ["functional"] | ["performance"] | both +# long-running : boolean; omit unless the test may run for hours +# required-capabilities : capability tokens the image must declare +# +# ============================================================================ + +# ============================================================================= +# Static image checks (pytest) +# ============================================================================= + +# Single shared pytest suite for static (offline) image validation. +[tests.static-image-checks] +type = "pytest" +description = "Offline image validation (shared + image-specific tests)" +kind = ["functional"] + +[tests.static-image-checks.pytest] +working-dir = "tests" +install = "pyproject" +test-paths = ["cases/static/"] +# {capabilities} is substituted by azldev as a comma-separated list of +# the capabilities set to `true` for the image (e.g. +# "machine-bootable,systemd,runtime-package-management"). The pytest +# plugin parses this into a set; @pytest.mark.require_capability("...") +# tests that don't match are skipped. +extra-args = [ + "--image-path", "{image-path}", + "--image-name", "{image-name}", + "--capabilities", "{capabilities}", +] + +# Runtime container tests — validate live container behavior via podman. +[tests.runtime-container-tests] +type = "pytest" +description = "Runtime container validation (exec into live containers)" +kind = ["functional"] +required-capabilities = ["container"] + +[tests.runtime-container-tests.pytest] +working-dir = "tests" +install = "pyproject" +test-paths = ["cases/runtime/"] +extra-args = [ + "--image-path", "{image-path}", + "--image-name", "{image-name}", + "--capabilities", "{capabilities}", +] + +# ============================================================================= +# LISA tests +# ============================================================================= + +[tests.smoke_test] +type = "lisa" +description = "LISA smoke test — fastest PR gate" +kind = ["functional"] +required-capabilities = ["machine-bootable"] +[tests.smoke_test.lisa] +criteria = { testcaseNames = ["smoke_test"] } + +[tests.functional_core] +type = "lisa" +description = "Core functional coverage (LISA priority=1)" +kind = ["functional"] +required-capabilities = ["machine-bootable"] +[tests.functional_core.lisa] +criteria = { priority = [1] } + +[tests.functional_extended] +type = "lisa" +description = "Extended functional coverage (LISA priority=2)" +kind = ["functional"] +long-running = true +required-capabilities = ["machine-bootable"] +[tests.functional_extended.lisa] +criteria = { priority = [2] } + +[tests.kernel_ltp] +type = "lisa" +description = "Kernel LTP (Linux Test Project) suite" +kind = ["functional"] +long-running = true +required-capabilities = ["machine-bootable"] + +[tests.kernel_ltp.lisa] +criteria = { testcaseNames = ["verify_ltp_lite"] } + +[tests.xfs_filesystem] +type = "lisa" +description = "XFS data-disk filesystem tests (standard + NVMe)" +kind = ["functional"] +required-capabilities = ["machine-bootable"] + +[tests.xfs_filesystem.lisa] +criteria = { testcaseNames = ["verify_xfs_standard_datadisk", "verify_xfs_nvme_datadisk"] } + +[tests.lisa_perf] +type = "lisa" +description = "LISA performance test areas (network, nested, nvme, perf_tool, storage)" +kind = ["performance"] +long-running = true +required-capabilities = ["machine-bootable"] +[tests.lisa_perf.lisa] +criteria = [ + { area = "network", category = "performance" }, + { area = "nested", category = "performance" }, + { area = "nvme", category = "performance" }, + { area = "perf_tool", category = "performance" }, + { area = "storage", category = "performance" }, +] + +# ---- VM base test groups ------------------------------------------------- + +# Functional coverage applicable to a VM-shaped Azure Linux image. +[test-groups.vm-base-functional] +description = "Functional coverage for VM base images" +tests = [ + "smoke_test", + "functional_core", + "functional_extended", + "kernel_ltp", + "xfs_filesystem", +] + +# Performance-oriented coverage. Kept as a separate group because +# performance runs typically have a different cadence and budget than +# functional runs, and consumers may want to opt in independently. +[test-groups.vm-base-performance] +description = "Performance coverage for VM base images" +tests = [ + "lisa_perf", +] diff --git a/base/images/images.toml b/base/images/images.toml index f1d5967ef1a..6b2e5d8fa22 100644 --- a/base/images/images.toml +++ b/base/images/images.toml @@ -22,7 +22,7 @@ # `` points at azl4-dev for local builds; koji overrides # this for distro builds). # -# Capabilities and test-suites are identical between a canonical / `-dev` +# Capabilities and tests are identical between a canonical / `-dev` # pair (the variants only differ in what `-repos` package ships and, # for `core`, the OCI tag). # ============================================================================ @@ -32,13 +32,10 @@ [images.vm-base] description = "VM Base Image" definition = { type = "kiwi", path = "vm-base/vm-base.kiwi", profile = "vm-base" } -tests.test-suites = [ - { name = "static-image-checks" }, - { name = "lisa-main" }, - { name = "lisa-xfs" }, - { name = "lisa-perf" }, - { name = "lisa-smoke" }, - { name = "lisa-kernel-ltp" }, +tests.tests = [ + { name = "static-image-checks" }, + { group = "vm-base-functional" }, + { group = "vm-base-performance" }, ] [images.vm-base.capabilities] @@ -47,25 +44,13 @@ container = false systemd = true runtime-package-management = true -[images.vm-base.test-workflows] -# Test labels selected per validation tier. Labels are declared in -# base/tests/framework-labels.toml. Capability gating is applied by the -# resolver: labels whose `requires_capabilities` are not satisfied by -# this image's `[images.vm-base.capabilities]` block are skipped. -pr_validation = ["openqa_install_smoke", "tmt_tier0", "lisa_priority0", "lisa_smoke"] -nightly_validation = ["lisa_priority1", "lisa_priority2", "lisa_ltp", "lisa_xfs", "lisa_perf"] -release_validation = ["tmt_tier2", "lisa_priority2", "openqa_install_comprehensive", "security_tests", "compatibility_tests"] - [images.vm-base-dev] description = "VM Base Image (dev)" definition = { type = "kiwi", path = "vm-base/vm-base.kiwi", profile = "vm-base-dev" } -tests.test-suites = [ - { name = "static-image-checks" }, - { name = "lisa-main" }, - { name = "lisa-xfs" }, - { name = "lisa-perf" }, - { name = "lisa-smoke" }, - { name = "lisa-kernel-ltp" }, +tests.tests = [ + { name = "static-image-checks" }, + { group = "vm-base-functional" }, + { group = "vm-base-performance" }, ] [images.vm-base-dev.capabilities] @@ -74,16 +59,11 @@ container = false systemd = true runtime-package-management = true -[images.vm-base-dev.test-workflows] -pr_validation = ["openqa_install_smoke", "tmt_tier0", "lisa_priority0", "lisa_smoke"] -nightly_validation = ["lisa_priority1", "lisa_priority2", "lisa_ltp", "lisa_xfs", "lisa_perf"] -release_validation = ["tmt_tier2", "lisa_priority2", "openqa_install_comprehensive", "security_tests", "compatibility_tests"] - # ---- container-base (core profile) ------------------------------------- [images.container-base] description = "Container Base Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "core" } -tests.test-suites = [ +tests.tests = [ { name = "static-image-checks" }, { name = "runtime-container-tests" }, ] @@ -94,15 +74,10 @@ container = true systemd = false runtime-package-management = true -[images.container-base.test-workflows] -pr_validation = ["container_smoke", "base_container_validation", "pytest_container_static"] -nightly_validation = ["container_runtime_full", "oci_compliance", "registry_integration", "pytest_container_static", "pytest_container_runtime"] -release_validation = ["container_security_scan", "oci_security", "distroless_validation", "pytest_container_release"] - [images.container-base-dev] description = "Container Base Image (dev)" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "core-dev" } -tests.test-suites = [ +tests.tests = [ { name = "static-image-checks" }, { name = "runtime-container-tests" }, ] @@ -113,11 +88,6 @@ container = true systemd = false runtime-package-management = true -[images.container-base-dev.test-workflows] -pr_validation = ["container_smoke", "base_container_validation", "pytest_container_static"] -nightly_validation = ["container_runtime_full", "oci_compliance", "registry_integration", "pytest_container_static", "pytest_container_runtime"] -release_validation = ["container_security_scan", "oci_security", "distroless_validation", "pytest_container_release"] - # ---- distroless containers (single-variant only) ----------------------- # No -dev sibling: with a single build-time repo and no runtime package # management, a -dev distroless build would be byte-identical. @@ -126,23 +96,14 @@ release_validation = ["container_security_scan", "oci_security", "distroless_val description = "Container Distroless Minimal Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "distroless-minimal" } -[images.container-distroless-minimal.test-workflows] -release_validation = ["minimal_surface_tests", "security_surface", "static_analysis"] - [images.container-distroless-base] description = "Container Distroless Base Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "distroless-base" } -[images.container-distroless-base.test-workflows] -release_validation = ["minimal_surface_tests", "distroless_validation", "static_analysis"] - [images.container-distroless-debug] description = "Container Distroless Debug Image" definition = { type = "kiwi", path = "container-base/container-base.kiwi", profile = "distroless-debug" } -[images.container-distroless-debug.test-workflows] -release_validation = ["minimal_surface_tests", "distroless_validation", "static_analysis"] - # ---- wsl --------------------------------------------------------------- [images.wsl] @@ -162,67 +123,3 @@ definition = { type = "kiwi", path = "vm-iso-installer/vm-iso-installer.kiwi", p [images.vm-iso-installer-dev] description = "VM ISO Installer (dev)" definition = { type = "kiwi", path = "vm-iso-installer/vm-iso-installer.kiwi", profile = "vm-iso-installer-dev" } - -# ============================================================================ -# Test suites -# ============================================================================ - -[test-suites.lisa-main] -type = "lisa" -description = "Lisa tests which are run by default" - -[test-suites.lisa-xfs] -type = "lisa" -description = "Lisa xfs tests" - -[test-suites.lisa-perf] -type = "lisa" -description = "Lisa perf tests" - -[test-suites.lisa-smoke] -type = "lisa" -description = "Lisa smoke tests" - -[test-suites.lisa-cvm] -type = "lisa" -description = "Lisa cvm tests" - -[test-suites.lisa-kernel-ltp] -type = "lisa" -description = "Lisa kernel LTP tests" - -# Single shared pytest suite for static (offline) image validation. -[test-suites.static-image-checks] -type = "pytest" -description = "Offline image validation (shared + image-specific tests)" - -[test-suites.static-image-checks.pytest] -working-dir = "tests" -install = "pyproject" -test-paths = ["cases/static/"] -# {capabilities} is substituted by azldev as a comma-separated list of -# the names of capabilities set to `true` for the image (e.g. -# "machine-bootable,systemd,runtime-package-management"). Names use -# the same dash-case form declared above. The pytest plugin parses -# this string into a set; @pytest.mark.require_capability("...") tests -# that don't match are skipped. -extra-args = [ - "--image-path", "{image-path}", - "--image-name", "{image-name}", - "--capabilities", "{capabilities}", -] - -# Runtime container tests — validate live container behavior via podman. -[test-suites.runtime-container-tests] -type = "pytest" -description = "Runtime container validation (exec into live containers)" - -[test-suites.runtime-container-tests.pytest] -working-dir = "tests" -install = "pyproject" -test-paths = ["cases/runtime/"] -extra-args = [ - "--image-path", "{image-path}", - "--image-name", "{image-name}", - "--capabilities", "{capabilities}", -] diff --git a/base/project.toml b/base/project.toml index 1db4bb60f9c..a6abd0ed546 100644 --- a/base/project.toml +++ b/base/project.toml @@ -2,7 +2,7 @@ # Base project # -includes = ["comps/components.toml", "images/images.toml"] +includes = ["comps/components.toml", "images/images.toml", "images/images.tests.toml"] [project] description = 'azurelinux-base' diff --git a/base/tests/framework-labels.toml b/base/tests/framework-labels.toml deleted file mode 100644 index 7059a6a88f2..00000000000 --- a/base/tests/framework-labels.toml +++ /dev/null @@ -1,400 +0,0 @@ -# Azure Linux Framework Labels Catalog -# -# Declares the test labels referenced by: -# - base/images/images.toml ([images..test-workflows] tier maps) -# - base/comps/components.toml ([components.] `test_labels`) -# -# A label is a named, reusable test-selection record. Each label maps to -# a framework-native filter (`fmf_filter` / `lisa_criteria` / -# `openqa_suites` / `pytest_markers`) or is `pending = true` and emits -# no filter. -# -# Execution policy lives on the label, not on the gate: -# retry_count — integer; retries-on-failure for this label's runner -# timeout_min — integer; per-label wall-clock ceiling in minutes -# -# Capability tokens used by `requires_capabilities` here are produced by -# image definitions in base/images/images.toml (one bool per capability -# under each `[images..capabilities]` table). -# -# Ownership -# --------- -# - This file is owned by the Azure Linux team and lives in this repo -# so test selection is reviewable here. -# - azldev consumes this file as a generic resolver — it does NOT -# hard-code label semantics in Go. -# - The resolver dispatches on which filter field is present, not on -# the label name. Prefixes (tmt_, lisa_, ...) are descriptive only. - -[metadata] -version = "1.0.0" -schema_version = 1 -author = "Azure Linux Team" -description = "Azure Linux framework label catalog" - -# ============================================================================= -# Framework enablement -# -# Distro-wide kill switch for label dispatch. A label whose framework is -# disabled here is skipped (with a clear reason in --format explain) and -# its filter is not emitted. -# -# Defaults to enabled if a framework is not listed. -# ============================================================================= - -[frameworks] -tmt = { enabled = false } -lisa = { enabled = true } -openqa = { enabled = false } -pytest = { enabled = true } -image = { enabled = true } - -# ============================================================================= -# TMT labels (Test Management Tool / fmf filter syntax) -# ============================================================================= - -[tmt_labels.tmt_tier0] -description = "TMT smoke tier — fastest, most critical" -type = "tmt" -fmf_filter = "tier:0" -estimated_minutes = 15 -retry_count = 2 -timeout_min = 30 - -[tmt_labels.tmt_tier1] -description = "TMT integration tier — medium coverage" -type = "tmt" -fmf_filter = "tier:1" -estimated_minutes = 45 -retry_count = 1 -timeout_min = 90 - -[tmt_labels.tmt_tier2] -description = "TMT full tier — comprehensive" -type = "tmt" -fmf_filter = "tier:2" -estimated_minutes = 120 -retry_count = 1 -timeout_min = 240 - - -# ============================================================================= -# LISA labels (Linux Integration Services Automation) -# ============================================================================= - -[lisa_labels.lisa_priority0] -description = "LISA PR-gate priority — critical path" -type = "lisa" -lisa_criteria = { priority = [0] } -estimated_minutes = 20 -retry_count = 2 -timeout_min = 40 - -[lisa_labels.lisa_smoke] -description = "LISA smoke test — fastest PR gate" -type = "lisa" -lisa_criteria = { testcaseNames = ["smoke_test"] } -estimated_minutes = 10 -retry_count = 2 -timeout_min = 20 - -[lisa_labels.lisa_priority1] -description = "LISA important functional tests" -type = "lisa" -lisa_criteria = { priority = [1] } -estimated_minutes = 60 -retry_count = 1 -timeout_min = 120 - -[lisa_labels.lisa_priority2] -description = "LISA extended coverage" -type = "lisa" -lisa_criteria = { priority = [2] } -estimated_minutes = 180 -retry_count = 1 -timeout_min = 360 - -[lisa_labels.lisa_ltp] -description = "LISA kernel LTP (Linux Test Project) suite" -type = "lisa" -lisa_criteria = { testcaseNames = ["verify_ltp_lite"] } -estimated_minutes = 90 -retry_count = 1 -timeout_min = 180 - -[lisa_labels.lisa_xfs] -description = "LISA XFS data-disk tests (standard + NVMe)" -type = "lisa" -lisa_criteria = { testcaseNames = ["verify_xfs_standard_datadisk", "verify_xfs_nvme_datadisk"] } -requires_capabilities = ["storage"] -estimated_minutes = 60 -retry_count = 1 -timeout_min = 120 - -[lisa_labels.lisa_perf] -description = "LISA performance test areas (network, nested, nvme, perf_tool, storage)" -type = "lisa" -lisa_criteria = { criteria = [ - { area = "network", category = "performance" }, - { area = "nested", category = "performance" }, - { area = "nvme", category = "performance" }, - { area = "perf_tool", category = "performance" }, - { area = "storage", category = "performance" }, -] } -estimated_minutes = 240 -retry_count = 1 -timeout_min = 480 - - -# ============================================================================= -# OpenQA labels (suite-list selection) -# ============================================================================= - -[openqa_labels.openqa_install_smoke] -description = "OpenQA basic installation validation" -type = "openqa" -openqa_suites = ["install_default", "install_package_set_minimal"] -estimated_minutes = 30 -retry_count = 2 -timeout_min = 60 - -[openqa_labels.openqa_install_comprehensive] -description = "OpenQA full installation matrix" -type = "openqa" -openqa_suites = [ - "install_default", - "install_delete_pata", - "install_sata", - "install_multi", - "install_software_raid", - "install_lvmthin", -] -estimated_minutes = 120 -retry_count = 1 -timeout_min = 240 - - -# ============================================================================= -# pytest labels (in-tree, filesystem-based image tests) -# -# Pytest labels target image tests that live under base/images/tests/cases/ -# and are dispatched via `pytest`. Each label declares: -# - `pytest_markers` : -m expressions to select tests -# - `pytest_files` : optional path globs to limit collection -# - `pytest_args` : optional extra args passed verbatim -# -# The resolver groups all pytest labels in a workflow into a single -# combined invocation (union of markers/files, ordered args) so the -# runner can fire one pytest process per workflow. -# ============================================================================= - -[pytest_labels.pytest_container_static] -description = "Static container validation (image-tree pytest)" -type = "pytest" -pytest_markers = ["static_container_test"] -pytest_files = ["cases/**/*.py"] -requires_capabilities = ["container"] -estimated_minutes = 25 -retry_count = 2 -timeout_min = 50 - -[pytest_labels.pytest_container_runtime] -description = "Runtime container behavior (image-tree pytest)" -type = "pytest" -pytest_markers = ["runtime_container_tests"] -pytest_files = ["cases/**/*.py"] -requires_capabilities = ["container"] -estimated_minutes = 45 -retry_count = 1 -timeout_min = 90 - -[pytest_labels.pytest_vm_basic] -description = "In-VM pytest smoke tests" -type = "pytest" -pytest_markers = ["vm_basic"] -pytest_files = ["cases/**/*.py"] -requires_capabilities = ["machine-bootable"] -estimated_minutes = 30 -retry_count = 2 -timeout_min = 60 - -[pytest_labels.pytest_container_release] -description = "Release-gate container pytest with coverage collection" -type = "pytest" -pytest_markers = ["static_container_test"] -pytest_files = ["cases/**/*.py"] -pytest_args = [ - "--strict-markers", - "--durations=20", - "--cov=base/images/tests/cases", - "--cov-report=term-missing", -] -requires_capabilities = ["container"] -estimated_minutes = 60 -retry_count = 1 -timeout_min = 120 - - - -# Each label is `pending = true` until a concrete framework binding (pytest -# marker, lisa criteria, etc.) is provided. -[image_labels] -vm_smoke = { description = "Lightweight VM smoke validation", requires_capabilities = ["machine-bootable"], estimated_minutes = 20, pending = true } -vm_basic_tests = { description = "Basic VM functional coverage", requires_capabilities = ["machine-bootable"], estimated_minutes = 40, pending = true } -container_smoke = { description = "Lightweight container smoke validation", requires_capabilities = ["container"], estimated_minutes = 15, pending = true } -base_container_validation = { description = "Base-image container validation", requires_capabilities = ["container"], estimated_minutes = 25, pending = true } -distroless_validation = { description = "Distroless / minimal image validation", requires_capabilities = ["container"], estimated_minutes = 30, pending = true } -minimal_surface_tests = { description = "Validation of minimal image surface", estimated_minutes = 20, pending = true } -oci_compliance = { description = "OCI specification compliance", requires_capabilities = ["container"], estimated_minutes = 25, pending = true } -oci_security = { description = "OCI security validation", requires_capabilities = ["container"], estimated_minutes = 30, pending = true } -registry_integration = { description = "Container registry integration", requires_capabilities = ["container"], estimated_minutes = 20, pending = true } -container_security_scan = { description = "Container security scanning", requires_capabilities = ["container"], estimated_minutes = 40, pending = true } - -# ============================================================================= -# Component dynamic labels -# -# Referenced from `test_labels = [...]` on individual -# `[components.]` entries in base/comps/components.toml. All are -# currently `pending = true` — the references are aspirational metadata. -# Concrete filters should be added incrementally as each component's -# dynamic test infrastructure lands. -# ============================================================================= - -[component_labels] -# Kernel & boot -kmod_load = { description = "Kernel module load/unload", pending = true } -kernel_integration = { description = "Kernel runtime integration", pending = true } -boot_validation = { description = "Boot path validation", pending = true } -boot_integration = { description = "Boot process integration", pending = true } -syscall_tests = { description = "Syscall surface validation", pending = true } -uefi_compatibility = { description = "UEFI compatibility", pending = true } -secure_boot_validation = { description = "Secure boot validation", pending = true } -firmware_loading = { description = "Firmware loading", pending = true } -hardware_compatibility = { description = "Hardware compatibility", pending = true } -microcode_update = { description = "Microcode update flow", pending = true } - -# System services -systemd_lifecycle = { description = "systemd lifecycle", pending = true } -systemd_integration = { description = "systemd integration", pending = true } -service_management = { description = "Service management", pending = true } -service_lifecycle = { description = "Service lifecycle", pending = true } -journal_validation = { description = "journald validation", pending = true } -network_service_integration = { description = "Network service integration", pending = true } -connectivity_validation = { description = "Connectivity validation", pending = true } -selinux_enforcement = { description = "SELinux enforcement", pending = true } -tpm_functionality = { description = "TPM functionality", pending = true } -crypto_operations = { description = "Crypto operations", pending = true } -security_integration = { description = "Security stack integration", pending = true } -storage_integration = { description = "Storage integration", pending = true } -filesystem_operations = { description = "Filesystem operations", pending = true } -device_management = { description = "Device management", pending = true } - -# Toolchains & languages -compiler_runtime_tests = { description = "Compiler runtime tests", pending = true } -cross_compile_validation = { description = "Cross-compilation validation", pending = true } -glibc_functionality = { description = "glibc functionality", pending = true } -python_runtime_tests = { description = "Python runtime tests", pending = true } -python_module_integration = { description = "Python module integration", pending = true } -python_package_functionality = { description = "Python package functionality", pending = true } -rust_runtime_tests = { description = "Rust runtime tests", pending = true } -rust_integration = { description = "Rust integration", pending = true } -rust_performance = { description = "Rust performance", pending = true } -cargo_test = { description = "Cargo test execution", pending = true } -go_test = { description = "Go test execution", pending = true } -go_integration = { description = "Go integration", pending = true } -go_performance = { description = "Go performance", pending = true } -go_race_detection = { description = "Go race detection", pending = true } -nodejs_runtime_tests = { description = "Node.js runtime tests", pending = true } -nodejs_performance = { description = "Node.js performance", pending = true } -npm_integration = { description = "npm integration", pending = true } -java_runtime_tests = { description = "Java runtime tests", pending = true } -java_application_tests = { description = "Java application tests", pending = true } -maven_integration = { description = "Maven integration", pending = true } - -# Container & virtualization -container_lifecycle = { description = "Container lifecycle", pending = true } -container_networking = { description = "Container networking", pending = true } -container_security = { description = "Container security", pending = true } -oci_runtime_tests = { description = "OCI runtime tests", pending = true } -vm_lifecycle = { description = "VM lifecycle", pending = true } -vm_migration = { description = "VM migration", pending = true } -libvirt_integration = { description = "libvirt integration", pending = true } -hypervisor_performance = { description = "Hypervisor performance", pending = true } - -# Databases & data -mariadb_functionality = { description = "MariaDB functionality", pending = true } -mariadb_replication = { description = "MariaDB replication", pending = true } -mysql_functionality = { description = "MySQL functionality", pending = true } -mysql_clustering = { description = "MySQL clustering", pending = true } -mysql_performance = { description = "MySQL performance", pending = true } -postgresql_integration = { description = "PostgreSQL integration", pending = true } -postgresql_extension_tests = { description = "PostgreSQL extension tests", pending = true } -database_performance = { description = "Database performance", pending = true } - -# Networking & protocols -network_integration = { description = "Network integration", pending = true } -network_performance = { description = "Network performance", pending = true } -network_protocol_tests = { description = "Network protocol tests", pending = true } -network_tool_functionality = { description = "Network tool functionality", pending = true } -protocol_tests = { description = "Protocol tests", pending = true } -http_load_tests = { description = "HTTP load tests", pending = true } -web_server_functionality = { description = "Web server functionality", pending = true } -web_performance = { description = "Web performance", pending = true } - -# Security & auth -auth_integration = { description = "Authentication integration", pending = true } -kerberos_functionality = { description = "Kerberos functionality", pending = true } -ldap_operations = { description = "LDAP operations", pending = true } -crypto_functionality = { description = "Crypto functionality", pending = true } -encryption_tests = { description = "Encryption tests", pending = true } -compliance_validation = { description = "Compliance validation", pending = true } -security_monitoring = { description = "Security monitoring", pending = true } - -# Filesystem & storage -fs_integrity_tests = { description = "Filesystem integrity tests", pending = true } -fs_performance = { description = "Filesystem performance", pending = true } -archive_operations = { description = "Archive operations", pending = true } -compression_performance = { description = "Compression performance", pending = true } -backup_functionality = { description = "Backup functionality", pending = true } - -# Build & dev tools -build_system_tests = { description = "Build system tests", pending = true } -build_performance = { description = "Build performance", pending = true } -cross_platform_build = { description = "Cross-platform build", pending = true } -debugger_functionality = { description = "Debugger functionality", pending = true } -profiler_tests = { description = "Profiler tests", pending = true } -symbol_resolution = { description = "Symbol resolution", pending = true } -library_integration = { description = "Library integration", pending = true } -library_runtime_tests = { description = "Library runtime tests", pending = true } -package_operations = { description = "Package operations", pending = true } -package_dependency_resolution = { description = "Package dependency resolution", pending = true } -repository_management = { description = "Repository management", pending = true } -vcs_operations = { description = "VCS operations", pending = true } -vcs_integration = { description = "VCS integration", pending = true } -vcs_performance = { description = "VCS performance", pending = true } -test_execution = { description = "Test execution", pending = true } -test_framework_integration = { description = "Test framework integration", pending = true } -test_reporting = { description = "Test reporting", pending = true } -monitoring_functionality = { description = "Monitoring functionality", pending = true } -log_processing = { description = "Log processing", pending = true } -performance_tuning = { description = "Performance tuning", pending = true } - -# Desktop / GUI / multimedia -gui_interaction = { description = "GUI interaction", pending = true } -gui_rendering = { description = "GUI rendering", pending = true } -gui_performance = { description = "GUI performance", pending = true } -graphics_rendering = { description = "Graphics rendering", pending = true } -graphics_performance = { description = "Graphics performance", pending = true } -font_rendering = { description = "Font rendering", pending = true } -multimedia_playback = { description = "Multimedia playback", pending = true } -multimedia_processing = { description = "Multimedia processing", pending = true } -multimedia_performance = { description = "Multimedia performance", pending = true } -audio_processing = { description = "Audio processing", pending = true } -text_editor_functionality = { description = "Text editor functionality", pending = true } -text_processing = { description = "Text processing", pending = true } -text_processing_functionality = { description = "Text processing functionality", pending = true } -file_manager_tests = { description = "File manager tests", pending = true } -spell_check_tests = { description = "Spell check tests", pending = true } -productivity_integration = { description = "Productivity integration", pending = true } -language_processing = { description = "Language processing", pending = true } -internationalization_tests = { description = "Internationalization tests", pending = true } From e3f1da56a8b329dff8f169614fc98b3dedead541 Mon Sep 17 00:00:00 2001 From: Bhagyashri Date: Fri, 5 Jun 2026 17:09:53 +0530 Subject: [PATCH 3/3] Fix copilot suggestions --- base/comps/components.toml | 2 +- .../{kernel-tests.toml => kernel.tests.toml} | 0 base/images/images.tests.toml | 17 +++++++++-------- 3 files changed, 10 insertions(+), 9 deletions(-) rename base/comps/{kernel-tests.toml => kernel.tests.toml} (100%) diff --git a/base/comps/components.toml b/base/comps/components.toml index bef1e8fdf52..9693790a118 100644 --- a/base/comps/components.toml +++ b/base/comps/components.toml @@ -1,4 +1,4 @@ -includes = ["**/*.comp.toml", "**/*.tests.toml", "container-runtime.tests.toml", "kernel-tests.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"] +includes = ["**/*.comp.toml", "**/*.tests.toml", "component-check-disablement.toml", "component-mingw-disablement.toml", "components-publish-channels.toml"] # # Components imported from Fedora with no per-component modifications. diff --git a/base/comps/kernel-tests.toml b/base/comps/kernel.tests.toml similarity index 100% rename from base/comps/kernel-tests.toml rename to base/comps/kernel.tests.toml diff --git a/base/images/images.tests.toml b/base/images/images.tests.toml index 71e922a2e76..87dee6b4fce 100644 --- a/base/images/images.tests.toml +++ b/base/images/images.tests.toml @@ -23,7 +23,7 @@ # # Declarative metadata on [tests.X]: # description : human-readable summary -# type : "pytest" | "tmt" | "lisa" | "openqa" | "pending" +# type : "pytest" | "tmt" | "lisa" # kind : ["functional"] | ["performance"] | both # long-running : boolean; omit unless the test may run for hours # required-capabilities : capability tokens the image must declare @@ -78,10 +78,11 @@ extra-args = [ [tests.smoke_test] type = "lisa" -description = "LISA smoke test — fastest PR gate" +description = "LISA smoke test — fastest PR gate (pinned)" kind = ["functional"] required-capabilities = ["machine-bootable"] [tests.smoke_test.lisa] +source = { git-url = "https://github.com/microsoft/lisa.git", ref = "20260330.1" } criteria = { testcaseNames = ["smoke_test"] } [tests.functional_core] @@ -141,11 +142,11 @@ criteria = [ [test-groups.vm-base-functional] description = "Functional coverage for VM base images" tests = [ - "smoke_test", - "functional_core", - "functional_extended", - "kernel_ltp", - "xfs_filesystem", + { name = "smoke_test" }, + { name = "functional_core" }, + { name = "functional_extended" }, + { name = "kernel_ltp" }, + { name = "xfs_filesystem" }, ] # Performance-oriented coverage. Kept as a separate group because @@ -154,5 +155,5 @@ tests = [ [test-groups.vm-base-performance] description = "Performance coverage for VM base images" tests = [ - "lisa_perf", + { name = "lisa_perf" }, ]