From 94c328d6601cddc1d81169c1b6e9a2356629c12e Mon Sep 17 00:00:00 2001 From: Daniel Gomez Date: Fri, 26 Sep 2025 13:27:16 +0200 Subject: [PATCH 1/2] github: unify docker-tests and linux-ab This consolidates docker-tests.yml and linux-ab.yml into a single compatibility.yml workflow that builds kdevops containers once and reuses them across all test jobs via GitHub Container Registry. The unified approach eliminates duplicate container builds while providing comprehensive validation across Debian, Fedora, and OpenSUSE distributions. The A/B testing script is modified to work in GitHub containers by generating only the configuration files needed for validation (extra_vars.yaml) instead of running the full make target that requires systemd services unavailable in container environments. Generated-by: Claude AI Signed-off-by: Daniel Gomez --- .github/workflows/compatibility.yml | 253 ++++++++++++++++++++++++++++ .github/workflows/docker-tests.yml | 55 ------ .github/workflows/linux-ab.yml | 47 ------ scripts/test-linux-ab.sh | 17 +- 4 files changed, 265 insertions(+), 107 deletions(-) create mode 100644 .github/workflows/compatibility.yml delete mode 100644 .github/workflows/docker-tests.yml delete mode 100644 .github/workflows/linux-ab.yml diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.yml new file mode 100644 index 000000000..3c71f41a8 --- /dev/null +++ b/.github/workflows/compatibility.yml @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Fast Compatibility & Config Tests + +on: + push: + branches: + - '**' + pull_request: + branches: + - '**' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/kdevops-ci + +jobs: + build-kdevops-containers: + name: Build kdevops Container (${{ matrix.base_image_name }}) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + outputs: + debian-image: ${{ steps.set-outputs-debian.outputs.debian-image }} + fedora-image: ${{ steps.set-outputs-fedora.outputs.fedora-image }} + opensuse-image: ${{ steps.set-outputs-opensuse.outputs.opensuse-image }} + strategy: + fail-fast: false + matrix: + include: + - base_image: debian:testing + base_image_name: debian + install_cmd: > + apt update && apt-get install -y + ansible-core + bash + bison + coreutils + flex + gawk + gcc + git + hostname + libvirt-clients + libvirt0 + make + ncurses-dev + netcat-openbsd + pkg-config + python3 + sudo + traceroute + which + - base_image: fedora:latest + base_image_name: fedora + install_cmd: > + dnf install -y + ansible + bash + bison + coreutils + flex + gawk + gcc + git + hostname + libvirt-client + make + ncurses-devel + nmap-ncat + pkgconf + python3 + sudo + traceroute + which + - base_image: opensuse/tumbleweed + base_image_name: opensuse + install_cmd: > + zypper refresh && zypper install -y + ansible + bash + bison + coreutils + flex + gawk + gcc + git + hostname + libvirt + make + ncurses-devel + netcat-openbsd + pkg-config + python3 + sudo + traceroute + which + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create Dockerfile for ${{ matrix.base_image_name }} + run: | + cat > Dockerfile.${{ matrix.base_image_name }} << 'EOF' + FROM ${{ matrix.base_image }} + RUN ${{ matrix.install_cmd }} + RUN git config --global --add safe.directory '*' && \ + git config --global user.name "kdevops-ci" && \ + git config --global user.email "kdevops@lists.linux.dev" + RUN mkdir -p /github/home/.ssh && chmod 700 /github/home/.ssh + ENV USER=kdevops-ci + ENV HOME=/github/home + WORKDIR /workspace + EOF + + - name: Build and test ${{ matrix.base_image_name }} container + run: | + IMAGE_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.base_image_name }}:${{ github.sha }}" + + # Build container + docker build -f Dockerfile.${{ matrix.base_image_name }} -t "$IMAGE_TAG" . + + # Test basic functionality + docker run --rm -v "$PWD:/workspace" "$IMAGE_TAG" sh -c " + echo '๐Ÿงช Testing kdevops build in ${{ matrix.base_image_name }} container' + make mrproper + echo 'โœ… Basic kdevops build validation passed' + " + + # Push container + docker push "$IMAGE_TAG" + + - name: Set outputs for downstream jobs (debian only) + id: set-outputs-debian + if: matrix.base_image_name == 'debian' + run: | + ./scripts/github_output.sh debian-image "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-debian:${{ github.sha }}" + + - name: Set outputs for downstream jobs (fedora only) + id: set-outputs-fedora + if: matrix.base_image_name == 'fedora' + run: | + ./scripts/github_output.sh fedora-image "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-fedora:${{ github.sha }}" + + - name: Set outputs for downstream jobs (opensuse only) + id: set-outputs-opensuse + if: matrix.base_image_name == 'opensuse' + run: | + ./scripts/github_output.sh opensuse-image "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-opensuse:${{ github.sha }}" + + linux-ab-config-tests: + name: Linux A/B Tests (${{ matrix.distro_name }}) + runs-on: ubuntu-latest + needs: build-kdevops-containers + strategy: + fail-fast: false + matrix: + include: + - distro_name: debian + container_image: ${{ needs.build-kdevops-containers.outputs.debian-image }} + - distro_name: fedora + container_image: ${{ needs.build-kdevops-containers.outputs.fedora-image }} + - distro_name: opensuse + container_image: ${{ needs.build-kdevops-containers.outputs.opensuse-image }} + container: ${{ matrix.container_image }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run Linux A/B testing configuration validation + run: | + echo "๐Ÿงช Running comprehensive A/B configuration tests on ${{ matrix.distro_name }}..." + echo "This validates all 3 build methods without requiring VMs" + make check-linux-ab + + - name: Validate A/B test results and generated files + run: | + echo "โœ… All A/B configuration tests passed on ${{ matrix.distro_name }}!" + echo "๐Ÿ“‹ Validated configurations:" + echo " - Target build method A/B setup" + echo " - 9P build method A/B setup" + echo " - Builder method A/B setup" + echo " - Kernel reference differentiation" + echo " - Configuration file generation (.config, .extra_vars_auto.yaml, extra_vars.yaml)" + echo " - Cross-distribution compatibility: ${{ matrix.distro_name }}" + + quick-config-validation: + name: Quick Config Tests (${{ matrix.defconfig }}) on ${{ matrix.distro_name }} + runs-on: ubuntu-latest + needs: build-kdevops-containers + strategy: + fail-fast: false + matrix: + defconfig: [blktests_nvme, xfs_reflink_4k, lbs-xfs, linux-ab-testing] + distro_name: [debian, fedora, opensuse] + include: + - distro_name: debian + container_image: ${{ needs.build-kdevops-containers.outputs.debian-image }} + - distro_name: fedora + container_image: ${{ needs.build-kdevops-containers.outputs.fedora-image }} + - distro_name: opensuse + container_image: ${{ needs.build-kdevops-containers.outputs.opensuse-image }} + container: ${{ matrix.container_image }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Test defconfig-${{ matrix.defconfig }} configuration + run: | + echo "๐Ÿ”ง Testing defconfig-${{ matrix.defconfig }} configuration on ${{ matrix.distro_name }}..." + make mrproper + make defconfig-${{ matrix.defconfig }} + echo "๐Ÿ“ Checking files after defconfig..." + test -f .config || (echo "โŒ .config not generated by defconfig" && exit 1) + test -f .extra_vars_auto.yaml || (echo "โŒ .extra_vars_auto.yaml not generated by defconfig" && exit 1) + echo "โœ… defconfig generated .config and .extra_vars_auto.yaml successfully" + + - name: Generate configuration files (container-safe) + run: | + echo "๐Ÿ”จ Generating core configuration files without systemd services..." + # Generate the essential files from DEFAULT_DEPS but skip LOCALHOST_SETUP_WORK (systemd) + # This generates: .kdevops.depcheck, extra_vars.yaml, ansible.cfg, hosts + make .kdevops.depcheck + make extra_vars.yaml + make $PWD/ansible.cfg + make $PWD/hosts + echo "โœ… Core configuration files generated successfully" + + - name: Verify all generated configuration files + run: | + echo "๐Ÿ“ Validating all generated configuration files..." + test -f .config || (echo "โŒ .config not found" && exit 1) + test -f .extra_vars_auto.yaml || (echo "โŒ .extra_vars_auto.yaml not found" && exit 1) + test -f extra_vars.yaml || (echo "โŒ extra_vars.yaml not generated by make" && exit 1) + test -f .kdevops.depcheck || (echo "โŒ .kdevops.depcheck not found" && exit 1) + test -f ansible.cfg || (echo "โŒ ansible.cfg not found" && exit 1) + test -f hosts || (echo "โŒ hosts not found" && exit 1) + echo "โœ… All required configuration files generated:" + echo " - .config (kernel-style configuration)" + echo " - .extra_vars_auto.yaml (auto-generated from defconfig)" + echo " - extra_vars.yaml (final ansible variables from make)" + echo " - .kdevops.depcheck (dependency verification)" + echo " - ansible.cfg (ansible configuration)" + echo " - hosts (inventory file)" diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml deleted file mode 100644 index 33d67f097..000000000 --- a/.github/workflows/docker-tests.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: kdevops Docker Tests - -on: - push: - branches: - - '**' - pull_request: - branches: - - '**' - -jobs: - setup-and-make: - name: Setup and Run Make Targets - runs-on: ubuntu-latest - strategy: - matrix: - distro_container: - - debian:testing - - fedora:latest - - opensuse/tumbleweed - - container: ${{ matrix.distro_container }} - steps: - - name: Document supported kdevops distribution - run: | - echo "Running test on ${{ matrix.distro_container }} container" - uname -a - - - name: Install kdevops dependencies - run: | - # Conditional package installation based on the container - if [ "${{ matrix.distro_container }}" = "debian:testing" ]; then - echo "Installing packages for Debian" - apt-get update - apt-get install -y ansible-core make gcc ncurses-dev bison flex - elif [ "${{ matrix.distro_container }}" = "fedora:latest" ]; then - echo "Installing packages for Fedora" - dnf install -y ansible make gcc ncurses-devel bison flex - elif [ "${{ matrix.distro_container }}" = "opensuse/tumbleweed" ]; then - echo "Installing packages for OpenSUSE" - zypper refresh - zypper install -y ansible make gcc ncurses-devel bison flex - else - echo "Unknown distribution: ${{ matrix.distro_container }}" - exit 1 - fi - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Run make targets - run: | - echo "Running simple make targets on ${{ matrix.distro_container }} environment" - make mrproper - diff --git a/.github/workflows/linux-ab.yml b/.github/workflows/linux-ab.yml deleted file mode 100644 index 9162c8879..000000000 --- a/.github/workflows/linux-ab.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Run kdevops linux-ab tests on self-hosted runner - -on: - push: - branches: - - '**' - pull_request: - branches: - - '**' - workflow_dispatch: # Add this for manual triggering of the workflow - -jobs: - run-kdevops: - name: Run kdevops CI - runs-on: [self-hosted, Linux, X64] - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set CI metadata for kdevops-results-archive - run: | - echo "$(basename ${{ github.repository }})" > ci.trigger - git log -1 --pretty=format:"%s" > ci.subject - # Start out pessimistic - echo "not ok" > ci.result - echo "Nothing to write home about." > ci.commit_extra - - - name: Set kdevops path - run: echo "KDEVOPS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - - name: Configure git - run: | - git config --global --add safe.directory '*' - git config --global user.name "kdevops" - git config --global user.email "kdevops@lists.linux.dev" - - - name: Run kdevops check-linux-ab - run: | - make check-linux-ab - echo "ok" > ci.result - - # Ensure make destroy always runs, even on failure - - name: Run kdevops make destroy - if: always() # This ensures the step runs even if previous steps failed - run: | - make destroy - make mrproper diff --git a/scripts/test-linux-ab.sh b/scripts/test-linux-ab.sh index a13964e43..02f9fb12d 100755 --- a/scripts/test-linux-ab.sh +++ b/scripts/test-linux-ab.sh @@ -1,11 +1,18 @@ #!/bin/bash # SPDX-License-Identifier: copyleft-next-0.3.1 # -# Test A/B configuration locally of all Linux AB configurations posisble. +# Test A/B configuration locally of all Linux AB configurations possible. # The goal is to verify your extra_vars.yaml ends up with different kernel # target refs for A and B group hosts. It does so also by checking that # ansible will use these. No real bringup or live test is done. # +# Note: Originally this script ran full 'make' for each build method, but +# GitHub Actions containers lack systemd and infrastructure dependencies +# that kdevops requires. Since we only need to verify configuration +# generation (not infrastructure setup), we run 'make extra_vars.yaml' +# which generates the ansible variables needed for A/B validation while +# skipping systemd services and other GitHub-incompatible components. +# # Outputs TAP (Test Anything Protocol) format results set -e @@ -115,11 +122,11 @@ for method in $BUILD_METHODS; do continue fi - # Generate configuration - if make >/dev/null 2>&1; then - tap_result "ok" "$method: Generate configuration (make)" + # Generate configuration (container-safe) + if make extra_vars.yaml >/dev/null 2>&1; then + tap_result "ok" "$method: Generate configuration (extra_vars.yaml)" else - tap_result "not ok" "$method: Generate configuration (make)" "Failed to run make" + tap_result "not ok" "$method: Generate configuration (extra_vars.yaml)" "Failed to generate extra_vars.yaml" continue fi From 3d13be48484cf08c0aa563e4afdb0f598914bb6c Mon Sep 17 00:00:00 2001 From: Daniel Gomez Date: Fri, 26 Sep 2025 21:36:14 +0200 Subject: [PATCH 2/2] github: fix self-hosted runner checkout issues The unified workflow had two critical issues with repository checkout on self-hosted runners. For pull requests, it attempted to fetch non-existent merge refs like "35/merge". For push events, it used incomplete HTTPS URLs missing the .git suffix. This fixes both by using proper GitHub PR refs (refs/pull/N/head) and complete HTTPS clone URLs with .git suffix for the public repository. Fixes: 1b740047 ("github: unify self-hosted runner workflows") Generated-by: Claude AI Signed-off-by: Daniel Gomez --- .github/workflows/kdevops.yml | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/kdevops.yml b/.github/workflows/kdevops.yml index ac2ac6251..e278b3065 100644 --- a/.github/workflows/kdevops.yml +++ b/.github/workflows/kdevops.yml @@ -184,23 +184,37 @@ jobs: # Repo exists - clean and update (like actions/checkout) git clean -ffdx git reset --hard HEAD - # Ensure correct remote URL - git remote set-url origin $GITHUB_SERVER_URL/$GITHUB_REPOSITORY - git fetch --depth=1 origin ${{ github.ref_name }} - git reset --hard FETCH_HEAD + # Ensure correct remote URL for public repo + git remote set-url origin ${{ github.server_url }}/${{ github.repository }}.git + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + # For PRs, fetch the specific PR ref that GitHub creates + git fetch --depth=1 origin refs/pull/${{ github.event.number }}/head:pr-branch + git checkout pr-branch + else + # For push/schedule/workflow_dispatch, use the commit SHA directly + git fetch --depth=1 origin ${{ github.sha }} + git checkout ${{ github.sha }} + fi else # No repo - fresh clone with local mirror optimization if [[ -f "/mirror/kdevops.git/HEAD" ]]; then git clone --verbose --progress \ --reference /mirror/kdevops.git \ --depth=1 \ - --branch ${{ github.ref_name }} \ - $GITHUB_SERVER_URL/$GITHUB_REPOSITORY . + ${{ github.server_url }}/${{ github.repository }}.git . else git clone --verbose --progress \ --depth=1 \ - --branch ${{ github.ref_name }} \ - $GITHUB_SERVER_URL/$GITHUB_REPOSITORY . + ${{ github.server_url }}/${{ github.repository }}.git . + fi + + # Checkout the specific commit + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + git fetch --depth=1 origin refs/pull/${{ github.event.number }}/head:pr-branch + git checkout pr-branch + else + git checkout ${{ github.sha }} fi fi