From b788ac14613a2f437c752a3f81107ca298f022b0 Mon Sep 17 00:00:00 2001 From: KPD Date: Mon, 2 Jun 2025 09:03:01 +0000 Subject: [PATCH 1/2] adding ci files --- .github/actions/archive/action.yml | 24 ++++ .github/actions/cleanup/action.yml | 24 ++++ .github/actions/setup/action.yml | 181 +++++++++++++++++++++++++++++ .github/actions/test/action.yml | 44 +++++++ .github/workflows/main.yml | 107 +++++++++++++++++ .github/workflows/manual.yml | 58 +++++++++ .github/workflows/schedule.yml | 50 ++++++++ scripts/github_output.sh | 9 ++ scripts/korg-releases.py | 92 +++++++++++++++ 9 files changed, 589 insertions(+) create mode 100644 .github/actions/archive/action.yml create mode 100644 .github/actions/cleanup/action.yml create mode 100644 .github/actions/setup/action.yml create mode 100644 .github/actions/test/action.yml create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/manual.yml create mode 100644 .github/workflows/schedule.yml create mode 100755 scripts/github_output.sh create mode 100755 scripts/korg-releases.py diff --git a/.github/actions/archive/action.yml b/.github/actions/archive/action.yml new file mode 100644 index 00000000000000..40a84b864e1ae6 --- /dev/null +++ b/.github/actions/archive/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Archive results +description: Archive kdevops results in https://github.com/linux-kdevops/kdevops-results-archive.git +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' +runs: + using: "composite" + steps: + - name: Get systemd journal files + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make journal-dump + + - name: Build our kdevops archive results + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-archive CI_WORKFLOW="${{ inputs.ci_workflow }}" + diff --git a/.github/actions/cleanup/action.yml b/.github/actions/cleanup/action.yml new file mode 100644 index 00000000000000..18a8656aa165b6 --- /dev/null +++ b/.github/actions/cleanup/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Cleanup kdevops VMs +description: Destroy VMs and cleanup workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + +runs: + using: "composite" + steps: + - name: Run kdevops make destroy + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make destroy + + - name: Remove working-directory + shell: bash + run: | + rm --recursive --force --verbose ${{ inputs.dir }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000000000..37d06d9e0c8d78 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + kernel_tree: + required: false + type: string + default: 'linux' + kernel_ref: + required: false + type: string + default: 'master' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Create workspace directory + shell: bash + run: | + pwd + rm --recursive --force --verbose ${{ inputs.dir }} + mkdir --parent --verbose ${{ inputs.dir }} + find . + + - name: Configure git + shell: bash + run: | + git config --global --add safe.directory '*' + git config --global user.name "kdevops" + git config --global user.email "kdevops@lists.linux.dev" + + - name: Checkout kdevops + working-directory: ${{ inputs.dir }} + shell: bash + run: | + rm --recursive --force --verbose kdevops/ + git clone https://github.com/dkruces/kdevops.git --branch ci-workflow kdevops + + - name: Checkout custom branch with delta on kdevops/linux + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="${{ inputs.kernel_ref }}" + git clone $LINUX_TREE linux + cd linux + git checkout $LINUX_TREE_REF + git log -1 + + - name: Make sure our repo kdevops defconfig exists + id: defconfig + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + if [[ -z "${{ inputs.kdevops_defconfig }}" ]]; then + KDEVOPS_DEFCONFIG=${{ inputs.ci_workflow }} + else + KDEVOPS_DEFCONFIG="${{ inputs.kdevops_defconfig }}" + fi + + if [[ ! -f defconfigs/$KDEVOPS_DEFCONFIG ]]; then + echo "kdevops lacks a defconfig for this repository, expected to find: defconfigs/$KDEVOPS_DEFCONFIG" + exit 1 + fi + + "${{ github.workspace }}/scripts/github_output.sh" KDEVOPS_DEFCONFIG "$KDEVOPS_DEFCONFIG" + + - name: Initialize CI metadata for kdevops-results-archive for linux + id: metadata + working-directory: ${{ inputs.dir }}/kdevops/linux + shell: bash + run: | + set -euxo pipefail + echo "${{ inputs.kernel_tree }}" > ../ci.trigger + echo "testing" > ../ci.subject + echo "${{ inputs.kernel_ref }}" > ../ci.ref + + RELEVANT_GIT_TAG=$(cat ../ci.ref) + RELEVANT_GIT_REF=$(git rev-parse --short=12 $RELEVANT_GIT_TAG) + + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_REF "$RELEVANT_GIT_REF" + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_TAG "$RELEVANT_GIT_TAG" + + # Start out pessimistic + echo "unknown" > ../ci.result + echo "Nothing to write home about." > ../ci.commit_extra + + - name: Run a quick Linux kernel defconfig build test + working-directory: ${{ inputs.dir }}/kdevops/linux + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + shell: bash + run: | + set -euxo pipefail + git reset --hard "$LINUX_GIT_TAG" + make defconfig + make -j$(nproc) + + - name: Run kdevops make defconfig-repo + working-directory: ${{ inputs.dir }}/kdevops + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + LINUX_GIT_REF: ${{ steps.metadata.outputs.LINUX_GIT_REF }} + KDEVOPS_DEFCONFIG: ${{ steps.defconfig.outputs.KDEVOPS_DEFCONFIG }} + shell: bash + run: | + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="$LINUX_GIT_TAG" + + # We make the compromise here to use a relevant git tag for the + # host prefix so that folks can easily tell what exact kernel tree + # is being tested by using the relevant git ref. That is, if you + # pushed a tree with the .github/ directory as the top of the tree, + # that commit will not be used, we'll use the last one as that is + # the relevant git ref we want to annotate a test for. + # + # The compromise here is that we expect no two same identical tests + # on the same self-hosted server. We could extend this with something + # like github.run_id but its not yet clear if we can have kdevops + # hosts with a bundled prefix ID like that ref-runid-testname. Tests + # have been done with the full lenght sha1sum though and we know that + # does work. + KDEVOPS_HOSTS_PREFIX="$LINUX_GIT_REF" + + echo "Going to use defconfig-$KDEVOPS_DEFCONFIG" + + echo "Linux tree: $LINUX_TREE" + echo "Linux trigger ref: $LINUX_TREE_REF" + echo "Linux tag: $LINUX_GIT_TAG" + echo "Runner ID: ${{ github.run_id }}" + echo "kdevops host prefix: $KDEVOPS_HOSTS_PREFIX" + echo "kdevops defconfig: defconfig-$KDEVOPS_DEFCONFIG" + + KDEVOPS_ARGS="\ + KDEVOPS_HOSTS_PREFIX=$KDEVOPS_HOSTS_PREFIX \ + LINUX_TREE=$LINUX_TREE \ + LINUX_TREE_REF=$LINUX_TREE_REF \ + ANSIBLE_CFG_CALLBACK_PLUGIN="debug" \ + defconfig-$KDEVOPS_DEFCONFIG" + echo "Going to run:" + echo "make $KDEVOPS_ARGS" + + make $KDEVOPS_ARGS + + - name: Run kdevops make + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make -j$(nproc) + + - name: Run kdevops make bringup + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + ls -ld linux + make destroy + make bringup + + - name: Build linux and boot test nodes on test kernel + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make linux + + - name: Build required ci tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-build-test CI_WORKFLOW=${{ inputs.ci_workflow }} diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 00000000000000..75b95c3721c2a5 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Run CI tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-test CI_WORKFLOW="${{ inputs.ci_workflow }}" + + echo -e "Kernel tests results:\n" > ci.commit_extra + + - name: Generate CI commit info + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + find workflows/blktests/results/last-run/ -name '*.dmesg.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + + echo -e "\n\n" >> ci.commit_extra + echo -e "Userspace test results:\n" >> ci.commit_extra + find workflows/blktests/results/last-run/ -name '*.userspace.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + echo -e "\n\n" >> ci.commit_extra + + if grep -i -q "fail" ci.commit_extra ; then + echo "fail" > ci.result + else + echo "ok" > ci.result + fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000000..13dba4eb512f0f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Reusable + +on: + workflow_call: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: string + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: string + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + check_ref: + name: Check Linux kernel Git Reference + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Check kernel_ref exists + id: check_kernel_ref + run: | + set -euxo pipefail + + ref="${{ github.event.inputs.kernel_ref }}" + tree="${{ github.event.inputs.kernel_tree }}" + mirror="/mirror/${tree}.git" + ls_remote="$(git ls-remote "$mirror" "refs/*/${ref}" || true)" + contains_branch="$(git -C "$mirror" branch --contains "${ref}" || true)" + contains_tag="$(git -C "$mirror" branch --contains "${ref}" || true)" + + if [ -z "$ls_remote" ] && [ -z "$contains_branch" ] && [ -z "$contains_tag" ]; then + echo "Linux kernel ${ref} does not exist." + exit 1 + fi + + setup: + name: Setup kdevops workspace + runs-on: [self-hosted] + needs: [check_ref] + steps: + - name: Checkout kdevops-ci + uses: actions/checkout@v4 + + - name: kdevops setup + uses: ./.github/actions/setup + with: + dir: ${{ inputs.ci_workflow }} + kernel_tree: ${{ inputs.kernel_tree }} + kernel_ref: ${{ inputs.kernel_ref }} + ci_workflow: ${{ inputs.ci_workflow }} + + test: + name: Run kdevops ci-test + runs-on: [self-hosted] + needs: [setup] + steps: + - name: kdevops ci-test + uses: ./.github/actions/test + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} + + archive: + name: Archive kdevops + runs-on: [self-hosted] + needs: [setup, test] + steps: + - name: Start SSH Agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Archive ci-test results + uses: ./.github/actions/archive + with: + dir: ${{ inputs.ci_workflow }} + + - name: Upload our kdevops results archive + uses: actions/upload-artifact@v4 + with: + name: kdevops-ci-results + path: ${{ inputs.ci_workflow }}/kdevops/archive/*.zip + + cleanup: + name: Cleanup kdevops workspace + runs-on: [self-hosted] + needs: [setup, test, archive] + if: always() + steps: + - name: kdevops cleanup + uses: ./.github/actions/cleanup + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml new file mode 100644 index 00000000000000..7b570265c45685 --- /dev/null +++ b/.github/workflows/manual.yml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Manual + +on: + workflow_dispatch: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: choice + options: + - blktests + - blktests_block + - blktests_loop + - blktests_meta + - blktests_nbd + - blktests_nvme + - blktests_nvmemp + - blktests_scsi + - blktests_srp + - blktests_zbd + - tmpfs + - tmpfs_default + - tmpfs_huge + - tmpfs_noswap + - linux-btrfs-kpd + - linux-ext4-kpd + - linux-firmware-kpd + - linux-mm-kpd + - linux-modules-kpd + - linux-xfs-kpd + - selftests + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: choice + options: + - linux + - linux-next + - linux-stable + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + manual: + name: Manual kdevops CI + uses: ./.github/workflows/main.yml + secrets: inherit + with: + ci_workflow: ${{ inputs.ci_workflow }} + kernel_ref: ${{ inputs.kernel_ref }} + kernel_tree: ${{ inputs.kernel_tree }} diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml new file mode 100644 index 00000000000000..d0221550845fa6 --- /dev/null +++ b/.github/workflows/schedule.yml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Schedule + +on: + schedule: + - cron: '0 14 * * *' + + workflow_dispatch: + +jobs: + check_ref: + name: Check Linux kernel Git Reference + outputs: + kernel_ref: ${{ steps.check_kernel_ref.outputs.kernel_ref }} + kernel_tree: ${{ steps.check_kernel_ref.outputs.kernel_tree }} + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Generate kernel_ref + id: check_kernel_ref + run: | + set -euxo pipefail + day_of_week=$(date +%u) + + kernel_ref=$(./scripts/korg-releases.py --moniker linux-next) + kernel_tree=linux-next + if [ "$day_of_week" -eq 1 ]; then + kernel_ref=$(./scripts/korg-releases.py --moniker mainline) + kernel_tree=linux + fi + + "${{ github.workspace }}/scripts/github_output.sh" kernel_ref "$kernel_ref" + "${{ github.workspace }}/scripts/github_output.sh" kernel_tree "$kernel_tree" + + schedule: + name: Scheduled kdevops CI + needs: [check_ref] + runs-on: [self-hosted] + secrets: inherit + uses: ./.github/workflows/main.yml + strategy: + matrix: + ci_workflow: [blktests-block, blktests-loop, blktests-meta] + with: + ci_workflow: ${{ matrix.ci_workflow }} + kernel_ref: ${{ needs.check_ref.outputs.kernel_ref }} + kernel_tree: ${{ needs.check_ref.outputs.kernel_tree }} diff --git a/scripts/github_output.sh b/scripts/github_output.sh new file mode 100755 index 00000000000000..711306c38cb5ff --- /dev/null +++ b/scripts/github_output.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Usage: ./github_output.sh key value +set -euxo pipefail + +key="$1" +value="$2" + +echo "$key=$value" >> "$GITHUB_OUTPUT" diff --git a/scripts/korg-releases.py b/scripts/korg-releases.py new file mode 100755 index 00000000000000..bce182e28d7926 --- /dev/null +++ b/scripts/korg-releases.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: copyleft-next-0.3.1 +import sys +import logging +import argparse +import json +import urllib.request +import socket +import re + + +def parser(): + parser = argparse.ArgumentParser(description="kernel.org/releases.json checker") + parser.add_argument("--debug", action="store_true", help="debug") + parser.add_argument( + "--moniker", + help="moniker (mainline, stable, longterm or linux-next)", + required=True, + ) + parser.add_argument( + "--pname", + help="project name for User-Agent request", + default="kdevops", + ) + parser.add_argument( + "--pversion", + help="project version for User-Agent request", + default="5.0.2", + ) + return parser + + +def _check_connection(host, port, timeout=2): + try: + with socket.create_connection((host, port), timeout): + logging.debug(f"Connection to {host} on port {port} succeeded!") + return True + except (socket.timeout, socket.error) as e: + logging.debug(f"Connection to {host} on port {port} failed: {e}") + return False + + +def kreleases(args) -> None: + """Get the latest kernel releases from kernel.org/releases.json""" + + reflist = [] + if _check_connection("kernel.org", 80): + _url = "https://www.kernel.org/releases.json" + req = urllib.request.Request( + _url, + headers={ + "User-Agent": f"{args.pname}/{args.pversion} (kdevops@lists.linux.dev)" + }, + ) + with urllib.request.urlopen(req) as url: + data = json.load(url) + + for release in data["releases"]: + if release["moniker"] == args.moniker: + # Check if release.json is aa.bb.cc type + if re.compile(r'^\d+\.\d+(\.\d+|-rc\d+)?$').match(release["version"]): + reflist.append("v" + release["version"]) + else: + reflist.append(release["version"]) + + logging.debug(f"{reflist}") + for r in reflist: + print(r) + + +def main() -> None: + """Kconfig choice generator for git refereces""" + log = logging.getLogger() + log.setLevel(logging.INFO) + p = parser() + args, _ = p.parse_known_args() + if args.debug: + log.setLevel(logging.DEBUG) + + kreleases(args) + + +if __name__ == "__main__": + ret = 0 + try: + main() + except Exception: + ret = 1 + import traceback + + traceback.print_exc() + sys.exit(ret) From b78d15529701081b272f8de6e36f30eb588856d4 Mon Sep 17 00:00:00 2001 From: Petr Pavlu Date: Thu, 13 Mar 2025 15:29:20 +0100 Subject: [PATCH 2/2] codetag: Avoid unused alloc_tags sections/symbols With CONFIG_MEM_ALLOC_PROFILING=n, vmlinux and all modules unnecessarily contain the symbols __start_alloc_tags and __stop_alloc_tags, which define an empty range. In the case of modules, the presence of these symbols also forces the linker to create an empty .codetag.alloc_tags section. Update codetag.lds.h to make the data conditional on CONFIG_MEM_ALLOC_PROFILING. Signed-off-by: Petr Pavlu Reviewed-by: Kent Overstreet Reviewed-by: Suren Baghdasaryan --- include/asm-generic/codetag.lds.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/asm-generic/codetag.lds.h b/include/asm-generic/codetag.lds.h index 372c320c504378..0ea1fa67840511 100644 --- a/include/asm-generic/codetag.lds.h +++ b/include/asm-generic/codetag.lds.h @@ -2,6 +2,12 @@ #ifndef __ASM_GENERIC_CODETAG_LDS_H #define __ASM_GENERIC_CODETAG_LDS_H +#ifdef CONFIG_MEM_ALLOC_PROFILING +#define IF_MEM_ALLOC_PROFILING(...) __VA_ARGS__ +#else +#define IF_MEM_ALLOC_PROFILING(...) +#endif + #define SECTION_WITH_BOUNDARIES(_name) \ . = ALIGN(8); \ __start_##_name = .; \ @@ -9,7 +15,7 @@ __stop_##_name = .; #define CODETAG_SECTIONS() \ - SECTION_WITH_BOUNDARIES(alloc_tags) + IF_MEM_ALLOC_PROFILING(SECTION_WITH_BOUNDARIES(alloc_tags)) /* * Module codetags which aren't used after module unload, therefore have the @@ -28,6 +34,6 @@ * unload them individually once unused. */ #define MOD_SEPARATE_CODETAG_SECTIONS() \ - MOD_SEPARATE_CODETAG_SECTION(alloc_tags) + IF_MEM_ALLOC_PROFILING(MOD_SEPARATE_CODETAG_SECTION(alloc_tags)) #endif /* __ASM_GENERIC_CODETAG_LDS_H */