From b788ac14613a2f437c752a3f81107ca298f022b0 Mon Sep 17 00:00:00 2001 From: KPD Date: Mon, 2 Jun 2025 09:03:01 +0000 Subject: [PATCH 1/2] adding ci files --- .github/actions/archive/action.yml | 24 ++++ .github/actions/cleanup/action.yml | 24 ++++ .github/actions/setup/action.yml | 181 +++++++++++++++++++++++++++++ .github/actions/test/action.yml | 44 +++++++ .github/workflows/main.yml | 107 +++++++++++++++++ .github/workflows/manual.yml | 58 +++++++++ .github/workflows/schedule.yml | 50 ++++++++ scripts/github_output.sh | 9 ++ scripts/korg-releases.py | 92 +++++++++++++++ 9 files changed, 589 insertions(+) create mode 100644 .github/actions/archive/action.yml create mode 100644 .github/actions/cleanup/action.yml create mode 100644 .github/actions/setup/action.yml create mode 100644 .github/actions/test/action.yml create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/manual.yml create mode 100644 .github/workflows/schedule.yml create mode 100755 scripts/github_output.sh create mode 100755 scripts/korg-releases.py diff --git a/.github/actions/archive/action.yml b/.github/actions/archive/action.yml new file mode 100644 index 00000000000000..40a84b864e1ae6 --- /dev/null +++ b/.github/actions/archive/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Archive results +description: Archive kdevops results in https://github.com/linux-kdevops/kdevops-results-archive.git +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' +runs: + using: "composite" + steps: + - name: Get systemd journal files + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make journal-dump + + - name: Build our kdevops archive results + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-archive CI_WORKFLOW="${{ inputs.ci_workflow }}" + diff --git a/.github/actions/cleanup/action.yml b/.github/actions/cleanup/action.yml new file mode 100644 index 00000000000000..18a8656aa165b6 --- /dev/null +++ b/.github/actions/cleanup/action.yml @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Cleanup kdevops VMs +description: Destroy VMs and cleanup workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + +runs: + using: "composite" + steps: + - name: Run kdevops make destroy + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make destroy + + - name: Remove working-directory + shell: bash + run: | + rm --recursive --force --verbose ${{ inputs.dir }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000000000..37d06d9e0c8d78 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + kernel_tree: + required: false + type: string + default: 'linux' + kernel_ref: + required: false + type: string + default: 'master' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Create workspace directory + shell: bash + run: | + pwd + rm --recursive --force --verbose ${{ inputs.dir }} + mkdir --parent --verbose ${{ inputs.dir }} + find . + + - name: Configure git + shell: bash + run: | + git config --global --add safe.directory '*' + git config --global user.name "kdevops" + git config --global user.email "kdevops@lists.linux.dev" + + - name: Checkout kdevops + working-directory: ${{ inputs.dir }} + shell: bash + run: | + rm --recursive --force --verbose kdevops/ + git clone https://github.com/dkruces/kdevops.git --branch ci-workflow kdevops + + - name: Checkout custom branch with delta on kdevops/linux + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="${{ inputs.kernel_ref }}" + git clone $LINUX_TREE linux + cd linux + git checkout $LINUX_TREE_REF + git log -1 + + - name: Make sure our repo kdevops defconfig exists + id: defconfig + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + set -euxo pipefail + if [[ -z "${{ inputs.kdevops_defconfig }}" ]]; then + KDEVOPS_DEFCONFIG=${{ inputs.ci_workflow }} + else + KDEVOPS_DEFCONFIG="${{ inputs.kdevops_defconfig }}" + fi + + if [[ ! -f defconfigs/$KDEVOPS_DEFCONFIG ]]; then + echo "kdevops lacks a defconfig for this repository, expected to find: defconfigs/$KDEVOPS_DEFCONFIG" + exit 1 + fi + + "${{ github.workspace }}/scripts/github_output.sh" KDEVOPS_DEFCONFIG "$KDEVOPS_DEFCONFIG" + + - name: Initialize CI metadata for kdevops-results-archive for linux + id: metadata + working-directory: ${{ inputs.dir }}/kdevops/linux + shell: bash + run: | + set -euxo pipefail + echo "${{ inputs.kernel_tree }}" > ../ci.trigger + echo "testing" > ../ci.subject + echo "${{ inputs.kernel_ref }}" > ../ci.ref + + RELEVANT_GIT_TAG=$(cat ../ci.ref) + RELEVANT_GIT_REF=$(git rev-parse --short=12 $RELEVANT_GIT_TAG) + + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_REF "$RELEVANT_GIT_REF" + "${{ github.workspace }}/scripts/github_output.sh" LINUX_GIT_TAG "$RELEVANT_GIT_TAG" + + # Start out pessimistic + echo "unknown" > ../ci.result + echo "Nothing to write home about." > ../ci.commit_extra + + - name: Run a quick Linux kernel defconfig build test + working-directory: ${{ inputs.dir }}/kdevops/linux + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + shell: bash + run: | + set -euxo pipefail + git reset --hard "$LINUX_GIT_TAG" + make defconfig + make -j$(nproc) + + - name: Run kdevops make defconfig-repo + working-directory: ${{ inputs.dir }}/kdevops + env: + LINUX_GIT_TAG: ${{ steps.metadata.outputs.LINUX_GIT_TAG }} + LINUX_GIT_REF: ${{ steps.metadata.outputs.LINUX_GIT_REF }} + KDEVOPS_DEFCONFIG: ${{ steps.defconfig.outputs.KDEVOPS_DEFCONFIG }} + shell: bash + run: | + LINUX_TREE="/mirror/${{ inputs.kernel_tree }}.git" + LINUX_TREE_REF="$LINUX_GIT_TAG" + + # We make the compromise here to use a relevant git tag for the + # host prefix so that folks can easily tell what exact kernel tree + # is being tested by using the relevant git ref. That is, if you + # pushed a tree with the .github/ directory as the top of the tree, + # that commit will not be used, we'll use the last one as that is + # the relevant git ref we want to annotate a test for. + # + # The compromise here is that we expect no two same identical tests + # on the same self-hosted server. We could extend this with something + # like github.run_id but its not yet clear if we can have kdevops + # hosts with a bundled prefix ID like that ref-runid-testname. Tests + # have been done with the full lenght sha1sum though and we know that + # does work. + KDEVOPS_HOSTS_PREFIX="$LINUX_GIT_REF" + + echo "Going to use defconfig-$KDEVOPS_DEFCONFIG" + + echo "Linux tree: $LINUX_TREE" + echo "Linux trigger ref: $LINUX_TREE_REF" + echo "Linux tag: $LINUX_GIT_TAG" + echo "Runner ID: ${{ github.run_id }}" + echo "kdevops host prefix: $KDEVOPS_HOSTS_PREFIX" + echo "kdevops defconfig: defconfig-$KDEVOPS_DEFCONFIG" + + KDEVOPS_ARGS="\ + KDEVOPS_HOSTS_PREFIX=$KDEVOPS_HOSTS_PREFIX \ + LINUX_TREE=$LINUX_TREE \ + LINUX_TREE_REF=$LINUX_TREE_REF \ + ANSIBLE_CFG_CALLBACK_PLUGIN="debug" \ + defconfig-$KDEVOPS_DEFCONFIG" + echo "Going to run:" + echo "make $KDEVOPS_ARGS" + + make $KDEVOPS_ARGS + + - name: Run kdevops make + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make -j$(nproc) + + - name: Run kdevops make bringup + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + ls -ld linux + make destroy + make bringup + + - name: Build linux and boot test nodes on test kernel + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make linux + + - name: Build required ci tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-build-test CI_WORKFLOW=${{ inputs.ci_workflow }} diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 00000000000000..75b95c3721c2a5 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Setup kdevops +description: Setup kdevops workspace + +inputs: + dir: + description: 'Directory' + required: true + default: 'workdir' + ci_workflow: + required: false + type: string + default: 'demo' + +runs: + using: "composite" + steps: + - name: Run CI tests + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + make ci-test CI_WORKFLOW="${{ inputs.ci_workflow }}" + + echo -e "Kernel tests results:\n" > ci.commit_extra + + - name: Generate CI commit info + working-directory: ${{ inputs.dir }}/kdevops + shell: bash + run: | + find workflows/blktests/results/last-run/ -name '*.dmesg.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + + echo -e "\n\n" >> ci.commit_extra + echo -e "Userspace test results:\n" >> ci.commit_extra + find workflows/blktests/results/last-run/ -name '*.userspace.log' \ + -exec tail -n 1 {} + >> ci.commit_extra + echo -e "\n\n" >> ci.commit_extra + + if grep -i -q "fail" ci.commit_extra ; then + echo "fail" > ci.result + else + echo "ok" > ci.result + fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000000..13dba4eb512f0f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Reusable + +on: + workflow_call: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: string + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: string + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + check_ref: + name: Check Linux kernel Git Reference + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Check kernel_ref exists + id: check_kernel_ref + run: | + set -euxo pipefail + + ref="${{ github.event.inputs.kernel_ref }}" + tree="${{ github.event.inputs.kernel_tree }}" + mirror="/mirror/${tree}.git" + ls_remote="$(git ls-remote "$mirror" "refs/*/${ref}" || true)" + contains_branch="$(git -C "$mirror" branch --contains "${ref}" || true)" + contains_tag="$(git -C "$mirror" branch --contains "${ref}" || true)" + + if [ -z "$ls_remote" ] && [ -z "$contains_branch" ] && [ -z "$contains_tag" ]; then + echo "Linux kernel ${ref} does not exist." + exit 1 + fi + + setup: + name: Setup kdevops workspace + runs-on: [self-hosted] + needs: [check_ref] + steps: + - name: Checkout kdevops-ci + uses: actions/checkout@v4 + + - name: kdevops setup + uses: ./.github/actions/setup + with: + dir: ${{ inputs.ci_workflow }} + kernel_tree: ${{ inputs.kernel_tree }} + kernel_ref: ${{ inputs.kernel_ref }} + ci_workflow: ${{ inputs.ci_workflow }} + + test: + name: Run kdevops ci-test + runs-on: [self-hosted] + needs: [setup] + steps: + - name: kdevops ci-test + uses: ./.github/actions/test + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} + + archive: + name: Archive kdevops + runs-on: [self-hosted] + needs: [setup, test] + steps: + - name: Start SSH Agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Archive ci-test results + uses: ./.github/actions/archive + with: + dir: ${{ inputs.ci_workflow }} + + - name: Upload our kdevops results archive + uses: actions/upload-artifact@v4 + with: + name: kdevops-ci-results + path: ${{ inputs.ci_workflow }}/kdevops/archive/*.zip + + cleanup: + name: Cleanup kdevops workspace + runs-on: [self-hosted] + needs: [setup, test, archive] + if: always() + steps: + - name: kdevops cleanup + uses: ./.github/actions/cleanup + with: + dir: ${{ inputs.ci_workflow }} + ci_workflow: ${{ inputs.ci_workflow }} diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml new file mode 100644 index 00000000000000..7b570265c45685 --- /dev/null +++ b/.github/workflows/manual.yml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Manual + +on: + workflow_dispatch: + inputs: + ci_workflow: + description: "CI Workflow" + required: true + default: 'blktests_nvme' + type: choice + options: + - blktests + - blktests_block + - blktests_loop + - blktests_meta + - blktests_nbd + - blktests_nvme + - blktests_nvmemp + - blktests_scsi + - blktests_srp + - blktests_zbd + - tmpfs + - tmpfs_default + - tmpfs_huge + - tmpfs_noswap + - linux-btrfs-kpd + - linux-ext4-kpd + - linux-firmware-kpd + - linux-mm-kpd + - linux-modules-kpd + - linux-xfs-kpd + - selftests + kernel_tree: + description: "Linux kernel tree to use" + required: true + default: 'linux' + type: choice + options: + - linux + - linux-next + - linux-stable + kernel_ref: + description: "Linux tree git reference (branch/tag/commit-id)" + required: true + default: 'master' + type: string + +jobs: + manual: + name: Manual kdevops CI + uses: ./.github/workflows/main.yml + secrets: inherit + with: + ci_workflow: ${{ inputs.ci_workflow }} + kernel_ref: ${{ inputs.kernel_ref }} + kernel_tree: ${{ inputs.kernel_tree }} diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml new file mode 100644 index 00000000000000..d0221550845fa6 --- /dev/null +++ b/.github/workflows/schedule.yml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0 +--- +name: Run kdevops CI Workflow - Schedule + +on: + schedule: + - cron: '0 14 * * *' + + workflow_dispatch: + +jobs: + check_ref: + name: Check Linux kernel Git Reference + outputs: + kernel_ref: ${{ steps.check_kernel_ref.outputs.kernel_ref }} + kernel_tree: ${{ steps.check_kernel_ref.outputs.kernel_tree }} + runs-on: [self-hosted] + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Generate kernel_ref + id: check_kernel_ref + run: | + set -euxo pipefail + day_of_week=$(date +%u) + + kernel_ref=$(./scripts/korg-releases.py --moniker linux-next) + kernel_tree=linux-next + if [ "$day_of_week" -eq 1 ]; then + kernel_ref=$(./scripts/korg-releases.py --moniker mainline) + kernel_tree=linux + fi + + "${{ github.workspace }}/scripts/github_output.sh" kernel_ref "$kernel_ref" + "${{ github.workspace }}/scripts/github_output.sh" kernel_tree "$kernel_tree" + + schedule: + name: Scheduled kdevops CI + needs: [check_ref] + runs-on: [self-hosted] + secrets: inherit + uses: ./.github/workflows/main.yml + strategy: + matrix: + ci_workflow: [blktests-block, blktests-loop, blktests-meta] + with: + ci_workflow: ${{ matrix.ci_workflow }} + kernel_ref: ${{ needs.check_ref.outputs.kernel_ref }} + kernel_tree: ${{ needs.check_ref.outputs.kernel_tree }} diff --git a/scripts/github_output.sh b/scripts/github_output.sh new file mode 100755 index 00000000000000..711306c38cb5ff --- /dev/null +++ b/scripts/github_output.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Usage: ./github_output.sh key value +set -euxo pipefail + +key="$1" +value="$2" + +echo "$key=$value" >> "$GITHUB_OUTPUT" diff --git a/scripts/korg-releases.py b/scripts/korg-releases.py new file mode 100755 index 00000000000000..bce182e28d7926 --- /dev/null +++ b/scripts/korg-releases.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: copyleft-next-0.3.1 +import sys +import logging +import argparse +import json +import urllib.request +import socket +import re + + +def parser(): + parser = argparse.ArgumentParser(description="kernel.org/releases.json checker") + parser.add_argument("--debug", action="store_true", help="debug") + parser.add_argument( + "--moniker", + help="moniker (mainline, stable, longterm or linux-next)", + required=True, + ) + parser.add_argument( + "--pname", + help="project name for User-Agent request", + default="kdevops", + ) + parser.add_argument( + "--pversion", + help="project version for User-Agent request", + default="5.0.2", + ) + return parser + + +def _check_connection(host, port, timeout=2): + try: + with socket.create_connection((host, port), timeout): + logging.debug(f"Connection to {host} on port {port} succeeded!") + return True + except (socket.timeout, socket.error) as e: + logging.debug(f"Connection to {host} on port {port} failed: {e}") + return False + + +def kreleases(args) -> None: + """Get the latest kernel releases from kernel.org/releases.json""" + + reflist = [] + if _check_connection("kernel.org", 80): + _url = "https://www.kernel.org/releases.json" + req = urllib.request.Request( + _url, + headers={ + "User-Agent": f"{args.pname}/{args.pversion} (kdevops@lists.linux.dev)" + }, + ) + with urllib.request.urlopen(req) as url: + data = json.load(url) + + for release in data["releases"]: + if release["moniker"] == args.moniker: + # Check if release.json is aa.bb.cc type + if re.compile(r'^\d+\.\d+(\.\d+|-rc\d+)?$').match(release["version"]): + reflist.append("v" + release["version"]) + else: + reflist.append(release["version"]) + + logging.debug(f"{reflist}") + for r in reflist: + print(r) + + +def main() -> None: + """Kconfig choice generator for git refereces""" + log = logging.getLogger() + log.setLevel(logging.INFO) + p = parser() + args, _ = p.parse_known_args() + if args.debug: + log.setLevel(logging.DEBUG) + + kreleases(args) + + +if __name__ == "__main__": + ret = 0 + try: + main() + except Exception: + ret = 1 + import traceback + + traceback.print_exc() + sys.exit(ret) From d8fbc1b3b46f3732d89bbb021abfaa37b4730eba Mon Sep 17 00:00:00 2001 From: David Wang <00107082@163.com> Date: Tue, 20 May 2025 00:38:23 +0800 Subject: [PATCH 2/2] module: release codetag section when module load fails When module load fails after memory for codetag section is ready, codetag section memory will not be properly released. This causes memory leak, and if next module load happens to get the same module address, codetag may pick the uninitialized section when manipulating tags during module unload, and leads to "unable to handle page fault" BUG. Closes: https://lore.kernel.org/all/20250516131246.6244-1-00107082@163.com/ Signed-off-by: David Wang <00107082@163.com> Acked-by: Suren Baghdasaryan --- kernel/module/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/module/main.c b/kernel/module/main.c index 2b6bfaade36a7c..3d64e69cc03e93 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2816,6 +2816,7 @@ static void module_deallocate(struct module *mod, struct load_info *info) { percpu_modfree(mod); module_arch_freeing_init(mod); + codetag_free_module_sections(mod); free_mod_mem(mod); }