diff --git a/.github/workflows/checkpatch_pull.yml b/.github/workflows/checkpatch_pull.yml index 992238ea5..120a8d300 100644 --- a/.github/workflows/checkpatch_pull.yml +++ b/.github/workflows/checkpatch_pull.yml @@ -5,21 +5,34 @@ on: branches: - master +permissions: + contents: read + +concurrency: + group: checkpatch-pull-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + CHECKPATCH_LINUX_REF: v6.18 + jobs: checkpatch: runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout code - uses: actions/checkout@main + uses: actions/checkout@v6 with: ref: ${{github.event.pull_request.head.sha}} fetch-depth: 0 - name: Download checkpatch.pl run: | - curl https://raw.githubusercontent.com/torvalds/linux/master/scripts/checkpatch.pl -o checkpatch.pl - curl https://raw.githubusercontent.com/torvalds/linux/master/scripts/spelling.txt -o spelling.txt - curl https://raw.githubusercontent.com/torvalds/linux/master/scripts/const_structs.checkpatch -o const_structs.checkpatch + set -euxo pipefail + base_url="https://raw.githubusercontent.com/torvalds/linux/${CHECKPATCH_LINUX_REF}/scripts" + curl -fsSL "${base_url}/checkpatch.pl" -o checkpatch.pl + curl -fsSL "${base_url}/spelling.txt" -o spelling.txt + curl -fsSL "${base_url}/const_structs.checkpatch" -o const_structs.checkpatch chmod +x checkpatch.pl - name: Run checkpatch.pl diff --git a/.github/workflows/checkpatch_push.yml b/.github/workflows/checkpatch_push.yml index a1ea507df..3e7722de6 100644 --- a/.github/workflows/checkpatch_push.yml +++ b/.github/workflows/checkpatch_push.yml @@ -5,26 +5,39 @@ on: branches: - master +permissions: + contents: read + +concurrency: + group: checkpatch-push-${{ github.ref }} + cancel-in-progress: true + +env: + CHECKPATCH_LINUX_REF: v6.18 + jobs: checkpatch: name: ${{matrix.commit.message}} runs-on: ubuntu-latest + timeout-minutes: 20 strategy: fail-fast: false matrix: commit: ${{github.event.commits}} steps: - name: Checkout code - uses: actions/checkout@main + uses: actions/checkout@v6 with: ref: ${{matrix.commit.id}} fetch-depth: 2 - name: Download checkpatch.pl run: | - curl https://raw.githubusercontent.com/torvalds/linux/master/scripts/checkpatch.pl -o checkpatch.pl - curl https://raw.githubusercontent.com/torvalds/linux/master/scripts/spelling.txt -o spelling.txt - curl https://raw.githubusercontent.com/torvalds/linux/master/scripts/const_structs.checkpatch -o const_structs.checkpatch + set -euxo pipefail + base_url="https://raw.githubusercontent.com/torvalds/linux/${CHECKPATCH_LINUX_REF}/scripts" + curl -fsSL "${base_url}/checkpatch.pl" -o checkpatch.pl + curl -fsSL "${base_url}/spelling.txt" -o spelling.txt + curl -fsSL "${base_url}/const_structs.checkpatch" -o const_structs.checkpatch chmod +x checkpatch.pl - name: Run checkpatch.pl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02bd99829..ce69df342 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,27 +1,177 @@ name: CI -on: [push, pull_request] +on: + pull_request: + branches: + - master + push: + branches: + - master + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_MAXSIZE: 500M jobs: build: + name: build-${{ matrix.compiler }} runs-on: ubuntu-latest + timeout-minutes: 45 strategy: + fail-fast: false matrix: compiler: [gcc, clang] - arch: [x86_64] + continue-on-error: ${{ matrix.compiler == 'clang' }} steps: - - uses: actions/checkout@main - - name: Install build-essential + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install dependencies run: | + set -euxo pipefail sudo apt-get update - sudo apt-get install -y build-essential debhelper devscripts dpkg-dev quilt - - name: Install multilib + sudo apt-get install -y \ + build-essential \ + ccache \ + clang \ + debhelper \ + devscripts \ + dpkg-dev \ + libelf-dev \ + libsqlite3-dev \ + quilt \ + sparse + if ! [ -e "/lib/modules/$(uname -r)/build/Makefile" ]; then + sudo apt-get install -y "linux-headers-$(uname -r)" || \ + sudo apt-get install -y linux-headers-generic + fi + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ matrix.compiler }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ matrix.compiler }}-${{ github.ref_name }}- + ccache-${{ runner.os }}-${{ matrix.compiler }}- + + - name: Print toolchain and kernel headers run: | - sudo apt-get install -y gcc-multilib g++-multilib - if: ${{ matrix.arch == 'x86' }} - - name: Build + set -euxo pipefail + uname -a + gcc --version | head -n 1 + clang --version | head -n 1 + ccache --version | head -n 1 + printf 'KDIR=%s\n' "$(readlink -f /lib/modules/$(uname -r)/build || echo missing)" + make --version | head -n 1 + + - name: Build modules and userspace env: CC: ${{ matrix.compiler }} - CFLAGS: -Werror ${{ matrix.arch == 'x86' && '-m32' || '' }} - LDFLAGS: ${{ matrix.arch == 'x86' && '-m32' || '' }} - run: make CC=${{matrix.compiler}} dpkg + PASS_CC_TO_MAKE: "1" + run: | + set -euxo pipefail + export PATH="/usr/lib/ccache:${PATH}" + mkdir -p artifacts + ccache --set-config=max_size="${CCACHE_MAXSIZE}" + ccache --zero-stats + + make 2release + make all 2>&1 | tee "artifacts/build-${CC}.log" + build_rc=${PIPESTATUS[0]} + + ccache --show-stats | tee "artifacts/ccache-${CC}.txt" + exit "${build_rc}" + + - name: Upload build logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: ci-${{ matrix.compiler }}-logs + path: artifacts/ + if-no-files-found: error + + - name: Job summary + if: always() + run: | + { + echo "### Build profile" + echo "- Compiler: \`${{ matrix.compiler }}\`" + echo "- KDIR: \`$(readlink -f /lib/modules/$(uname -r)/build || echo missing)\`" + echo "- ccache dir: \`${CCACHE_DIR}\`" + } >> "$GITHUB_STEP_SUMMARY" + + package-deb: + name: package-deb + runs-on: ubuntu-latest + timeout-minutes: 45 + needs: build + if: github.event_name == 'push' + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install dependencies + run: | + set -euxo pipefail + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + ccache \ + debhelper \ + devscripts \ + dpkg-dev \ + quilt + if ! [ -e "/lib/modules/$(uname -r)/build/Makefile" ]; then + sudo apt-get install -y "linux-headers-$(uname -r)" || \ + sudo apt-get install -y linux-headers-generic + fi + + - name: Restore ccache + uses: actions/cache@v5 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-gcc-packages-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-gcc-packages-${{ github.ref_name }}- + ccache-${{ runner.os }}-gcc-packages- + + - name: Build Debian packages + env: + CC: gcc + PASS_CC_TO_MAKE: "1" + run: | + set -euxo pipefail + export PATH="/usr/lib/ccache:${PATH}" + mkdir -p artifacts + ccache --set-config=max_size="${CCACHE_MAXSIZE}" + + make dpkg 2>&1 | tee artifacts/dpkg.log + build_rc=${PIPESTATUS[0]} + + if [ -d dpkg ]; then + find dpkg -maxdepth 1 -type f -printf '%P\n' | sort > artifacts/dpkg-files.txt + fi + + exit "${build_rc}" + + - name: Upload package artifacts + if: always() + uses: actions/upload-artifact@v6 + with: + name: ci-deb-packages + path: | + artifacts/ + dpkg/ + if-no-files-found: warn diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index e643a0169..b8e25ac1b 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -5,13 +5,21 @@ on: branches: - master +permissions: + contents: read + +concurrency: + group: coverity-${{ github.ref }} + cancel-in-progress: true + jobs: coverity: if: github.repository == 'SCST-project/scst' runs-on: ubuntu-latest + timeout-minutes: 120 steps: - name: Checkout Project - uses: actions/checkout@main + uses: actions/checkout@v6 - name: Get version id: get_version @@ -68,7 +76,7 @@ jobs: # Command to pass to cov-build. # # Default: 'make' - command: 'make -i cov-build' + command: 'make cov-build' # (Informational) The source version being built. # diff --git a/.github/workflows/mail_notification.yml b/.github/workflows/mail_notification.yml index 06b5fd046..1161cc3e8 100644 --- a/.github/workflows/mail_notification.yml +++ b/.github/workflows/mail_notification.yml @@ -6,18 +6,22 @@ on: branches: - master +permissions: + contents: read + jobs: send_mails: name: ${{matrix.commit.message}} if: github.repository == 'SCST-project/scst' runs-on: ubuntu-latest + timeout-minutes: 20 strategy: max-parallel: 1 matrix: commit: ${{github.event.commits}} steps: - name: Checkout Project - uses: actions/checkout@main + uses: actions/checkout@v6 with: ref: ${{matrix.commit.id}} fetch-depth: 2 @@ -58,7 +62,7 @@ jobs: fi - name: Send mail - uses: dawidd6/action-send-mail@v3 + uses: dawidd6/action-send-mail@v9 with: # Required mail server address: server_address: smtp.yandex.com diff --git a/.github/workflows/run_regression_tests.yaml b/.github/workflows/run_regression_tests.yaml index ccdb10b26..c0ed3e3fb 100644 --- a/.github/workflows/run_regression_tests.yaml +++ b/.github/workflows/run_regression_tests.yaml @@ -1,97 +1,76 @@ name: Run regression tests on: - push: + pull_request: branches: - master - pull_request: + push: branches: - master + schedule: + - cron: '35 2 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: regression-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name != 'schedule' }} + +env: + MATRIX_PR: >- + ["6.12.63"] + MATRIX_PUSH: >- + ["6.12.63","5.15.197","5.14.0-611.16.1.el9_7^AlmaLinux^9.7"] + MATRIX_NIGHTLY: >- + ["6.18.2","6.12.63","6.6.119","6.1.159","5.15.197","5.10.247","4.19.325","3.10.108","6.12.0-124.21.1.el10_1^AlmaLinux^10.1","5.14.0-611.16.1.el9_7^AlmaLinux^9.7","4.18.0-553.87.1.el8_10^AlmaLinux^8.10","3.10.0-1160.118.1.el7^CentOS^7.9.2009","6.12.0-106.55.4.2.el10uek^UEK^10","5.4.17-2136.350.3.2.el8uek^UEK^8"] jobs: regression_tests: - name: ${{matrix.version}} + name: ${{ matrix.version }} runs-on: ubuntu-latest + timeout-minutes: 240 strategy: fail-fast: false matrix: - version: [ - '6.18.2', - '6.17.13', - '6.16.12', - '6.15.11', - '6.14.11', - '6.13.12', - '6.12.63', - '6.11.11', - '6.10.14', - '6.9.12', - '6.8.12', - '6.7.12', - '6.6.119', - '6.1.159', - '5.15.197', - '5.10.247', - '5.4.302', - '4.19.325', - '4.14.336', - '4.9.337', - '3.18.140', - '3.10.108', - '6.12.0-124.21.1.el10_1^AlmaLinux^10.1', - '6.12.0-55.43.1.el10_0^AlmaLinux^10.0', - '5.14.0-611.16.1.el9_7^AlmaLinux^9.7', - '5.14.0-570.62.1.el9_6^AlmaLinux^9.6', - '5.14.0-503.40.1.el9_5^AlmaLinux^9.5', - '5.14.0-427.42.1.el9_4^AlmaLinux^9.4', - '5.14.0-362.24.1.el9_3^AlmaLinux^9.3', - '5.14.0-284.30.1.el9_2^AlmaLinux^9.2', - '5.14.0-162.23.1.el9_1^AlmaLinux^9.1', - '5.14.0-70.30.1.el9_0^AlmaLinux^9.0', - '4.18.0-553.87.1.el8_10^AlmaLinux^8.10', - '4.18.0-513.24.1.el8_9^AlmaLinux^8.9', - '4.18.0-477.13.1.el8_8^AlmaLinux^8.8', - '4.18.0-425.19.2.el8_7^AlmaLinux^8.7', - '4.18.0-372.32.1.el8_6^AlmaLinux^8.6', - '4.18.0-348.23.1.el8_5^AlmaLinux^8.5', - '4.18.0-305.25.1.el8_4^AlmaLinux^8.4', - '4.18.0-240.22.1.el8_3^AlmaLinux^8.3', - '3.10.0-1160.118.1.el7^CentOS^7.9.2009', - '3.10.0-862.14.4.el7^CentOS^7.5.1804', - '6.12.0-106.55.4.2.el10uek^UEK^10', - '5.15.0-315.196.5.2.el9uek^UEK^9', - '5.4.17-2136.350.3.2.el8uek^UEK^8', - '4.14.35-2047.543.3.1.el7uek^UEK^7', - '4.1.12-124.93.1.el7uek^UEK^7' - ] + version: ${{ fromJson(github.event_name == 'schedule' && env.MATRIX_NIGHTLY || github.event_name == 'workflow_dispatch' && env.MATRIX_NIGHTLY || github.event_name == 'push' && env.MATRIX_PUSH || env.MATRIX_PR) }} steps: - name: Checkout code - uses: actions/checkout@main + uses: actions/checkout@v6 - - name: Install libelf-dev + - name: Install dependencies run: | + set -euxo pipefail sudo apt-get update - sudo apt-get install -y libelf-dev libsqlite3-dev - - - name: Install sparse - run: | - sudo apt-get install sparse - - - name: Install smatch - run: | - git clone https://github.com/error27/smatch.git - cd smatch - make -j - sudo BINDIR=/bin SHAREDIR=/home/runner/share make install + sudo apt-get install -y libelf-dev libsqlite3-dev sparse + if ! command -v smatch >/dev/null 2>&1; then + sudo apt-get install -y smatch || true + fi + if ! command -v smatch >/dev/null 2>&1; then + git clone --depth 1 https://github.com/error27/smatch.git + cd smatch + make -j"$(nproc)" + sudo BINDIR=/bin SHAREDIR=/home/runner/share make install + fi - name: Run regression tests run: | + set -euo pipefail + err=0 - version="${{matrix.version}}" - workdir="/tmp/scst-${version}" - out="output.txt" + version="${{ matrix.version }}" + safe_version="${version//[^[:alnum:]._-]/_}" + workdir="/tmp/scst-${safe_version}" + out="artifacts/output-${safe_version}.txt" + + mkdir -p artifacts ./scripts/run-regression-tests -l -q -d "${workdir}" "${version}-nc-ns-nm" | tee "${out}" + cmd_rc=${PIPESTATUS[0]} + if [ "${cmd_rc}" -ne 0 ]; then + err=1 + fi if ! grep -q "Compiling the patched kernel" "${out}"; then echo "::error title=Regression tests::run-regression-tests failed" @@ -103,14 +82,82 @@ jobs: else echo "::notice title=Regression tests::Succeeded" fi + fi - f=( "${workdir}"/compilation-*-output.txt ) + if comp=( "${workdir}"/compilation-*-output.txt ) && [ -e "${comp[0]}" ]; then + cp "${comp[0]}" "artifacts/$(basename "${comp[0]}")" echo - echo "===== $(basename "${f[0]}") =====" - cat "${f[0]}" + echo "===== $(basename "${comp[0]}") =====" + cat "${comp[0]}" fi + { + echo "### Regression profile" + echo "- Event: \`${{ github.event_name }}\`" + echo "- Kernel: \`${version}\`" + echo "- Result: \`$([ "${err}" -eq 0 ] && echo pass || echo fail)\`" + } >> "$GITHUB_STEP_SUMMARY" + rm -rf "${workdir}" - rm -f "${out}" + exit "${err}" + + - name: Upload regression logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: regression-${{ replace(replace(replace(matrix.version, '^', '_'), '/', '_'), ':', '_') }}-logs + path: artifacts/ + if-no-files-found: error + + nightly_quality: + name: nightly-quality + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Install dependencies + run: | + set -euxo pipefail + sudo apt-get update + sudo apt-get install -y build-essential libelf-dev libsqlite3-dev sparse + if ! [ -e "/lib/modules/$(uname -r)/build/Makefile" ]; then + sudo apt-get install -y "linux-headers-$(uname -r)" || \ + sudo apt-get install -y linux-headers-generic + fi + if ! command -v smatch >/dev/null 2>&1; then + sudo apt-get install -y smatch || true + fi + if ! command -v smatch >/dev/null 2>&1; then + git clone --depth 1 https://github.com/error27/smatch.git + cd smatch + make -j"$(nproc)" + sudo BINDIR=/bin SHAREDIR=/home/runner/share make install + fi + + - name: Run W=1, sparse and smatch builds + run: | + set -euxo pipefail + mkdir -p artifacts + + make 2release + make PASS_CC_TO_MAKE=1 W=1 all 2>&1 | tee artifacts/w1-build.log + make PASS_CC_TO_MAKE=1 C=2 -C scst/src all 2>&1 | tee artifacts/sparse-scst-src.log + make PASS_CC_TO_MAKE=1 C=2 CHECK=smatch -C scst/src all 2>&1 | tee artifacts/smatch-scst-src.log + + { + echo "### Nightly quality checks" + echo "- W=1 build: done" + echo "- sparse profile: done" + echo "- smatch profile: done" + } >> "$GITHUB_STEP_SUMMARY" - exit $err + - name: Upload nightly quality logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: nightly-quality-logs + path: artifacts/ + if-no-files-found: error diff --git a/Makefile b/Makefile index 9cfbaeabf..78569dbf6 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,14 @@ FCST_DIR = fcst EMULEX_DIR = emulex ISCSI_DIR = iscsi-scst +ifdef SOURCE_DATE_EPOCH +BUILD_DATE := $(shell \ + date -u -d "@$(SOURCE_DATE_EPOCH)" '+%Y/%m/%d-%H:%M:%S-%Z%:z' \ + 2>/dev/null || \ + date -u -r "$(SOURCE_DATE_EPOCH)" '+%Y/%m/%d-%H:%M:%S-%Z%:z') +else BUILD_DATE := $(shell date -u '+%Y/%m/%d-%H:%M:%S-%Z%:z') +endif GIT_COMMIT := $(shell git rev-parse --short=12 HEAD 2>/dev/null) ifndef BUILD_NUMBER @@ -177,6 +184,8 @@ help: @echo " rpm-dkms : make both SCST DKMS and scstadmin RPM packages" @echo "" @echo " dpkg : make SCST dpkg packages" + @echo " ci : CI-like release build (2release + all)" + @echo " ci-package : CI-like Debian package build (dpkg)" @echo "" @echo " 2perf : changes debug state to full performance" @echo " 2release : changes debug state to release" @@ -189,22 +198,29 @@ tags: find . -type f -name "*.[ch]" | ctags --c-kinds=+p --fields=+iaS --extra=+q -e -L- cov-build: - -for d in $(SCST_DIR) $(ISCSI_DIR) $(OLD_QLA_DIR) $(NEW_QLA_DIR) $(SRP_DIR) \ + @for d in $(SCST_DIR) $(ISCSI_DIR) $(OLD_QLA_DIR) $(NEW_QLA_DIR) $(SRP_DIR) \ $(SCST_LOCAL_DIR) $(FCST_DIR) $(USR_DIR) $(SCSTADM_DIR); do \ if [[ $$d = $(OLD_QLA_DIR) || $$d = $(NEW_QLA_DIR) ]]; then \ - BUILD_2X_MODULE=y $(MAKE) -j$$(nproc) -C "$$d" all || break; \ + BUILD_2X_MODULE=y $(MAKE) -j$$(nproc) -C "$$d" all || exit $$?; \ else \ - $(MAKE) -j$$(nproc) -C "$$d" all || break; \ + $(MAKE) -j$$(nproc) -C "$$d" all || exit $$?; \ fi \ done all clean extraclean install uninstall: - -if [ $@ = extraclean ]; then rm -f TAGS tags cscope.out; fi - -for d in $(SCST_DIR) $(ISCSI_DIR) $(QLA_DIR) $(SRP_DIR) \ + @if [ "$@" = extraclean ]; then rm -f TAGS tags cscope.out; fi + @for d in $(SCST_DIR) $(ISCSI_DIR) $(QLA_DIR) $(SRP_DIR) \ $(SCST_LOCAL_DIR) $(FCST_DIR) $(USR_DIR) $(SCSTADM_DIR); do \ - $(MAKE) -j$$(nproc) -C "$$d" $@ || break; \ + $(MAKE) -j$$(nproc) -C "$$d" "$@" || exit $$?; \ done +ci: + $(MAKE) 2release + $(MAKE) all + +ci-package: + $(MAKE) dpkg + scst: cd $(SCST_DIR) && $(MAKE) all @@ -528,5 +544,5 @@ multiple-release-archives: fcst fcst_clean fcst_extraclean fcst_install fcst_uninstall \ scst_local scst_local_clean scst_local_extraclean scst_local_install scst_local_uninstall \ usr usr_clean usr_extraclean usr_install usr_uninstall \ - scst-rpm scst-dkms-rpm scstadm-rpm rpm rpm-dkms dpkg \ + scst-rpm scst-dkms-rpm scstadm-rpm rpm rpm-dkms dpkg ci ci-package \ 2perf 2release 2debug diff --git a/scripts/run-regression-tests b/scripts/run-regression-tests index f16ab18ba..6d4824e58 100755 --- a/scripts/run-regression-tests +++ b/scripts/run-regression-tests @@ -114,7 +114,6 @@ function make_rpm { function make_dpkg { local outputfile="${outputdir}/make-dpkg-output.txt" - outputfile="${outputdir}/make-${t}-output.txt" echo "Testing whether 'make dpkg' works fine ..." if make dpkg > "${outputfile}" 2>&1; then echo "OK"