From 60ede7e81e310d63e891aeed81ab069134963a76 Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Wed, 23 Jul 2025 16:17:37 +0100 Subject: [PATCH 1/5] CCM-11345 Manual Repo Sync --- .github/actions/check-todo-usage/action.yaml | 10 + .../create-lines-of-code-report/action.yaml | 2 +- .github/actions/scan-dependencies/action.yaml | 2 +- .github/actions/trivy/action.yaml | 4 +- .github/workflows/cicd-3-deploy.yaml | 1 - .../scheduled-repository-template-sync.yaml | 55 ++++ .github/workflows/scorecard.yml | 14 +- .github/workflows/stage-1-commit.yaml | 11 + .github/workflows/stage-3-build.yaml | 4 +- .gitignore | 13 +- .gitleaksignore | 1 + .tool-versions | 17 +- infrastructure/terraform/.gitignore | 67 +++++ .../config/.repository-template-sync-ignore | 36 +++ .../config/.repository-template-sync-merge | 9 + scripts/config/gitleaks.toml | 7 + scripts/config/pre-commit.yaml | 15 +- .../config/vocabularies/words/accept.txt | 1 + scripts/githooks/check-todos.sh | 238 ++++++++++++++++++ scripts/githooks/sort-dictionary.sh | 49 ++++ scripts/githooks/sync-template-repo.sh | 144 +++++++++++ scripts/init.mk | 4 +- scripts/maintenance/merge.js | 81 ++++++ 23 files changed, 759 insertions(+), 26 deletions(-) create mode 100644 .github/actions/check-todo-usage/action.yaml create mode 100644 .github/workflows/scheduled-repository-template-sync.yaml create mode 100644 infrastructure/terraform/.gitignore create mode 100644 scripts/config/.repository-template-sync-ignore create mode 100644 scripts/config/.repository-template-sync-merge create mode 100755 scripts/githooks/check-todos.sh create mode 100755 scripts/githooks/sort-dictionary.sh create mode 100755 scripts/githooks/sync-template-repo.sh create mode 100644 scripts/maintenance/merge.js diff --git a/.github/actions/check-todo-usage/action.yaml b/.github/actions/check-todo-usage/action.yaml new file mode 100644 index 0000000..a403d58 --- /dev/null +++ b/.github/actions/check-todo-usage/action.yaml @@ -0,0 +1,10 @@ +name: "Check Todo usage" +description: "Check Todo usage" +runs: + using: "composite" + steps: + - name: "Check Todo usage" + shell: bash + run: | + export BRANCH_NAME=origin/${{ github.event.repository.default_branch }} + check=branch ./scripts/githooks/check-todos.sh diff --git a/.github/actions/create-lines-of-code-report/action.yaml b/.github/actions/create-lines-of-code-report/action.yaml index b21f066..86396f7 100644 --- a/.github/actions/create-lines-of-code-report/action.yaml +++ b/.github/actions/create-lines-of-code-report/action.yaml @@ -44,7 +44,7 @@ runs: echo "secrets_exist=${{ inputs.idp_aws_report_upload_role_name != '' && inputs.idp_aws_report_upload_bucket_endpoint != '' }}" >> $GITHUB_OUTPUT - name: "Authenticate to send the report" if: steps.check.outputs.secrets_exist == 'true' - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::${{ inputs.idp_aws_report_upload_account_id }}:role/${{ inputs.idp_aws_report_upload_role_name }} aws-region: ${{ inputs.idp_aws_report_upload_region }} diff --git a/.github/actions/scan-dependencies/action.yaml b/.github/actions/scan-dependencies/action.yaml index f8ed605..1000df1 100644 --- a/.github/actions/scan-dependencies/action.yaml +++ b/.github/actions/scan-dependencies/action.yaml @@ -58,7 +58,7 @@ runs: run: echo "secrets_exist=${{ inputs.idp_aws_report_upload_role_name != '' && inputs.idp_aws_report_upload_bucket_endpoint != '' }}" >> $GITHUB_OUTPUT - name: "Authenticate to send the reports" if: steps.check.outputs.secrets_exist == 'true' - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::${{ inputs.idp_aws_report_upload_account_id }}:role/${{ inputs.idp_aws_report_upload_role_name }} aws-region: ${{ inputs.idp_aws_report_upload_region }} diff --git a/.github/actions/trivy/action.yaml b/.github/actions/trivy/action.yaml index 4f5bbbe..be940ce 100644 --- a/.github/actions/trivy/action.yaml +++ b/.github/actions/trivy/action.yaml @@ -5,9 +5,11 @@ runs: - name: "Trivy Terraform IAC Scan" shell: bash run: | + components_exit_code=0 modules_exit_code=0 - ./scripts/terraform/trivy.sh ./infrastructure/modules || modules_exit_code=$? + ./scripts/terraform/trivy.sh ./infrastructure/terraform/components || components_exit_code=$? + ./scripts/terraform/trivy.sh ./infrastructure/terraform/modules || modules_exit_code=$? if [ $components_exit_code -ne 0 ] || [ $modules_exit_code -ne 0 ]; then echo "Trivy misconfigurations detected." diff --git a/.github/workflows/cicd-3-deploy.yaml b/.github/workflows/cicd-3-deploy.yaml index ff2ad64..5230577 100644 --- a/.github/workflows/cicd-3-deploy.yaml +++ b/.github/workflows/cicd-3-deploy.yaml @@ -48,7 +48,6 @@ jobs: echo "nodejs_version=$(grep "^nodejs\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT echo "python_version=$(grep "^python\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT echo "terraform_version=$(grep "^terraform\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT - # TODO: Get the version, but it may not be the .version file as this should come from the CI/CD Pull Request Workflow echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT # echo "tag=${{ github.event.inputs.tag }}" >> $GITHUB_OUTPUT - name: "List variables" diff --git a/.github/workflows/scheduled-repository-template-sync.yaml b/.github/workflows/scheduled-repository-template-sync.yaml new file mode 100644 index 0000000..e911486 --- /dev/null +++ b/.github/workflows/scheduled-repository-template-sync.yaml @@ -0,0 +1,55 @@ +name: Repository Template Sync + +on: + schedule: + - cron: '0 0 1 * *' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + checks: read + +jobs: + update-external-repo: + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Check out external repository + uses: actions/checkout@v4 + with: + repository: NHSDigital/nhs-notify-repository-template + path: nhs-notify-repository-template + token: ${{ github.token }} + + - name: Run syncronisation script + run: | + ./nhs-notify-repository-template/scripts/githooks/sync-template-repo.sh + rm -Rf ./nhs-notify-repository-template + + - name: Create Pull Request + if: ${{ !env.ACT }} + uses: peter-evans/create-pull-request@v7.0.8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Drift from template + branch: scheduledTemplateRepositorySync + delete-branch: true + title: '[Template Sync] Drift from template-repository remediation' + body: | + # Resultant drift from repository template + + ## Who should respond to this PR? + The team which owns the responsibility for this component repository. You may want to consult other contributors. + + ## How to progress this PR + The repositories guardians should review the contents of the PR and decide how to proceed, you may wish to back-out certain changes or accept them from the upstream `nhsdigital/nhs-notify-repository-template` repository. + + If there are changes you do not wish to see again, it is recommended you add exclusions to `scripts/config/.repository-template-sync-ignore`. + labels: | + template + automation + draft: false diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index ca5237a..03bf171 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,17 +27,17 @@ jobs: # Needed to publish results and get a badge (see publish_results below). id-token: write # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read + contents: read + actions: read steps: - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 with: results_file: results.sarif results_format: sarif @@ -45,7 +45,7 @@ jobs: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecard on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} + repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20 + uses: actions/upload-artifact@v4 with: name: SARIF file path: results.sarif @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@45775bd8235c68ba998cffa5171334d58593da47 # v3.28.15 + uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9 with: sarif_file: results.sarif diff --git a/.github/workflows/stage-1-commit.yaml b/.github/workflows/stage-1-commit.yaml index f352eb7..e83ee95 100644 --- a/.github/workflows/stage-1-commit.yaml +++ b/.github/workflows/stage-1-commit.yaml @@ -101,6 +101,17 @@ jobs: fetch-depth: 0 # Full history is needed to compare branches - name: "Check English usage" uses: ./.github/actions/check-english-usage + check-todo-usage: + name: "Check TODO usage" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: "Checkout code" + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history is needed to compare branches + - name: "Check TODO usage" + uses: ./.github/actions/check-todo-usage detect-terraform-changes: name: "Detect Terraform Changes" runs-on: ubuntu-latest diff --git a/.github/workflows/stage-3-build.yaml b/.github/workflows/stage-3-build.yaml index 201d5b1..fabfe11 100644 --- a/.github/workflows/stage-3-build.yaml +++ b/.github/workflows/stage-3-build.yaml @@ -60,7 +60,7 @@ jobs: - name: "Upload artefact 1" run: | echo "Uploading artefact 1 ..." - # TODO: Use either action/cache or action/upload-artifact + # Use either action/cache or action/upload-artifact artefact-n: name: "Artefact n" runs-on: ubuntu-latest @@ -77,4 +77,4 @@ jobs: - name: "Upload artefact n" run: | echo "Uploading artefact n ..." - # TODO: Use either action/cache or action/upload-artifact + # Use either action/cache or action/upload-artifact diff --git a/.gitignore b/.gitignore index aa6484a..e97bd34 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,15 @@ version.json *.code-workspace !project.code-workspace -infrastructure/modules/eventpub/lambda/*.zip +# Please, add your custom content below! + +# dependencies +node_modules +.node-version +*/node_modules +/.pnp +.pnp.js +/build +dist +.DS_Store +.reports diff --git a/.gitleaksignore b/.gitleaksignore index 1c92293..3356f79 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -1,5 +1,6 @@ # SEE: https://github.com/gitleaks/gitleaks/blob/master/README.md#gitleaksignore cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md:generic-api-key:37 +cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md:sonar-api-token:37 96096685ab3d6876671e2bc9a6ff4d48fc56e521:src/helloworld/helloworld.sln:ipv4:4 4f4e8c15629b2cb09356a7fed4d72953590227ce:docs/Gemfile.lock:ipv4:4 diff --git a/.tool-versions b/.tool-versions index 88a5daf..34d9b8a 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,22 +1,21 @@ -# This file is for you! Please, updated to the versions agreed by your team. - -gitleaks 8.18.4 -nodejs 22.15.1 +act 0.2.64 +gitleaks 8.24.0 +jq 1.6 +nodejs 22.11.0 pre-commit 3.6.0 terraform 1.9.2 terraform-docs 0.19.0 -terraform-docs 0.19.0 trivy 0.61.0 vale 3.6.0 - +# python 3.13.2 # ============================================================================== # The section below is reserved for Docker image versions. # TODO: Move this section - consider using a different file for the repository template dependencies. -# docker/ghcr.io/anchore/grype v0.69.1@sha256:d41fcb371d0af59f311e72123dff46900ebd6d0482391b5a830853ee4f9d1a76 # SEE: https://github.com/anchore/grype/pkgs/container/grype -# docker/ghcr.io/anchore/syft v0.92.0@sha256:63c60f0a21efb13e80aa1359ab243e49213b6cc2d7e0f8179da38e6913b997e0 # SEE: https://github.com/anchore/syft/pkgs/container/syft -# docker/ghcr.io/gitleaks/gitleaks v8.18.0@sha256:fd2b5cab12b563d2cc538b14631764a1c25577780e3b7dba71657d58da45d9d9 # SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks +# docker/ghcr.io/anchore/grype v0.92.2@sha256:651e558f9ba84f2a790b3449c8a57cbbf4f34e004f7d3f14ae8f8cbeede4cd33 # SEE: https://github.com/anchore/grype/pkgs/container/grype +# docker/ghcr.io/anchore/syft v1.26.0@sha256:de078f51704a213906970b1475edd6006b8af50aa159852e125518237487b8c6 # SEE: https://github.com/anchore/syft/pkgs/container/syft +# docker/ghcr.io/gitleaks/gitleaks:v8.24.0@sha256:b8e9bf46893c2f20e10bfb4b2e783adaef519dea981b01ca6221ac325e836040 # SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks # docker/ghcr.io/igorshubovych/markdownlint-cli v0.37.0@sha256:fb3e79946fce78e1cde84d6798c6c2a55f2de11fc16606a40d49411e281d950d # SEE: https://github.com/igorshubovych/markdownlint-cli/pkgs/container/markdownlint-cli # docker/ghcr.io/make-ops-tools/gocloc latest@sha256:6888e62e9ae693c4ebcfed9f1d86c70fd083868acb8815fe44b561b9a73b5032 # SEE: https://github.com/make-ops-tools/gocloc/pkgs/container/gocloc # docker/ghcr.io/nhs-england-tools/github-runner-image 20230909-321fd1e-rt@sha256:ce4fd6035dc450a50d3cbafb4986d60e77cb49a71ab60a053bb1b9518139a646 # SEE: https://github.com/nhs-england-tools/github-runner-image/pkgs/container/github-runner-image diff --git a/infrastructure/terraform/.gitignore b/infrastructure/terraform/.gitignore new file mode 100644 index 0000000..579b641 --- /dev/null +++ b/infrastructure/terraform/.gitignore @@ -0,0 +1,67 @@ +### Terraform ### + +# Transient backends +components/**/backend_tfscaffold.tf + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Compiled files +**/*.tfstate +**/*.tfplan +**/*.tfstate.backup +**/.terraform +**/.terraform.lock.hcl +**/.terraform/* +**/build/* +**/work/* +**/*tfstate.lock.info + +# Scaffold Plugin Cache +plugin-cache/* + +# PyCache +**/__pycache__ + +### OSX ### +**/.DS_Store +**/.AppleDouble +**/.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +*.swp +.nyc_output + +# VS Code +.vscode + +# IntelliJ Idea +.idea +**/*.iml + +# js +node_modules diff --git a/scripts/config/.repository-template-sync-ignore b/scripts/config/.repository-template-sync-ignore new file mode 100644 index 0000000..e162065 --- /dev/null +++ b/scripts/config/.repository-template-sync-ignore @@ -0,0 +1,36 @@ +# Files and folders to ignore when syncing nhs-notify-repository-template back in to this repository +nhs-notify-repository-template/ + +# Files and Folders in this repository to ignore +.editorconfig +.github/CODEOWNERS +.github/ISSUE_TEMPLATE +.github/workflows/cicd-*.yaml +.github/workflows/stage-*.yaml +.gitleaksignore +.vscode/ +CHANGELOG.md +Makefile +project.code-workspace +README.md +scripts/config/sonar-scanner.properties +scripts/githooks/check-terraform-docs.sh +scripts/tests/ +VERSION + +# Files and Folders in the template repository to disregard +.devcontainer/ +.github/actions/build-docs +.github/workflows/*.disabled +*/examples/ +docs/ +eslint.config.mjs +infrastructure/terraform/ +infrastructure/terraform/components/ +lambdas/example-lambda/ +package-lock.json +package.json +scripts/**/examples/ +scripts/terraform/terraform.lib.sh +scripts/terraform/terraform.mk +src/.vscode/ diff --git a/scripts/config/.repository-template-sync-merge b/scripts/config/.repository-template-sync-merge new file mode 100644 index 0000000..8b5774f --- /dev/null +++ b/scripts/config/.repository-template-sync-merge @@ -0,0 +1,9 @@ +# Files and folders to merge when syncing nhs-notify-repository-template back in to this repository +.github/workflows/cicd-*.yaml +scripts/config/.repository-template-sync-ignore +scripts/config/.repository-template-sync-merge +scripts/config/vale/vale.ini +.tool-versions +.gitignore +scripts/config/vale/styles/config/vocabularies/words/accept.txt +scripts/config/vale/styles/config/vocabularies/words/reject.txt diff --git a/scripts/config/gitleaks.toml b/scripts/config/gitleaks.toml index e4b6bc1..7b8360a 100644 --- a/scripts/config/gitleaks.toml +++ b/scripts/config/gitleaks.toml @@ -1,4 +1,5 @@ # SEE: https://github.com/gitleaks/gitleaks/#configuration +# Do not edit this file directly as it will be overwritten by changes from the nhs-notify-repository-template on next sync [extend] useDefault = true # SEE: https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml @@ -22,3 +23,9 @@ paths = [ '''yarn.lock''', '''Gemfile.lock''', ] + +# Exclude Chrome version in user agent +regexTarget = "line" +regexes = [ + '''Chrome/[\d.]+''' +] diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index 8717994..8843370 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -9,12 +9,18 @@ repos: - id: check-symlinks - id: detect-private-key - id: end-of-file-fixer - exclude: .+\.cs - id: forbid-new-submodules - id: mixed-line-ending - id: pretty-format-json args: ['--autofix'] # - id: ... + - repo: local + hooks: + - id: sort-dictionary + name: Sort dictionary + entry: ./scripts/githooks/sort-dictionary.sh + language: script + pass_filenames: false - repo: local hooks: - id: scan-secrets @@ -57,3 +63,10 @@ repos: entry: ./scripts/githooks/check-terraform-docs.sh language: script pass_filenames: false + - repo: local + hooks: + - id: check-todo-usage + name: Check TODO usage + entry: /usr/bin/env check=branch ./scripts/githooks/check-todos.sh + language: script + pass_filenames: false diff --git a/scripts/config/vale/styles/config/vocabularies/words/accept.txt b/scripts/config/vale/styles/config/vocabularies/words/accept.txt index 3b07d50..f02f1fa 100644 --- a/scripts/config/vale/styles/config/vocabularies/words/accept.txt +++ b/scripts/config/vale/styles/config/vocabularies/words/accept.txt @@ -9,6 +9,7 @@ drawio endcapture endfor endraw +Git[Hh]ub GitHub Gitleaks Grype diff --git a/scripts/githooks/check-todos.sh b/scripts/githooks/check-todos.sh new file mode 100755 index 0000000..83b7a80 --- /dev/null +++ b/scripts/githooks/check-todos.sh @@ -0,0 +1,238 @@ +#!/bin/bash + +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail + +# Pre-commit git hook to scan for secrets hard-coded in the codebase. This is a +# gitleaks command wrapper. It will run gitleaks natively if it is installed, +# otherwise it will run it in a Docker container. +# +# Usage: +# $ [options] ./scan-secrets.sh +# +# Options: +# check=all # check all files in the repository +# check=staged-changes # check only files staged for commit. +# check=working-tree-changes # check modified, unstaged files. This is the default. +# check=branch # check for all changes since branching from $BRANCH_NAME +# VERBOSE=true # Show all the executed commands, default is 'false' +# +# Exit codes: +# 0 - No Todos +# 1 - Todos found or error encountered +# 126 - Unknown flag + +# ============================================================================== + +EXCLUDED_FILES=( + ".devcontainer/devcontainer.json" + ".tool-versions" + ".vscode/extensions.json" + "infrastructure/terraform/bin/terraform.sh" + "Makefile" + "project.code-workspace" + "src/jekyll-devcontainer/src/.devcontainer/devcontainer.json" +) + +EXCLUDED_DIRS=( + ".git/" + ".venv/" + "docs/" + "node_modules/" +) + + +# Get files to check based on mode +function get_files_to_check() { + local mode="$1" + case "$mode" in + staged-changes) + git diff --diff-filter=ACMRT --name-only --cached # ACMRT only show files added, copied, modified, renamed or that had their type changed (eg. file → symlink) in this commit. This leaves out deleted files. + ;; + working-tree-changes) + git ls-files --others --exclude-standard && git diff --diff-filter=ACMRT --name-only + ;; + branch) + git diff --diff-filter=ACMRT --name-only ${BRANCH_NAME:-origin/main} + ;; + all) + git ls-files && git ls-files --others --exclude-standard + ;; + *) + echo "Unknown check mode: $mode" >&2 + exit 126 + ;; + esac +} + + +function build_exclude_args() { + local args=( + --exclude=".github/actions/check-todo-usage/action.yaml" + --exclude=".github/workflows/stage-1-commit.yaml" + --exclude="scripts/config/pre-commit.yaml" + --exclude="scripts/githooks/check-todos.sh" + ) # Exclude this script and its references by default, as it naturally contains TODOs. Todo todo todo <- see? + + if [ ${#EXCLUDED_DIRS[@]} -gt 0 ]; then + for dir in "${EXCLUDED_DIRS[@]}"; do + args+=(--exclude-dir="$dir") + done + fi + + if [ ${#EXCLUDED_FILES[@]} -gt 0 ]; then + for file in "${EXCLUDED_FILES[@]}"; do + args+=(--exclude="$file") + done + fi + echo "${args[@]}" +} + + +function search_todos() { + local mode="$1" + shift # Shift positional parameters so $@ contains only exclude_args + local exclude_args=("$@") + local todos="" + + local files + files=$(get_files_to_check "$mode") + # flatten files to unique list + files=$(echo "$files" | tr ' ' '\n' | sort -u) + + for file in $files; do + skip=false + + # Check if the file matches any exclude patterns + # Exclude files based on provided arguments and predefined directories + for ex in "${exclude_args[@]}"; do + if [[ "$ex" == --exclude* ]]; then + pattern=${ex#--exclude=} + [[ "$file" == $pattern ]] && skip=true && break + fi + done + + # Check if the file is in any of the excluded directories + for exdir in "${EXCLUDED_DIRS[@]}"; do + [[ "$file" == $exdir* ]] && skip=true && break + done + + # If the file is excluded, skip it + if [ "$skip" = false ] && [ -f "$file" ]; then + file_todos=$(grep -nHiE '\bTODO\b' "$file" || true) + [ -n "$file_todos" ] && todos+="$file_todos\n" + fi + done + + echo -e "$todos" +} + + +function filter_todos_with_valid_jira_ticket() { + local todos="$1" + local jira_regex="[A-Z][A-Z0-9]+-[0-9]+" + local todos_without_ticket="" + + while IFS= read -r line; do + # Only lines with TODO but without a valid JIRA ticket + if grep -qnHiE '\bTODO\b' <<< "$line"; then + if ! [[ "$line" =~ $jira_regex ]]; then + todos_without_ticket+="$line\n" + fi + fi + done <<< "$(echo -e "$todos")" + + # Output only TODOs without a valid JIRA ticket + echo -e "$todos_without_ticket" +} + + +function print_output() { + local todos="$1" + local exclude_args="$2" + local todo_count=$(line_count "$todos") + + echo "TODO Check Configuration:" + echo "=========================================" + echo " Check Mode: ${check:-working-tree-changes}" + echo " Total TODOs found: $todo_count" + + if [ ${#EXCLUDED_DIRS[@]} -gt 0 ]; then + echo " Excluded Directories: ${EXCLUDED_DIRS[*]}" + else + echo " Excluded Directories: (none)" + fi + + if [ ${#EXCLUDED_FILES[@]} -gt 0 ]; then + echo " Excluded Files: ${EXCLUDED_FILES[*]}" + else + echo " Excluded Files: (none)" + fi + + if is-arg-true "${VERBOSE:-false}"; then + echo "Grep Exclude Args: $exclude_args" + fi + + echo -e "\n=========================================" + echo "All TODOs found: $todo_count" + echo "=========================================" + + if [ "$todo_count" -gt 0 ]; then + echo "$todos" + else + echo "No TODOs found." + fi + + local results=$(filter_todos_with_valid_jira_ticket "$todos") + local results_count=$(line_count "$results") + + echo -e "\n=========================================" + echo "TODOs without a Jira ticket: $results_count" + echo "=========================================" + + if [ "$results_count" -gt 0 ]; then + echo "$results" + exit 1 + else + echo "No TODOs found without a Jira reference." + fi +} + + +function main() { + cd "$(git rev-parse --show-toplevel)" + + local check_mode="${check:-working-tree-changes}" + local exclude_args=$(build_exclude_args) + local todos=$(search_todos "$check_mode" $exclude_args) + print_output "$todos" "$exclude_args" +} + +# ============================================================================== + +# Count non-empty lines in a string +function line_count() { + local input="$1" + if [ -n "$input" ]; then + echo -e "$input" | wc -l + else + echo 0 + fi +} + +function is-arg-true() { + if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then + return 0 + else + return 1 + fi +} + +# ============================================================================== + +is-arg-true "${VERBOSE:-false}" && set -x + +main "$@" + +exit 0 diff --git a/scripts/githooks/sort-dictionary.sh b/scripts/githooks/sort-dictionary.sh new file mode 100755 index 0000000..2fea139 --- /dev/null +++ b/scripts/githooks/sort-dictionary.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -euo pipefail + +# Pre-commit git hook to sort the Vale dictionary in a consistent manner to avoid future merge conflicts and aid insertion of new terms +# +# Usage: +# $ [options] ./sort-dictionary.sh +# +# Options: +# +# +# Exit codes: +# 0 - Successfully sorted the dictionary +# non-zero - failed to sort dictionary + +# ============================================================================== + +function main() { + root=scripts/config/vale/styles/config/vocabularies/words + opts="--dictionary-order --ignore-case -s" + sort $opts $root/accept.txt > $root/accept.sorted.txt + sort $opts $root/reject.txt > $root/reject.sorted.txt + + mv $root/accept.sorted.txt $root/accept.txt + mv $root/reject.sorted.txt $root/reject.txt + + # Update the sorted files in the staged git index + git add --update --verbose $root/* +} + +# ============================================================================== + +function is-arg-true() { + + if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then + return 0 + else + return 1 + fi +} + +# ============================================================================== + +is-arg-true "${VERBOSE:-false}" && set -x + +main "$@" + +exit 0 diff --git a/scripts/githooks/sync-template-repo.sh b/scripts/githooks/sync-template-repo.sh new file mode 100755 index 0000000..6e731db --- /dev/null +++ b/scripts/githooks/sync-template-repo.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +set -euo pipefail + +# Script to synchronise the nhs-notify-template-repository with this repository +# +# Usage: +# $ [options] ./sync-template-repo.sh +# +# Options: +# new_only=true # Only identify new files from the template-repository +# changes_only=true # Only identify files which have drifted from the template-repository + +# ============================================================================== + +scriptdir=$(realpath "$(dirname "$0")") + +# Command line parameters +new_only=${new_only:-false} +changes_only=${changes_only:-false} + +# Set variables +TEMPLATE_REPO_DIR="nhs-notify-repository-template" +IGNORE_FILE="scripts/config/.repository-template-sync-ignore" +MERGE_FILE="scripts/config/.repository-template-sync-merge" + +# Check if the template directory exists +if [ ! -d "${TEMPLATE_REPO_DIR}" ]; then + echo "Template directory ${TEMPLATE_REPO_DIR} not found!" + exit 1 +fi + +# Check if the .template-ignore file exists, create an empty one if not +if [ ! -f "${IGNORE_FILE}" ]; then + echo "# Files and folders to ignore when syncing ${TEMPLATE_REPO_DIR} back in to this repository" > ${IGNORE_FILE} + echo "# Files and Folders in this repository to ignore" >> ${IGNORE_FILE} + echo "# Files and Folders in the template repository to disregard" >> ${IGNORE_FILE} +fi + +# Check if the .template-merge file exists, create an empty one if not +if [ ! -f "${MERGE_FILE}" ]; then + echo "# Files and folders to merge when syncing ${TEMPLATE_REPO_DIR} back in to this repository" > ${MERGE_FILE} +fi + +TMP_SYNC_IGNORE=${PWD}/tmp-sync-ignore +mkdir -p "${TMP_SYNC_IGNORE}" +cp "${IGNORE_FILE}" "${TMP_SYNC_IGNORE}/.gitignore" + +TMP_SYNC_MERGE=${PWD}/tmp-sync-merge +mkdir -p "${TMP_SYNC_MERGE}" +cp "${MERGE_FILE}" "${TMP_SYNC_MERGE}/.gitignore" + +# Check if a file is ignored. +is_ignored() { + local file=${1} + + # Ignore .git directories and files + if [[ "$file" == *.git/* ]]; then + return 0 + fi + + pushd "${TMP_SYNC_IGNORE}" > /dev/null + git check-ignore -q "${file}" + R=$? + popd > /dev/null + return $R +} + +is_merge() { + local file=${1} + + pushd "${TMP_SYNC_MERGE}" > /dev/null + git check-ignore -q "${file}" + R=$? + popd > /dev/null + return $R +} + +# Navigate to the template directory +pushd "${TEMPLATE_REPO_DIR}" || exit +FILES_ADDED=() +FILES_WITH_CHANGES=() + +# Loop through all files in the template directory +while IFS= read -r -d '' file || [[ -n $file ]]; do + relative_path="${file#./}" # Remove leading './' + + # Check if the file is ignored + if is_ignored "$relative_path"; then + echo "Ignoring $relative_path" + continue + fi + + target_path="../$relative_path" + mkdir -p "$(dirname "$target_path")" + + # Copy the file to the root directory if it doesn't exist or is different + if [ ! -f "$target_path" ] && [ "$changes_only" == false ]; then + echo "Copying $relative_path to the repository" + FILES_ADDED+=("${relative_path}") + cp "$file" "$target_path" + + else + # If the file exists, check if it's different + if [ "$new_only" == false ]; then + if ! diff -q "$file" "$target_path" > /dev/null 2>&1; then + if is_merge "$relative_path"; then + echo "Merging changes from $relative_path" + cp "$target_path" "${target_path}.bak" + node "${scriptdir}/../maintenance/merge.js" "$target_path" "$file" > "${target_path}.merged" + if ! cmp -s "${target_path}.merged" "${target_path}.bak"; then + FILES_WITH_CHANGES+=("${relative_path}") + mv "${target_path}.merged" "$target_path" + fi + rm -f "${target_path}.merged" "${target_path}.bak" + else + echo "Copying changes from $relative_path" + cp "$file" "$target_path" + FILES_WITH_CHANGES+=("${relative_path}") + fi + fi + fi + fi +done < <(find . -type f -print0) + +popd +rm -rf "${TMP_SYNC_IGNORE}" "${TMP_SYNC_MERGE}" + +echo ------------------------------------------ +echo "${#FILES_ADDED[@]} files added, ${#FILES_WITH_CHANGES[@]} files with changes detected." + +if [[ "$changes_only" == false && ${#FILES_ADDED[@]} -gt 0 ]]; then + echo ------------------------------------------ + echo "New files added:" + printf ' - %s\n' "${FILES_ADDED[@]}" +fi + +if [[ "$new_only" == false && ${#FILES_WITH_CHANGES[@]} -gt 0 ]]; then + echo ------------------------------------------ + echo "Changed files:" + printf ' - %s\n' "${FILES_WITH_CHANGES[@]}" +fi + +echo ------------------------------------------ diff --git a/scripts/init.mk b/scripts/init.mk index 373f8a4..e12255c 100644 --- a/scripts/init.mk +++ b/scripts/init.mk @@ -7,7 +7,7 @@ include scripts/tests/test.mk # ============================================================================== runner-act: # Run GitHub Actions locally - mandatory: workflow=[workflow file name], job=[job name] @Development - source ./scripts/docker/docker.lib.sh + . ./scripts/docker/docker.lib.sh act $(shell [[ "${VERBOSE}" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]] && echo --verbose) \ --container-architecture linux/amd64 \ --platform ubuntu-latest=$$(name="ghcr.io/nhs-england-tools/github-runner-image" docker-get-image-version-and-pull) \ @@ -21,7 +21,7 @@ runner-act: # Run GitHub Actions locally - mandatory: workflow=[workflow file na --job ${job} version-create-effective-file: # Create effective version file - optional: dir=[path to the VERSION file to use, default is '.'], BUILD_DATETIME=[build date and time in the '%Y-%m-%dT%H:%M:%S%z' format generated by the CI/CD pipeline, default is current date and time] @Development - source scripts/docker/docker.lib.sh + . ./scripts/docker/docker.lib.sh version-create-effective-file shellscript-lint-all: # Lint all shell scripts in this project, do not fail on error, just print the error messages @Quality diff --git a/scripts/maintenance/merge.js b/scripts/maintenance/merge.js new file mode 100644 index 0000000..8f5bd6c --- /dev/null +++ b/scripts/maintenance/merge.js @@ -0,0 +1,81 @@ +/******************************************************************************* + * Script to merge all added lines from source to target. + * Modified lines will be left intact. + * + * This is intended for updating config files like .tool-versions and .gitignore + * where the first token on the line remains the same + * + * Usage: + * $ node merge.js + * + * Output: + * Outputs the merged file to stdout + * + *******************************************************************************/ + +const fs = require("fs"); + +// Read files +const [file1, file2] = process.argv.slice(2); +const lines1 = fs.readFileSync(file1).toString().split('\n'); +const lines2 = fs.readFileSync(file2).toString().split('\n'); + +// Tokenize lines in file1 for later comparison +const tokenize = line => { + if (line === '') { + return []; + } + return line.split(/\s+|[:=]/).filter(x => x !== '#' && x !== '').slice(0, 1); +}; +const lines1Tokens = lines1.flatMap(tokenize); + +// Step through the files +let pos1 = 0, pos2 = 0; +while (pos1 < lines1.length || pos2 < lines2.length) { + + const l1 = pos1 < lines1.length && lines1[pos1] || ''; + const l2 = pos2 < lines2.length && lines2[pos2] || ''; + + // If the lines match, print l1 and skip l2 + if (l1 !== '' && l1 === l2) { + process.stdout.write(`${l1}\n`); + pos1++; + pos2++; + continue; + } + + if (pos2 < lines2.length) { + // If l2 is empty, skip l2 + if (l2 === '') { + pos2++; + continue; + } + + const [l2token] = tokenize(l2); + + // If l2 token matches l1 token, print l1 + if (pos1 < lines1.length && lines1Tokens[pos1] === l2token) { + process.stdout.write(`${l1}\n`); + pos1++; + pos2++; + continue; + } + + // If l2 doesn't match any lines in file1, print l2 + if (l2token && !lines1Tokens.includes(l2token)) { + process.stdout.write(`${l2}\n`); + } + pos2++ + continue; + } + + // If we're not at the end of file1, print l1 + if (pos1 < lines1.length) { + if (pos1 === lines1.length - 1 && l1 === '') { + // Don't print tailing newline + } else { + process.stdout.write(`${l1}\n`); + } + pos1++; + } +} From 46cc15a383f64bf5085be98b295251c3a47884ff Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Wed, 23 Jul 2025 16:40:15 +0100 Subject: [PATCH 2/5] CCM-11345 Manual Repo Sync --- .github/SECURITY.md | 4 ++-- .github/workflows/stage-1-commit.yaml | 2 +- LICENCE.md | 2 +- scripts/config/pre-commit.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 241f1e3..0382fd5 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -21,8 +21,8 @@ If you wish to notify us of a vulnerability via email, please include detailed i You can reach us at: -- _[ A product team email address ]_ -- [cybersecurity@nhs.net](cybersecurity@nhs.net) +- [england.nhsnotify@nhs.net](mailto:england.nhsnotify@nhs.net) +- [cybersecurity@nhs.net](mailto:cybersecurity@nhs.net) ### NCSC diff --git a/.github/workflows/stage-1-commit.yaml b/.github/workflows/stage-1-commit.yaml index e83ee95..811a15e 100644 --- a/.github/workflows/stage-1-commit.yaml +++ b/.github/workflows/stage-1-commit.yaml @@ -156,7 +156,7 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 - name: "Setup ASDF" - uses: asdf-vm/actions/setup@v3 + uses: asdf-vm/actions/setup@v4 - name: "Perform Setup" uses: ./.github/actions/setup - name: "Trivy Scan" diff --git a/LICENCE.md b/LICENCE.md index ed56eb2..02174c4 100644 --- a/LICENCE.md +++ b/LICENCE.md @@ -1,6 +1,6 @@ # MIT Licence -Copyright (c) 2024 Crown Copyright NHS England. +Copyright (c) 2025 Crown Copyright NHS England. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index 8843370..29397e2 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 # Use the ref you want to point at + rev: v5.0.0 # Use the ref you want to point at hooks: - id: trailing-whitespace - id: detect-aws-credentials From 75f316350cd29a36b0e76941e5c5352069c42b59 Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Thu, 24 Jul 2025 16:46:13 +0100 Subject: [PATCH 3/5] CCM-11345 Manual Repo Sync --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 03bf171..091fd47 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9 + uses: github/codeql-action/upload-sarif@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 with: sarif_file: results.sarif From f91a513d8846f3584a26fe7a41aa96a7a12d6fbe Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Fri, 25 Jul 2025 09:27:33 +0100 Subject: [PATCH 4/5] CCM-11345 Manual Repo Sync --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 34d9b8a..95cdb2d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,7 +1,7 @@ act 0.2.64 gitleaks 8.24.0 jq 1.6 -nodejs 22.11.0 +nodejs 22.15.1 pre-commit 3.6.0 terraform 1.9.2 terraform-docs 0.19.0 From df5b8f633b986194aa8027a6bfbca8ff51557a6b Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Fri, 25 Jul 2025 09:29:38 +0100 Subject: [PATCH 5/5] CCM-11345 Manual Repo Sync --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 091fd47..5552785 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -37,7 +37,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 with: results_file: results.sarif results_format: sarif