From db8e0472559a23af3936656001da790d7943b0e8 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 15 Sep 2021 09:02:56 -0700 Subject: [PATCH 01/64] changes to project-1 and project-3 --- project-1/add.js | 2 +- project-3/index.d.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project-1/add.js b/project-1/add.js index 1e691765..f1093b05 100644 --- a/project-1/add.js +++ b/project-1/add.js @@ -1,5 +1,5 @@ import createMathOperation from './.internal/createMathOperation.js' - +//changes for testing /** * Adds two numbers. * diff --git a/project-3/index.d.ts b/project-3/index.d.ts index ddaf8d46..7061e41f 100644 --- a/project-3/index.d.ts +++ b/project-3/index.d.ts @@ -1,5 +1,5 @@ /** -Basic foreground colors. +Basic foreground colors. changes for testing [More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) */ From 31d6ceca8d85317db0857f2a31dd5a990a1b7947 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 15 Sep 2021 09:32:34 -0700 Subject: [PATCH 02/64] new changes to detect --- .github/scripts/list-dirs | 18 ++++++------------ .github/scripts/old-list-dirs | 16 ++++++++++++++++ project with spaces/app with spaces.js | 1 + 3 files changed, 23 insertions(+), 12 deletions(-) create mode 100755 .github/scripts/old-list-dirs create mode 100644 project with spaces/app with spaces.js diff --git a/.github/scripts/list-dirs b/.github/scripts/list-dirs index 3c530b53..ecc3e7c3 100755 --- a/.github/scripts/list-dirs +++ b/.github/scripts/list-dirs @@ -1,15 +1,9 @@ -#!/usr/bin/env python3 +#!/usr/bin/env bash -# -# This script prints a JSON array containing all non-hidden subdirectories of -# the current working directory. As an example, if the current working -# directory contains the subdirectories "foo", "bar" and "baz", the output -# will be (the order of the directories is not necessarily alphabetical): -# -# ["foo", "bar", "baz"] -# +base=origin/main +commit_sha=db8e0472559a23af3936656001da790d7943b0e8 -import glob -import json +filelist=`git diff --name-only $base $commit_sha` +array=($filelist) -print(json.dumps(glob.glob("*/")).replace("/", "")) \ No newline at end of file +echo ${array[@]} diff --git a/.github/scripts/old-list-dirs b/.github/scripts/old-list-dirs new file mode 100755 index 00000000..36ff954b --- /dev/null +++ b/.github/scripts/old-list-dirs @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +# +# This script prints a JSON array containing all non-hidden subdirectories of +# the current working directory. As an example, if the current working +# directory contains the subdirectories "foo", "bar" and "baz", the output +# will be (the order of the directories is not necessarily alphabetical): +# +# ["foo", "bar", "baz"] +# + +import glob +import json + +#print(json.dumps(glob.glob("*/")).replace("/", "")) +print(glob.glob("*/")) \ No newline at end of file diff --git a/project with spaces/app with spaces.js b/project with spaces/app with spaces.js new file mode 100644 index 00000000..0f504264 --- /dev/null +++ b/project with spaces/app with spaces.js @@ -0,0 +1 @@ +console.log("hello world"); \ No newline at end of file From 4379617972273fac1dc68bced4b9167832ae499a Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 16 Sep 2021 16:55:08 -0700 Subject: [PATCH 03/64] only run scan on top level dirs with changes --- .github/scripts/list-dirs | 22 ++++++++++++++++------ .github/scripts/old-list-dirs | 16 ---------------- .github/workflows/code-scanning.yml | 1 + 3 files changed, 17 insertions(+), 22 deletions(-) delete mode 100755 .github/scripts/old-list-dirs diff --git a/.github/scripts/list-dirs b/.github/scripts/list-dirs index ecc3e7c3..448c3dfc 100755 --- a/.github/scripts/list-dirs +++ b/.github/scripts/list-dirs @@ -1,9 +1,19 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 -base=origin/main -commit_sha=db8e0472559a23af3936656001da790d7943b0e8 +# +# This script prints a JSON array containing all non-hidden subdirectories of +# the current working directory. As an example, if the current working +# directory contains the subdirectories "foo", "bar" and "baz", the output +# will be (the order of the directories is not necessarily alphabetical): +# +# ["foo", "bar", "baz"] +# +import json -filelist=`git diff --name-only $base $commit_sha` -array=($filelist) +lines = list(open('diff.txt').readlines()) +outlines = set() -echo ${array[@]} +for line in lines: + outlines.add(line.split('/')[0]) + +print(json.dumps(list(outlines))) diff --git a/.github/scripts/old-list-dirs b/.github/scripts/old-list-dirs deleted file mode 100755 index 36ff954b..00000000 --- a/.github/scripts/old-list-dirs +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 - -# -# This script prints a JSON array containing all non-hidden subdirectories of -# the current working directory. As an example, if the current working -# directory contains the subdirectories "foo", "bar" and "baz", the output -# will be (the order of the directories is not necessarily alphabetical): -# -# ["foo", "bar", "baz"] -# - -import glob -import json - -#print(json.dumps(glob.glob("*/")).replace("/", "")) -print(glob.glob("*/")) \ No newline at end of file diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 4048ae1e..f7df11d0 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -41,6 +41,7 @@ jobs: - name: Find existing directories id: find-dirs run: | + git diff --name-only $base $GITHUB_SHA >diff.txt echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" codeql: From 06ab50d3fc33e32bac58a3fbcaa89890e138f762 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 16 Sep 2021 17:04:17 -0700 Subject: [PATCH 04/64] put diff.txt in right place --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index f7df11d0..10e3c3f1 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -41,7 +41,7 @@ jobs: - name: Find existing directories id: find-dirs run: | - git diff --name-only $base $GITHUB_SHA >diff.txt + git diff --name-only $base $GITHUB_SHA >./.github/scripts/diff.txt echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" codeql: From 1db87fee1ee45ff4734e6d87e753a45c64506fd1 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 16 Sep 2021 17:10:41 -0700 Subject: [PATCH 05/64] Get diff.txt from right place --- .github/scripts/list-dirs | 2 +- .gitignore | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.github/scripts/list-dirs b/.github/scripts/list-dirs index 448c3dfc..2e86ad44 100755 --- a/.github/scripts/list-dirs +++ b/.github/scripts/list-dirs @@ -10,7 +10,7 @@ # import json -lines = list(open('diff.txt').readlines()) +lines = list(open('./.github/scripts/diff.txt').readlines()) outlines = set() for line in lines: diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..fac6d138 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.github/scripts/diff.txt From 54acbe88fdc424866514fba2836ac7a335025ac0 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 16 Sep 2021 17:14:25 -0700 Subject: [PATCH 06/64] remove target-dir exclude --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 10e3c3f1..fb717cb3 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -55,8 +55,8 @@ jobs: # Prevent the creation of jobs for directories where code scanning is # not necessary/desired. # - exclude: - - target-dir: docs + #exclude: + #- target-dir: docs steps: - name: Checkout repository From 0b9e65543951996fce9366703bdd37c9ce6b076c Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 16 Sep 2021 17:16:08 -0700 Subject: [PATCH 07/64] right git diff command --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index fb717cb3..0ca1ad20 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -41,7 +41,7 @@ jobs: - name: Find existing directories id: find-dirs run: | - git diff --name-only $base $GITHUB_SHA >./.github/scripts/diff.txt + git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" codeql: From 0a5eb101f4da2ff0e32624e892f87ef6841b8097 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 16 Sep 2021 17:17:28 -0700 Subject: [PATCH 08/64] is checkout wrong? --- .github/workflows/code-scanning.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 0ca1ad20..24f47e09 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -29,6 +29,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 + with: + fetch-depth: 0 # # Generate a JSON array containing all non-hidden subdirectories of the From 5ad63b6dd8b3ec770c10eb0b008a8cf0209be2d4 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 07:39:42 -0700 Subject: [PATCH 09/64] Ensure only directories are listed --- .github/scripts/list-dirs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/list-dirs b/.github/scripts/list-dirs index 2e86ad44..d51ee957 100755 --- a/.github/scripts/list-dirs +++ b/.github/scripts/list-dirs @@ -8,12 +8,17 @@ # # ["foo", "bar", "baz"] # +from genericpath import isdir import json +import os lines = list(open('./.github/scripts/diff.txt').readlines()) outlines = set() +#only add items that are directories for line in lines: - outlines.add(line.split('/')[0]) + path = line.split('/')[0] + if (os.path.isdir(path)): + outlines.add(path) print(json.dumps(list(outlines))) From d1216cfd6e5e20cda1505c50c57540bf047bbba3 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 07:45:42 -0700 Subject: [PATCH 10/64] Changed 3 dirs and a file --- .gitignore | 1 + project with spaces/app with spaces.js | 2 +- project-1/add.js | 1 - project-3/index.d.ts | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index fac6d138..29127430 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .github/scripts/diff.txt +**/node_modules/** diff --git a/project with spaces/app with spaces.js b/project with spaces/app with spaces.js index 0f504264..7df7d571 100644 --- a/project with spaces/app with spaces.js +++ b/project with spaces/app with spaces.js @@ -1 +1 @@ -console.log("hello world"); \ No newline at end of file +console.log("Hello World"); \ No newline at end of file diff --git a/project-1/add.js b/project-1/add.js index f1093b05..f1993904 100644 --- a/project-1/add.js +++ b/project-1/add.js @@ -1,5 +1,4 @@ import createMathOperation from './.internal/createMathOperation.js' -//changes for testing /** * Adds two numbers. * diff --git a/project-3/index.d.ts b/project-3/index.d.ts index 7061e41f..ddaf8d46 100644 --- a/project-3/index.d.ts +++ b/project-3/index.d.ts @@ -1,5 +1,5 @@ /** -Basic foreground colors. changes for testing +Basic foreground colors. [More colors here.](https://github.com/chalk/chalk/blob/main/readme.md#256-and-truecolor-color-support) */ From 1396c7f4a1432ff2a77242a39477d4914dc680fa Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 09:16:12 -0700 Subject: [PATCH 11/64] one code scan workflow for all one for changes --- .github/workflows/code-scanning-changes.yml | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 .github/workflows/code-scanning-changes.yml diff --git a/.github/workflows/code-scanning-changes.yml b/.github/workflows/code-scanning-changes.yml new file mode 100644 index 00000000..a58dbf5b --- /dev/null +++ b/.github/workflows/code-scanning-changes.yml @@ -0,0 +1,88 @@ +name: Code scanning for changed apps + +# +# Scan the code using CodeQL whenever new commits are pushed to the main branch +# or whenever any activity happens inside a pull request targeting the main +# branch. This workflow will only be executed if at least one modified file +# resides outside the "docs" subdirectory and is not directly under the +# repository's root directory. +# +on: + push: + branches: [main] + paths-ignore: + - 'docs/**' + - '*' + pull_request: + branches: [main] + paths-ignore: + - 'docs/**' + - '*' + +jobs: + generate-dir-list: + name: Generate directory list + runs-on: ubuntu-latest + outputs: + dir-list: ${{steps.find-dirs.outputs.dir-list}} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # + # Generate a JSON array containing all non-hidden subdirectories of the + # repository's root directory and store it as a job output so it can be + # consumed by all downstream jobs depending on this one. For more + # information about this, visit: + # + # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs + # + - name: Find existing directories + id: find-dirs + run: | + git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt + echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" + + codeql: + name: Scan code with CodeQL + needs: generate-dir-list + runs-on: ubuntu-latest + strategy: + matrix: + target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} + # + # Prevent the creation of jobs for directories where code scanning is + # not necessary/desired. + # + #exclude: + #- target-dir: docs + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # + # Build the configuration file for CodeQL to instruct it to only scan the + # contents of a single directory. For more information about the + # configuration options available, visit: + # + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#using-a-custom-configuration-file + # + - name: Build CodeQL config file + env: + TARGET_DIR: ${{matrix.target-dir}} + run: | + cp .github/codeql/codeql-config-template.yml codeql-config.yml + sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + config-file: codeql-config.yml + languages: javascript + + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@v1 \ No newline at end of file From 1b9ca6fa1520447e5eb8799e9fa33179cf8c55bb Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 09:18:23 -0700 Subject: [PATCH 12/64] scan only changes on pr, scan all on push, schedule --- .github/scripts/list-changed-dirs | 24 +++++++++++++++++++++ .github/scripts/list-dirs | 15 +++---------- .github/workflows/code-scanning-changes.yml | 5 ----- .github/workflows/code-scanning.yml | 12 +++++------ 4 files changed, 32 insertions(+), 24 deletions(-) create mode 100755 .github/scripts/list-changed-dirs diff --git a/.github/scripts/list-changed-dirs b/.github/scripts/list-changed-dirs new file mode 100755 index 00000000..d51ee957 --- /dev/null +++ b/.github/scripts/list-changed-dirs @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# +# This script prints a JSON array containing all non-hidden subdirectories of +# the current working directory. As an example, if the current working +# directory contains the subdirectories "foo", "bar" and "baz", the output +# will be (the order of the directories is not necessarily alphabetical): +# +# ["foo", "bar", "baz"] +# +from genericpath import isdir +import json +import os + +lines = list(open('./.github/scripts/diff.txt').readlines()) +outlines = set() + +#only add items that are directories +for line in lines: + path = line.split('/')[0] + if (os.path.isdir(path)): + outlines.add(path) + +print(json.dumps(list(outlines))) diff --git a/.github/scripts/list-dirs b/.github/scripts/list-dirs index d51ee957..1210d0f9 100755 --- a/.github/scripts/list-dirs +++ b/.github/scripts/list-dirs @@ -8,17 +8,8 @@ # # ["foo", "bar", "baz"] # -from genericpath import isdir -import json -import os - -lines = list(open('./.github/scripts/diff.txt').readlines()) -outlines = set() -#only add items that are directories -for line in lines: - path = line.split('/')[0] - if (os.path.isdir(path)): - outlines.add(path) +import glob +import json -print(json.dumps(list(outlines))) +print(json.dumps(glob.glob("*/")).replace("/", "")) diff --git a/.github/workflows/code-scanning-changes.yml b/.github/workflows/code-scanning-changes.yml index a58dbf5b..99d90565 100644 --- a/.github/workflows/code-scanning-changes.yml +++ b/.github/workflows/code-scanning-changes.yml @@ -8,11 +8,6 @@ name: Code scanning for changed apps # repository's root directory. # on: - push: - branches: [main] - paths-ignore: - - 'docs/**' - - '*' pull_request: branches: [main] paths-ignore: diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 24f47e09..2864d621 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -1,4 +1,4 @@ -name: Code scanning +name: Code scanning for all apps # # Scan the code using CodeQL whenever new commits are pushed to the main branch @@ -13,11 +13,10 @@ on: paths-ignore: - 'docs/**' - '*' - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - '*' + schedule: + - cron: "35 13 * * 2" + workflow_dispatch: + jobs: generate-dir-list: @@ -43,7 +42,6 @@ jobs: - name: Find existing directories id: find-dirs run: | - git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" codeql: From 7d1ed23f1027880b57f3875aa66eb7ad70fb2013 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 09:27:45 -0700 Subject: [PATCH 13/64] do it all in one workflow file --- .github/workflows/code-scanning-changes.yml | 83 --------------------- .github/workflows/code-scanning.yml | 31 ++++++-- 2 files changed, 23 insertions(+), 91 deletions(-) delete mode 100644 .github/workflows/code-scanning-changes.yml diff --git a/.github/workflows/code-scanning-changes.yml b/.github/workflows/code-scanning-changes.yml deleted file mode 100644 index 99d90565..00000000 --- a/.github/workflows/code-scanning-changes.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Code scanning for changed apps - -# -# Scan the code using CodeQL whenever new commits are pushed to the main branch -# or whenever any activity happens inside a pull request targeting the main -# branch. This workflow will only be executed if at least one modified file -# resides outside the "docs" subdirectory and is not directly under the -# repository's root directory. -# -on: - pull_request: - branches: [main] - paths-ignore: - - 'docs/**' - - '*' - -jobs: - generate-dir-list: - name: Generate directory list - runs-on: ubuntu-latest - outputs: - dir-list: ${{steps.find-dirs.outputs.dir-list}} - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - # - # Generate a JSON array containing all non-hidden subdirectories of the - # repository's root directory and store it as a job output so it can be - # consumed by all downstream jobs depending on this one. For more - # information about this, visit: - # - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs - # - - name: Find existing directories - id: find-dirs - run: | - git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt - echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" - - codeql: - name: Scan code with CodeQL - needs: generate-dir-list - runs-on: ubuntu-latest - strategy: - matrix: - target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} - # - # Prevent the creation of jobs for directories where code scanning is - # not necessary/desired. - # - #exclude: - #- target-dir: docs - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # - # Build the configuration file for CodeQL to instruct it to only scan the - # contents of a single directory. For more information about the - # configuration options available, visit: - # - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#using-a-custom-configuration-file - # - - name: Build CodeQL config file - env: - TARGET_DIR: ${{matrix.target-dir}} - run: | - cp .github/codeql/codeql-config-template.yml codeql-config.yml - sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - config-file: codeql-config.yml - languages: javascript - - - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v1 \ No newline at end of file diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 2864d621..5c5e09ab 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -13,6 +13,11 @@ on: paths-ignore: - 'docs/**' - '*' + pull_request: + branches: [main] + paths-ignore: + - 'docs/**' + - '*' schedule: - cron: "35 13 * * 2" workflow_dispatch: @@ -26,24 +31,34 @@ jobs: dir-list: ${{steps.find-dirs.outputs.dir-list}} steps: + if: ${{ github.event_name != 'pull_request'}} + - name: Checkout repository uses: actions/checkout@v2 with: fetch-depth: 0 - # - # Generate a JSON array containing all non-hidden subdirectories of the - # repository's root directory and store it as a job output so it can be - # consumed by all downstream jobs depending on this one. For more - # information about this, visit: - # - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs - # - name: Find existing directories id: find-dirs run: | echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" + steps: + if: ${{ github.event_name == 'pull_request'}} + + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Find changed directories + id: find-dirs + run: | + git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt + echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" + + + codeql: name: Scan code with CodeQL needs: generate-dir-list From 8c058e4a065bce0548e93a347236859cfe099d15 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 09:28:40 -0700 Subject: [PATCH 14/64] fix yaml syntax error --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 5c5e09ab..e25ad356 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -31,7 +31,7 @@ jobs: dir-list: ${{steps.find-dirs.outputs.dir-list}} steps: - if: ${{ github.event_name != 'pull_request'}} + if: ${{ github.event_name != 'pull_request'}} - name: Checkout repository uses: actions/checkout@v2 @@ -44,7 +44,7 @@ jobs: echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" steps: - if: ${{ github.event_name == 'pull_request'}} + if: ${{ github.event_name == 'pull_request'}} - name: Checkout repository uses: actions/checkout@v2 From 34f1e5372c5e1bbf90b22e73d299778167833bee Mon Sep 17 00:00:00 2001 From: Dave <50186003+thedave42@users.noreply.github.com> Date: Fri, 17 Sep 2021 09:35:39 -0700 Subject: [PATCH 15/64] fixes for running all vs only changes --- .github/workflows/code-scanning.yml | 60 ++++++++++++++++------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index e25ad356..67d8995f 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -25,39 +25,47 @@ on: jobs: generate-dir-list: + # Find all the top level directories in the repostiory and use them for the scan + # when the workflow is not triggered by a pull_request + # + if: ${{ github.event_name != 'pull_request'}} name: Generate directory list runs-on: ubuntu-latest outputs: dir-list: ${{steps.find-dirs.outputs.dir-list}} steps: - if: ${{ github.event_name != 'pull_request'}} - - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Find existing directories - id: find-dirs - run: | - echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Find existing directories + id: find-dirs + run: | + echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" + + generate-changed-dir-list: + # Find only the changed top level directories and use them for the scan + # when the workflow is triggered by a pull_request + # + if: ${{ github.event_name == 'pull_request'}} + name: Generate changed directory list + runs-on: ubuntu-latest + outputs: + dir-list: ${{steps.find-dirs.outputs.dir-list}} steps: - if: ${{ github.event_name == 'pull_request'}} - - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Find changed directories - id: find-dirs - run: | - git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt - echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" - - + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Find changed directories + id: find-dirs + run: | + git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt + echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" codeql: name: Scan code with CodeQL @@ -98,4 +106,4 @@ jobs: languages: javascript - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v1 \ No newline at end of file + uses: github/codeql-action/analyze@v1 From cd7394bedb0ea7cd2db2b29c23ba61ad62abb424 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 13:44:28 -0700 Subject: [PATCH 16/64] separate jobs for changed vs all --- .github/workflows/code-scanning.yml | 43 ++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 67d8995f..9f690445 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -67,8 +67,8 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" - codeql: - name: Scan code with CodeQL + codeql-all-dirs: + name: Scan all directories with CodeQL needs: generate-dir-list runs-on: ubuntu-latest strategy: @@ -78,8 +78,43 @@ jobs: # Prevent the creation of jobs for directories where code scanning is # not necessary/desired. # - #exclude: - #- target-dir: docs + exclude: + - target-dir: docs + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # + # Build the configuration file for CodeQL to instruct it to only scan the + # contents of a single directory. For more information about the + # configuration options available, visit: + # + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#using-a-custom-configuration-file + # + - name: Build CodeQL config file + env: + TARGET_DIR: ${{matrix.target-dir}} + run: | + cp .github/codeql/codeql-config-template.yml codeql-config.yml + sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + config-file: codeql-config.yml + languages: javascript + + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@v1 + + codeql-changed-dirs: + name: Scan all directories with CodeQL + needs: generate-changed-dir-list + runs-on: ubuntu-latest + strategy: + matrix: + target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} steps: - name: Checkout repository From a563fbcd682b2c5f90f3a95a293598bf57869c49 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 13:48:34 -0700 Subject: [PATCH 17/64] upload diff as workflow artifact --- .github/workflows/code-scanning.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 9f690445..be5e31b3 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -67,6 +67,14 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" + - name: Upload diff as artifact + uses: actions/upload-artifact@v2 + with: + name: diff + path: | + ./.github/scripts/diff.txt + + codeql-all-dirs: name: Scan all directories with CodeQL needs: generate-dir-list From ebb0834954ffb6e2f7b48321041dc09c85e81b71 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 13:51:28 -0700 Subject: [PATCH 18/64] fix yaml --- .github/workflows/code-scanning.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index be5e31b3..20b9a6cc 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -72,7 +72,8 @@ jobs: with: name: diff path: | - ./.github/scripts/diff.txt + ./.github/scripts/diff.txt + codeql-all-dirs: @@ -122,7 +123,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} + target-dir: ${{fromJson(needs.generate-changed-dir-list.outputs.dir-list)}} steps: - name: Checkout repository From 6b04ec505bd8d9129073ba19045bb393a40246eb Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Fri, 17 Sep 2021 14:20:22 -0700 Subject: [PATCH 19/64] do not stop other matrix jobs if one fails --- .github/workflows/code-scanning.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 20b9a6cc..52c92baf 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -81,8 +81,10 @@ jobs: needs: generate-dir-list runs-on: ubuntu-latest strategy: + fail-fast: false matrix: target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} + # # Prevent the creation of jobs for directories where code scanning is # not necessary/desired. @@ -122,6 +124,7 @@ jobs: needs: generate-changed-dir-list runs-on: ubuntu-latest strategy: + fail-fast: false matrix: target-dir: ${{fromJson(needs.generate-changed-dir-list.outputs.dir-list)}} From 59c6107b401d32c1e3342ea394fe711ef05823a8 Mon Sep 17 00:00:00 2001 From: Dave <50186003+thedave42@users.noreply.github.com> Date: Fri, 17 Sep 2021 15:36:27 -0700 Subject: [PATCH 20/64] Update README.md update readme to include diff info --- README.md | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ff796aee..fb0c232f 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,11 @@ this repository (e.g. `project-4`) requires no changes to the workflow file as a dedicated code scanning job will be automatically generated for it when the workflow is executed. +If the workflow is triggered by a pull request the list of sub-directories that +will be scanned will be limited to the subdirectories that contain changes. The +changes are based on a `git diff` between the base and head repositories specified +in the pull request. + This strategy is possible because GitHub Actions workflows accept JSON input to define a job matrix, and the JSON contents can be generated during the workflow's execution. In other words, the job matrix can be defined dynamically. @@ -33,20 +38,7 @@ general capabilities of CodeQL before doing this. ## Answers to common questions -**1.** _Even if files in only one subdirectory in the repository are changed, -code scanning jobs will be generated for all subdirectories containing software -projects, which is wasteful. Is it possible to limit the generation of jobs so -that only subdirectories with modified files will be scanned?_ - -Yes. The list of subdirectories which is used as input for the code scanning job -matrix is produced by a [script](./.github/scripts/list-dirs) which simply -outputs all subdirectories under the repository's root directory. This script -can be modified in any way you want, so you can use [`git -diff`](https://stackoverflow.com/questions/50440420/git-diff-only-show-which-directories-changed) -to build a list containing only subdirectories with modified files and use that -list as input for the job matrix generation. - -**2.** _Every code scanning job checks out the repository in parallel. If a +**1.** _Every code scanning job checks out the repository in parallel. If a change is made to the repository during that time (e.g. a subdirectory is added or removed, or a file in a pre-existing subdirectory is modified), you essentially have a race condition which is not being properly handled._ @@ -63,4 +55,4 @@ very first job which is executed in the workflow and then consuming that artifact in all downstream jobs. The [`actions/upload-artifact`](https://github.com/actions/upload-artifact) and [`actions/download-artifact`](https://github.com/actions/download-artifact) -actions will help you accomplish this. \ No newline at end of file +actions will help you accomplish this. From 53931eba58203ac1745ab4a61bb337234f9da3df Mon Sep 17 00:00:00 2001 From: Dave <50186003+thedave42@users.noreply.github.com> Date: Thu, 2 Jun 2022 10:14:45 -0700 Subject: [PATCH 21/64] Update code-scanning.yml --- .github/workflows/code-scanning.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 52c92baf..6a8e8143 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -57,7 +57,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -94,7 +94,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # # Build the configuration file for CodeQL to instruct it to only scan the @@ -111,16 +111,16 @@ jobs: sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml languages: javascript - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 codeql-changed-dirs: - name: Scan all directories with CodeQL + name: Scan changed directories with CodeQL needs: generate-changed-dir-list runs-on: ubuntu-latest strategy: @@ -130,7 +130,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # # Build the configuration file for CodeQL to instruct it to only scan the @@ -147,10 +147,10 @@ jobs: sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml languages: javascript - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From cc15c307fb34e496ea0e4cffb03657547cca18d8 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 12:19:59 -0700 Subject: [PATCH 22/64] test what happens with a language matrix --- .github/workflows/code-scanning.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 6a8e8143..8d4a4434 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -84,6 +84,7 @@ jobs: fail-fast: false matrix: target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} + languages: ["javascript", "ruby"] # # Prevent the creation of jobs for directories where code scanning is @@ -114,7 +115,7 @@ jobs: uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml - languages: javascript + languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 From 50ab9f475bfc4bda402eace3fe5c76b9c29881f0 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 12:32:36 -0700 Subject: [PATCH 23/64] add script to list langauges found in diff.txt --- .github/scripts/list-changed-langs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 .github/scripts/list-changed-langs diff --git a/.github/scripts/list-changed-langs b/.github/scripts/list-changed-langs new file mode 100755 index 00000000..533feacc --- /dev/null +++ b/.github/scripts/list-changed-langs @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +# +# This script prints a JSON array containing all non-hidden subdirectories of +# the current working directory. As an example, if the current working +# directory contains the subdirectories "foo", "bar" and "baz", the output +# will be (the order of the directories is not necessarily alphabetical): +# +# ["foo", "bar", "baz"] +# +import json + +lines = list(open('./.github/scripts/diff.txt').readlines()) +outlines = set() + +#only add items that are directories +for line in lines: + if line.find(".js") != -1 or line.find(".ts") != -1: + outlines.add("javascript") + + +print(json.dumps(list(outlines))) From bd7ed6acb5277c83da34bea4bfc6d22ec2c2c2ea Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 17:15:40 -0700 Subject: [PATCH 24/64] remove all workflow referece to language config --- .github/scripts/list-changed-langs | 31 +++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/scripts/list-changed-langs b/.github/scripts/list-changed-langs index 533feacc..544d049f 100755 --- a/.github/scripts/list-changed-langs +++ b/.github/scripts/list-changed-langs @@ -1,22 +1,39 @@ #!/usr/bin/env python3 - # -# This script prints a JSON array containing all non-hidden subdirectories of -# the current working directory. As an example, if the current working -# directory contains the subdirectories "foo", "bar" and "baz", the output -# will be (the order of the directories is not necessarily alphabetical): +# This script prints a JSON array containing all the supported CodeQL programming languages based on the file extension # # ["foo", "bar", "baz"] # import json -lines = list(open('./.github/scripts/diff.txt').readlines()) +javascript = [".js", ".jsx", ".mjs", ".es", ".es6", ".htm", ".html", ".xhtm", ".xhtml", ".vue", ".hbs", ".ejs", ".njk", ".json", ".yaml", ".yml", ".raml", ".xml"] +typescript = [".ts", ".tsx", ".mts", ".cts"] +c_and_cplus = [".cpp", ".c++", ".cxx", ".hpp", ".hh", ".h++", ".hxx", ".c," ".cc", ".h"] +csharp = [".sln", ".csproj", ".cs", ".cshtml", ".xaml"] +golang = [".go"] +python_lang = [".py"] +java = [".java"] +ruby = [".rb", ".erb", ".gemspec", "Gemfile"] + + +lines = list(open("./.github/scripts/diff.txt").readlines()) outlines = set() +def find_in_list(list, string): + for item in list: + if item in string: + return True + return False + #only add items that are directories for line in lines: - if line.find(".js") != -1 or line.find(".ts") != -1: + if find_in_list(javascript, line): outlines.add("javascript") + if find_in_list(typescript, line): + outlines.add("javascript") + if find_in_list(javascript, line): + outlines.add("javascript") + print(json.dumps(list(outlines))) From 645edcc3aeea72baed5028cf812b2575eb2a9fd9 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 17:16:00 -0700 Subject: [PATCH 25/64] remove all reference to workflow config --- .github/workflows/code-scanning.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 8d4a4434..d2f7fc63 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -84,8 +84,6 @@ jobs: fail-fast: false matrix: target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} - languages: ["javascript", "ruby"] - # # Prevent the creation of jobs for directories where code scanning is # not necessary/desired. @@ -115,7 +113,6 @@ jobs: uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml - languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 @@ -151,7 +148,6 @@ jobs: uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml - languages: javascript - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 From b62b541359385369d44f9fb0587df58b22674821 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 17:17:56 -0700 Subject: [PATCH 26/64] See if it picks up python --- python-project/list-changed-dirs | 24 +++++++++++++++++++ python-project/list-changed-langs | 39 +++++++++++++++++++++++++++++++ python-project/list-dirs | 15 ++++++++++++ 3 files changed, 78 insertions(+) create mode 100755 python-project/list-changed-dirs create mode 100755 python-project/list-changed-langs create mode 100755 python-project/list-dirs diff --git a/python-project/list-changed-dirs b/python-project/list-changed-dirs new file mode 100755 index 00000000..d51ee957 --- /dev/null +++ b/python-project/list-changed-dirs @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# +# This script prints a JSON array containing all non-hidden subdirectories of +# the current working directory. As an example, if the current working +# directory contains the subdirectories "foo", "bar" and "baz", the output +# will be (the order of the directories is not necessarily alphabetical): +# +# ["foo", "bar", "baz"] +# +from genericpath import isdir +import json +import os + +lines = list(open('./.github/scripts/diff.txt').readlines()) +outlines = set() + +#only add items that are directories +for line in lines: + path = line.split('/')[0] + if (os.path.isdir(path)): + outlines.add(path) + +print(json.dumps(list(outlines))) diff --git a/python-project/list-changed-langs b/python-project/list-changed-langs new file mode 100755 index 00000000..544d049f --- /dev/null +++ b/python-project/list-changed-langs @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# +# This script prints a JSON array containing all the supported CodeQL programming languages based on the file extension +# +# ["foo", "bar", "baz"] +# +import json + +javascript = [".js", ".jsx", ".mjs", ".es", ".es6", ".htm", ".html", ".xhtm", ".xhtml", ".vue", ".hbs", ".ejs", ".njk", ".json", ".yaml", ".yml", ".raml", ".xml"] +typescript = [".ts", ".tsx", ".mts", ".cts"] +c_and_cplus = [".cpp", ".c++", ".cxx", ".hpp", ".hh", ".h++", ".hxx", ".c," ".cc", ".h"] +csharp = [".sln", ".csproj", ".cs", ".cshtml", ".xaml"] +golang = [".go"] +python_lang = [".py"] +java = [".java"] +ruby = [".rb", ".erb", ".gemspec", "Gemfile"] + + +lines = list(open("./.github/scripts/diff.txt").readlines()) +outlines = set() + +def find_in_list(list, string): + for item in list: + if item in string: + return True + return False + +#only add items that are directories +for line in lines: + if find_in_list(javascript, line): + outlines.add("javascript") + if find_in_list(typescript, line): + outlines.add("javascript") + if find_in_list(javascript, line): + outlines.add("javascript") + + + +print(json.dumps(list(outlines))) diff --git a/python-project/list-dirs b/python-project/list-dirs new file mode 100755 index 00000000..1210d0f9 --- /dev/null +++ b/python-project/list-dirs @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +# +# This script prints a JSON array containing all non-hidden subdirectories of +# the current working directory. As an example, if the current working +# directory contains the subdirectories "foo", "bar" and "baz", the output +# will be (the order of the directories is not necessarily alphabetical): +# +# ["foo", "bar", "baz"] +# + +import glob +import json + +print(json.dumps(glob.glob("*/")).replace("/", "")) From 49c24b7d2d28c0dc1475990aa14af59d88504a65 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 17:21:46 -0700 Subject: [PATCH 27/64] Try again --- python-project/{list-changed-dirs => list-changed-dirs.py} | 0 python-project/{list-changed-langs => list-changed-langs.py} | 0 python-project/{list-dirs => list-dirs.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename python-project/{list-changed-dirs => list-changed-dirs.py} (100%) rename python-project/{list-changed-langs => list-changed-langs.py} (100%) rename python-project/{list-dirs => list-dirs.py} (100%) diff --git a/python-project/list-changed-dirs b/python-project/list-changed-dirs.py similarity index 100% rename from python-project/list-changed-dirs rename to python-project/list-changed-dirs.py diff --git a/python-project/list-changed-langs b/python-project/list-changed-langs.py similarity index 100% rename from python-project/list-changed-langs rename to python-project/list-changed-langs.py diff --git a/python-project/list-dirs b/python-project/list-dirs.py similarity index 100% rename from python-project/list-dirs rename to python-project/list-dirs.py From 96592a5fc3a0931d646090557fb1a739bcac4c20 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 17:26:04 -0700 Subject: [PATCH 28/64] all codeql supported languages by extension --- .github/scripts/list-changed-langs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/scripts/list-changed-langs b/.github/scripts/list-changed-langs index 544d049f..6269b452 100755 --- a/.github/scripts/list-changed-langs +++ b/.github/scripts/list-changed-langs @@ -31,9 +31,17 @@ for line in lines: outlines.add("javascript") if find_in_list(typescript, line): outlines.add("javascript") - if find_in_list(javascript, line): - outlines.add("javascript") - - - + if find_in_list(c_and_cplus, line): + outlines.add("cpp") + if find_in_list(csharp, line): + outlines.add("csharp") + if find_in_list(golang, line): + outlines.add("go") + if find_in_list(python_lang, line): + outlines.add("python") + if find_in_list(java, line): + outlines.add("java") + if find_in_list(ruby, line): + outlines.add("ruby") + print(json.dumps(list(outlines))) From 38a065f9b7f22a2bbd71da84a9a06c3c3e6133ed Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 21:35:02 -0700 Subject: [PATCH 29/64] testing non-rec matrix --- .github/codeql/codeql-config-javascript.yml | 5 ++ .github/codeql/codeql-config-python.yml | 4 ++ .github/scripts/list-changed-langs | 60 ++++++++++++++------- .github/workflows/code-scanning.yml | 16 ++++++ 4 files changed, 66 insertions(+), 19 deletions(-) create mode 100644 .github/codeql/codeql-config-javascript.yml create mode 100644 .github/codeql/codeql-config-python.yml diff --git a/.github/codeql/codeql-config-javascript.yml b/.github/codeql/codeql-config-javascript.yml new file mode 100644 index 00000000..bc404c44 --- /dev/null +++ b/.github/codeql/codeql-config-javascript.yml @@ -0,0 +1,5 @@ +name: "CodeQL config" + +paths: + - project-1 + - project-3 \ No newline at end of file diff --git a/.github/codeql/codeql-config-python.yml b/.github/codeql/codeql-config-python.yml new file mode 100644 index 00000000..be22fddf --- /dev/null +++ b/.github/codeql/codeql-config-python.yml @@ -0,0 +1,4 @@ +name: "CodeQL config" + +paths: + - python-project \ No newline at end of file diff --git a/.github/scripts/list-changed-langs b/.github/scripts/list-changed-langs index 6269b452..9df07f58 100755 --- a/.github/scripts/list-changed-langs +++ b/.github/scripts/list-changed-langs @@ -4,7 +4,9 @@ # # ["foo", "bar", "baz"] # +from genericpath import isdir import json +import os javascript = [".js", ".jsx", ".mjs", ".es", ".es6", ".htm", ".html", ".xhtm", ".xhtml", ".vue", ".hbs", ".ejs", ".njk", ".json", ".yaml", ".yml", ".raml", ".xml"] typescript = [".ts", ".tsx", ".mts", ".cts"] @@ -15,9 +17,12 @@ python_lang = [".py"] java = [".java"] ruby = [".rb", ".erb", ".gemspec", "Gemfile"] - lines = list(open("./.github/scripts/diff.txt").readlines()) -outlines = set() +outlines = dict() + +def serialize_sets(obj): + if isinstance(obj, set): + return list(obj) def find_in_list(list, string): for item in list: @@ -27,21 +32,38 @@ def find_in_list(list, string): #only add items that are directories for line in lines: - if find_in_list(javascript, line): - outlines.add("javascript") - if find_in_list(typescript, line): - outlines.add("javascript") - if find_in_list(c_and_cplus, line): - outlines.add("cpp") - if find_in_list(csharp, line): - outlines.add("csharp") - if find_in_list(golang, line): - outlines.add("go") - if find_in_list(python_lang, line): - outlines.add("python") - if find_in_list(java, line): - outlines.add("java") - if find_in_list(ruby, line): - outlines.add("ruby") + path = line.split('/')[0] + if find_in_list(javascript, line) and (os.path.isdir(path)): + if (outlines.get("javascript") == None): + outlines["javascript"] = set() + outlines["javascript"].add(path) + if find_in_list(typescript, line) and (os.path.isdir(path)): + if (outlines.get("javascript") == None): + outlines["javascript"] = set() + outlines["javascript"].add(path) + if find_in_list(c_and_cplus, line) and (os.path.isdir(path)): + if (outlines.get("cpp") == None): + outlines["cpp"] = set() + outlines["cpp"].add(path) + if find_in_list(csharp, line) and (os.path.isdir(path)): + if (outlines.get("csharp") == None): + outlines["csharp"] = set() + outlines["csharp"].add(path) + if find_in_list(golang, line) and (os.path.isdir(path)): + if (outlines.get("go") == None): + outlines["go"] = set() + outlines["go"].add(path) + if find_in_list(python_lang, line) and (os.path.isdir(path)): + if (outlines.get("python") == None): + outlines["python"] = set() + outlines["python"].add(path) + if find_in_list(java, line) and (os.path.isdir(path)): + if (outlines.get("java") == None): + outlines["java"] = set() + outlines["java"].add(path) + if find_in_list(ruby, line) and (os.path.isdir(path)): + if (outlines.get("ruby") == None): + outlines["ruby"] = set() + outlines["ruby"].add(path) -print(json.dumps(list(outlines))) +print(json.dumps(outlines, default=serialize_sets)) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index d2f7fc63..d69d3988 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -33,6 +33,7 @@ jobs: runs-on: ubuntu-latest outputs: dir-list: ${{steps.find-dirs.outputs.dir-list}} + matrix: ${{steps.set-matrix.outputs.matrix}} steps: - name: Checkout repository @@ -45,6 +46,10 @@ jobs: run: | echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" + - id: set-matrix + run: echo "::set-output name=matrix::{\"include\":[{\"languages\":\"javascript\",\"config-file\":\"./.github/codeql/codeql-config-javascript.yml\"},{\"languages\":\"python\",\"config-file\":\"./.github/codeql/codeql-config-\"}]}" + + generate-changed-dir-list: # Find only the changed top level directories and use them for the scan # when the workflow is triggered by a pull_request @@ -54,6 +59,7 @@ jobs: runs-on: ubuntu-latest outputs: dir-list: ${{steps.find-dirs.outputs.dir-list}} + lang-list: ${{steps.find-langs.outputs.lang-list}} steps: - name: Checkout repository @@ -67,6 +73,11 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" + - name: Find changed languages + id: find-langs + run: | + echo "::set-output name=lang-list::$(./.github/scripts/list-changed-langs)" + - name: Upload diff as artifact uses: actions/upload-artifact@v2 with: @@ -84,6 +95,8 @@ jobs: fail-fast: false matrix: target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} + languages: ["javascript", "ruby"] + # # Prevent the creation of jobs for directories where code scanning is # not necessary/desired. @@ -113,6 +126,7 @@ jobs: uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml + languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 @@ -125,6 +139,7 @@ jobs: fail-fast: false matrix: target-dir: ${{fromJson(needs.generate-changed-dir-list.outputs.dir-list)}} + languages: ${{fromJson(needs.generate-changed-dir-list.outputs.lang-list)}} steps: - name: Checkout repository @@ -148,6 +163,7 @@ jobs: uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml + languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 From 1df457d7ea674aab9f36118b6d44aba00f7ce77d Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 21:37:56 -0700 Subject: [PATCH 30/64] fixes to use non-rec matrix --- .github/workflows/code-scanning.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index d69d3988..2e1afd3b 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -93,16 +93,16 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} - languages: ["javascript", "ruby"] + matrix: ${{fromJson(needs.generate-dir-list.outputs.matrix}} + # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} + # languages: ["javascript", "ruby"] # # Prevent the creation of jobs for directories where code scanning is # not necessary/desired. # - exclude: - - target-dir: docs + # exclude: + # - target-dir: docs steps: - name: Checkout repository From 8f43960dc30f1fd6b66fa85f33d40f3a135108c3 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 21:42:41 -0700 Subject: [PATCH 31/64] fix error with non-rec matrix --- .github/workflows/code-scanning.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 2e1afd3b..8056e583 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -47,7 +47,8 @@ jobs: echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" - id: set-matrix - run: echo "::set-output name=matrix::{\"include\":[{\"languages\":\"javascript\",\"config-file\":\"./.github/codeql/codeql-config-javascript.yml\"},{\"languages\":\"python\",\"config-file\":\"./.github/codeql/codeql-config-\"}]}" + run: | + echo "::set-output name=matrix::{\"include\":[{\"languages\":\"javascript\",\"config-file\":\"./.github/codeql/codeql-config-javascript.yml\"},{\"languages\":\"python\",\"config-file\":\"./.github/codeql/codeql-config-\"}]}" generate-changed-dir-list: @@ -93,7 +94,7 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{fromJson(needs.generate-dir-list.outputs.matrix}} + matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix) }} # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} # languages: ["javascript", "ruby"] From a5820d9faf4bdeb75299f4f6f3c95d9e7849cac8 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 21:44:29 -0700 Subject: [PATCH 32/64] getting closer --- .github/workflows/code-scanning.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 8056e583..b96b14ff 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -116,18 +116,18 @@ jobs: # # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#using-a-custom-configuration-file # - - name: Build CodeQL config file - env: - TARGET_DIR: ${{matrix.target-dir}} - run: | - cp .github/codeql/codeql-config-template.yml codeql-config.yml - sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml + #- name: Build CodeQL config file + # env: + # TARGET_DIR: ${{matrix.target-dir}} + # run: | + # cp .github/codeql/codeql-config-template.yml codeql-config.yml + # sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL uses: github/codeql-action/init@v2 - with: - config-file: codeql-config.yml - languages: ${{ matrix.languages }} + #with: + # config-file: codeql-config.yml + # languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 From 3249d814bebf316c8573d4e9b2d3e0db987406a5 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 21:50:34 -0700 Subject: [PATCH 33/64] test non-rec matrix hard code --- .github/workflows/code-scanning.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index b96b14ff..c8651517 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -94,7 +94,12 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix) }} + matrix: + include: + - languages: javascript + config-file: ./.github/codeql/codeql-config-javascript.yml + - languages: python + config-file: ./.github/codeql/codeql-config-python.yml # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} # languages: ["javascript", "ruby"] From 75eaaf3c275d9412b211217b369d4eeb1adaa308 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 21:55:31 -0700 Subject: [PATCH 34/64] testing again --- .github/workflows/code-scanning.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index c8651517..9aae2f22 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -130,9 +130,9 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v2 - #with: - # config-file: codeql-config.yml - # languages: ${{ matrix.languages }} + with: + config-file: ${{ matrix.config-file }} + languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 From a4cdcff7ba1dbef9c699afca9b117fcbfc762841 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 22:04:10 -0700 Subject: [PATCH 35/64] another try --- .github/workflows/code-scanning.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 9aae2f22..8c6d9dc5 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -97,9 +97,9 @@ jobs: matrix: include: - languages: javascript - config-file: ./.github/codeql/codeql-config-javascript.yml + target-dir: ["project-1", "project-3"] - languages: python - config-file: ./.github/codeql/codeql-config-python.yml + target-dir: ["python-project"] # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} # languages: ["javascript", "ruby"] @@ -121,12 +121,12 @@ jobs: # # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#using-a-custom-configuration-file # - #- name: Build CodeQL config file - # env: - # TARGET_DIR: ${{matrix.target-dir}} - # run: | - # cp .github/codeql/codeql-config-template.yml codeql-config.yml - # sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml + - name: Build CodeQL config file + env: + TARGET_DIR: ${{matrix.target-dir}} + run: | + cp .github/codeql/codeql-config-template.yml codeql-config.yml + sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL uses: github/codeql-action/init@v2 From 1b0e300ed38b0d4478674ae67e032e88efa1bedf Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 22:08:58 -0700 Subject: [PATCH 36/64] trying again --- .github/workflows/code-scanning.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 8c6d9dc5..cf7a2aa6 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -97,9 +97,9 @@ jobs: matrix: include: - languages: javascript - target-dir: ["project-1", "project-3"] + target-dir: fromJSON(["project-1", "project-3"]) - languages: python - target-dir: ["python-project"] + target-dir: fromJSON(["python-project"]) # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} # languages: ["javascript", "ruby"] @@ -123,7 +123,7 @@ jobs: # - name: Build CodeQL config file env: - TARGET_DIR: ${{matrix.target-dir}} + TARGET_DIR: ${{ matrix.target-dir }} run: | cp .github/codeql/codeql-config-template.yml codeql-config.yml sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml From 3109265adf8e41e5abe2a33bd413d5407deae9fc Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 22:14:15 -0700 Subject: [PATCH 37/64] testing again --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index cf7a2aa6..0f19601a 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -97,9 +97,9 @@ jobs: matrix: include: - languages: javascript - target-dir: fromJSON(["project-1", "project-3"]) + target-dir: fromJSON("[\"project-1\", \"project-3\"]") - languages: python - target-dir: fromJSON(["python-project"]) + target-dir: fromJSON("[\"python-project\"]") # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} # languages: ["javascript", "ruby"] From 6583b89912c26b3a1e8aaab731572a1666dda044 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 22:17:30 -0700 Subject: [PATCH 38/64] build matrix different --- .github/workflows/code-scanning.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 0f19601a..4a52e107 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -96,10 +96,13 @@ jobs: fail-fast: false matrix: include: - - languages: javascript - target-dir: fromJSON("[\"project-1\", \"project-3\"]") - - languages: python - target-dir: fromJSON("[\"python-project\"]") + - target-dir: "project-1" + languages: "javascript" + - target-dir: project-3" + languages: "javascript" + - target-dir: "python-project" + languages: "python" + # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} # languages: ["javascript", "ruby"] From b63aca9379644db90144ad86a18d9c5668fc2068 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 22:18:10 -0700 Subject: [PATCH 39/64] new test --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 4a52e107..d154d2b6 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -98,7 +98,7 @@ jobs: include: - target-dir: "project-1" languages: "javascript" - - target-dir: project-3" + - target-dir: "project-3" languages: "javascript" - target-dir: "python-project" languages: "python" From 7de6b05c3a7d23c73f84385543f12542b94f361f Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Wed, 17 Aug 2022 22:23:58 -0700 Subject: [PATCH 40/64] dynamic matrix test --- .github/workflows/code-scanning.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index d154d2b6..718f77dd 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -48,7 +48,7 @@ jobs: - id: set-matrix run: | - echo "::set-output name=matrix::{\"include\":[{\"languages\":\"javascript\",\"config-file\":\"./.github/codeql/codeql-config-javascript.yml\"},{\"languages\":\"python\",\"config-file\":\"./.github/codeql/codeql-config-\"}]}" + echo "::set-output name=matrix::{\"include\":[{\"languages\":\"javascript\",\"target-dir\":\"project-1\"},{\"languages\":\"javascript\",\"target-dir\":\"project-3\"},{\"languages\":\"python\",\"target-dir\":\"python-project\"}]}" generate-changed-dir-list: @@ -94,14 +94,14 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - include: - - target-dir: "project-1" - languages: "javascript" - - target-dir: "project-3" - languages: "javascript" - - target-dir: "python-project" - languages: "python" + matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix)}} + # include: + # - target-dir: "project-1" + # languages: "javascript" + # - target-dir: "project-3" + # languages: "javascript" + # - target-dir: "python-project" + # languages: "python" # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} # languages: ["javascript", "ruby"] From a4d22a7f7474a64432a7eec56760a105c8c84001 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 08:11:28 -0700 Subject: [PATCH 41/64] Revert "Update code-scanning.yml" This reverts commit 53931eba58203ac1745ab4a61bb337234f9da3df. reverting to main --- .github/workflows/code-scanning.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 718f77dd..7f455c75 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 @@ -64,7 +64,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 @@ -115,7 +115,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v2 # # Build the configuration file for CodeQL to instruct it to only scan the @@ -132,16 +132,16 @@ jobs: sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v1 with: config-file: ${{ matrix.config-file }} languages: ${{ matrix.languages }} - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v1 codeql-changed-dirs: - name: Scan changed directories with CodeQL + name: Scan all directories with CodeQL needs: generate-changed-dir-list runs-on: ubuntu-latest strategy: @@ -152,7 +152,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v2 # # Build the configuration file for CodeQL to instruct it to only scan the @@ -169,10 +169,10 @@ jobs: sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v1 with: config-file: codeql-config.yml languages: ${{ matrix.languages }} - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v1 From 86e1224c5c9b27c902bd79fcc6bd7dc2b0aa5fbf Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 09:37:56 -0700 Subject: [PATCH 42/64] Updates to use script --- .github/scripts/list-changed-langs | 39 +++++++++++------------------ .github/workflows/code-scanning.yml | 4 +-- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/.github/scripts/list-changed-langs b/.github/scripts/list-changed-langs index 9df07f58..f09ea5a9 100755 --- a/.github/scripts/list-changed-langs +++ b/.github/scripts/list-changed-langs @@ -19,10 +19,15 @@ ruby = [".rb", ".erb", ".gemspec", "Gemfile"] lines = list(open("./.github/scripts/diff.txt").readlines()) outlines = dict() +outlines["include"] = set() def serialize_sets(obj): if isinstance(obj, set): - return list(obj) + l = list() + for item in obj: + if isinstance(item, tuple): + l.append(dict((x, y) for x, y in item)) + return l def find_in_list(list, string): for item in list: @@ -34,36 +39,20 @@ def find_in_list(list, string): for line in lines: path = line.split('/')[0] if find_in_list(javascript, line) and (os.path.isdir(path)): - if (outlines.get("javascript") == None): - outlines["javascript"] = set() - outlines["javascript"].add(path) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "javascript"}).items())) if find_in_list(typescript, line) and (os.path.isdir(path)): - if (outlines.get("javascript") == None): - outlines["javascript"] = set() - outlines["javascript"].add(path) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "javascript"}).items())) if find_in_list(c_and_cplus, line) and (os.path.isdir(path)): - if (outlines.get("cpp") == None): - outlines["cpp"] = set() - outlines["cpp"].add(path) + outlines["include"].add(dict({"target-dir": path, "languages": "cpp"})) if find_in_list(csharp, line) and (os.path.isdir(path)): - if (outlines.get("csharp") == None): - outlines["csharp"] = set() - outlines["csharp"].add(path) + outlines["include"].add(dict({"target-dir": path, "languages": "csharp"})) if find_in_list(golang, line) and (os.path.isdir(path)): - if (outlines.get("go") == None): - outlines["go"] = set() - outlines["go"].add(path) + outlines["include"].add(dict({"target-dir": path, "languages": "go"})) if find_in_list(python_lang, line) and (os.path.isdir(path)): - if (outlines.get("python") == None): - outlines["python"] = set() - outlines["python"].add(path) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "python"}).items())) if find_in_list(java, line) and (os.path.isdir(path)): - if (outlines.get("java") == None): - outlines["java"] = set() - outlines["java"].add(path) + outlines["include"].add(dict({"target-dir": path, "languages": "java"})) if find_in_list(ruby, line) and (os.path.isdir(path)): - if (outlines.get("ruby") == None): - outlines["ruby"] = set() - outlines["ruby"].add(path) + outlines["include"].add(dict({"target-dir": path, "languages": "ruby"})) print(json.dumps(outlines, default=serialize_sets)) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 718f77dd..f487e834 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -9,7 +9,7 @@ name: Code scanning for all apps # on: push: - branches: [main] + branches: [main, dynamic-language-configuration] paths-ignore: - 'docs/**' - '*' @@ -48,7 +48,7 @@ jobs: - id: set-matrix run: | - echo "::set-output name=matrix::{\"include\":[{\"languages\":\"javascript\",\"target-dir\":\"project-1\"},{\"languages\":\"javascript\",\"target-dir\":\"project-3\"},{\"languages\":\"python\",\"target-dir\":\"python-project\"}]}" + echo "::set-output name=matrix::$(./.github/scripts/list-changed-langs)" generate-changed-dir-list: From 5267922493a914941af74362d6d3638282c59c09 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 09:54:48 -0700 Subject: [PATCH 43/64] changes to dynamic --- .github/scripts/list-all | 58 +++++++++++++++++++ .../{list-changed-langs => list-changed} | 10 ++-- .github/scripts/list-changed-dirs | 24 -------- .github/scripts/list-dirs | 15 ----- .github/workflows/code-scanning.yml | 24 +------- 5 files changed, 65 insertions(+), 66 deletions(-) create mode 100755 .github/scripts/list-all rename .github/scripts/{list-changed-langs => list-changed} (81%) delete mode 100755 .github/scripts/list-changed-dirs delete mode 100755 .github/scripts/list-dirs diff --git a/.github/scripts/list-all b/.github/scripts/list-all new file mode 100755 index 00000000..9b8bac09 --- /dev/null +++ b/.github/scripts/list-all @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# This script prints a JSON array containing all the supported CodeQL programming languages based on the file extension +# +# ["foo", "bar", "baz"] +# +from genericpath import isdir +import json +import os +import glob + +javascript = [".js", ".jsx", ".mjs", ".es", ".es6", ".htm", ".html", ".xhtm", ".xhtml", ".vue", ".hbs", ".ejs", ".njk", ".json", ".yaml", ".yml", ".raml", ".xml"] +typescript = [".ts", ".tsx", ".mts", ".cts"] +c_and_cplus = [".cpp", ".c++", ".cxx", ".hpp", ".hh", ".h++", ".hxx", ".c," ".cc", ".h"] +csharp = [".sln", ".csproj", ".cs", ".cshtml", ".xaml"] +golang = [".go"] +python_lang = [".py"] +java = [".java"] +ruby = [".rb", ".erb", ".gemspec", "Gemfile"] + +outlines = dict() +outlines["include"] = set() + +def serialize_sets(obj): + if isinstance(obj, set): + l = list() + for item in obj: + if isinstance(item, tuple): + l.append(dict((x, y) for x, y in item)) + return l + +def find_in_list(list, string): + for item in list: + if item in string: + return True + return False + +#only add items that are directories +for line in glob.glob('**', recursive=True): + path = line.split('/')[0] + if find_in_list(javascript, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "javascript"}).items())) + if find_in_list(typescript, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "javascript"}).items())) + if find_in_list(c_and_cplus, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "cpp"}).items())) + if find_in_list(csharp, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "csharp"}).items())) + if find_in_list(golang, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "go"}).items())) + if find_in_list(python_lang, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "python"}).items())) + if find_in_list(java, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "java"}).items())) + if find_in_list(ruby, line) and (os.path.isdir(path)): + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "ruby"}).items())) + +print(json.dumps(outlines, default=serialize_sets)) diff --git a/.github/scripts/list-changed-langs b/.github/scripts/list-changed similarity index 81% rename from .github/scripts/list-changed-langs rename to .github/scripts/list-changed index f09ea5a9..4e81e29d 100755 --- a/.github/scripts/list-changed-langs +++ b/.github/scripts/list-changed @@ -43,16 +43,16 @@ for line in lines: if find_in_list(typescript, line) and (os.path.isdir(path)): outlines["include"].add(tuple(dict({"target-dir": path, "languages": "javascript"}).items())) if find_in_list(c_and_cplus, line) and (os.path.isdir(path)): - outlines["include"].add(dict({"target-dir": path, "languages": "cpp"})) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "cpp"}).items())) if find_in_list(csharp, line) and (os.path.isdir(path)): - outlines["include"].add(dict({"target-dir": path, "languages": "csharp"})) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "csharp"}).items())) if find_in_list(golang, line) and (os.path.isdir(path)): - outlines["include"].add(dict({"target-dir": path, "languages": "go"})) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "go"}).items())) if find_in_list(python_lang, line) and (os.path.isdir(path)): outlines["include"].add(tuple(dict({"target-dir": path, "languages": "python"}).items())) if find_in_list(java, line) and (os.path.isdir(path)): - outlines["include"].add(dict({"target-dir": path, "languages": "java"})) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "java"}).items())) if find_in_list(ruby, line) and (os.path.isdir(path)): - outlines["include"].add(dict({"target-dir": path, "languages": "ruby"})) + outlines["include"].add(tuple(dict({"target-dir": path, "languages": "ruby"}).items())) print(json.dumps(outlines, default=serialize_sets)) diff --git a/.github/scripts/list-changed-dirs b/.github/scripts/list-changed-dirs deleted file mode 100755 index d51ee957..00000000 --- a/.github/scripts/list-changed-dirs +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -# -# This script prints a JSON array containing all non-hidden subdirectories of -# the current working directory. As an example, if the current working -# directory contains the subdirectories "foo", "bar" and "baz", the output -# will be (the order of the directories is not necessarily alphabetical): -# -# ["foo", "bar", "baz"] -# -from genericpath import isdir -import json -import os - -lines = list(open('./.github/scripts/diff.txt').readlines()) -outlines = set() - -#only add items that are directories -for line in lines: - path = line.split('/')[0] - if (os.path.isdir(path)): - outlines.add(path) - -print(json.dumps(list(outlines))) diff --git a/.github/scripts/list-dirs b/.github/scripts/list-dirs deleted file mode 100755 index 1210d0f9..00000000 --- a/.github/scripts/list-dirs +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 - -# -# This script prints a JSON array containing all non-hidden subdirectories of -# the current working directory. As an example, if the current working -# directory contains the subdirectories "foo", "bar" and "baz", the output -# will be (the order of the directories is not necessarily alphabetical): -# -# ["foo", "bar", "baz"] -# - -import glob -import json - -print(json.dumps(glob.glob("*/")).replace("/", "")) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index f487e834..059be9e9 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -42,13 +42,10 @@ jobs: fetch-depth: 0 - name: Find existing directories - id: find-dirs - run: | - echo "::set-output name=dir-list::$(./.github/scripts/list-dirs)" - - id: set-matrix run: | - echo "::set-output name=matrix::$(./.github/scripts/list-changed-langs)" + echo "::set-output name=matrix::$(./.github/scripts/list-all)" + generate-changed-dir-list: @@ -95,23 +92,6 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix)}} - # include: - # - target-dir: "project-1" - # languages: "javascript" - # - target-dir: "project-3" - # languages: "javascript" - # - target-dir: "python-project" - # languages: "python" - - # target-dir: ${{fromJson(needs.generate-dir-list.outputs.dir-list)}} - # languages: ["javascript", "ruby"] - - # - # Prevent the creation of jobs for directories where code scanning is - # not necessary/desired. - # - # exclude: - # - target-dir: docs steps: - name: Checkout repository From dd8bef14dfb1ca191abf9eb1f44c5f5eef9514ab Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 09:56:58 -0700 Subject: [PATCH 44/64] fixes for dynamic scannign --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 77c87fb9..1a3bd00d 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -42,7 +42,7 @@ jobs: fetch-depth: 0 - name: Find existing directories - - id: set-matrix + id: set-matrix run: | echo "::set-output name=matrix::$(./.github/scripts/list-all)" From 513f8a1e6ba8ac59762b9f691076e7698c9ee658 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 10:03:41 -0700 Subject: [PATCH 45/64] Changes to use new scripts --- .github/workflows/code-scanning.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 1a3bd00d..5aed029b 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -32,7 +32,6 @@ jobs: name: Generate directory list runs-on: ubuntu-latest outputs: - dir-list: ${{steps.find-dirs.outputs.dir-list}} matrix: ${{steps.set-matrix.outputs.matrix}} steps: @@ -56,8 +55,7 @@ jobs: name: Generate changed directory list runs-on: ubuntu-latest outputs: - dir-list: ${{steps.find-dirs.outputs.dir-list}} - lang-list: ${{steps.find-langs.outputs.lang-list}} + matrix: ${{steps.set-matrix.outputs.matrix}} steps: - name: Checkout repository @@ -74,7 +72,7 @@ jobs: - name: Find changed languages id: find-langs run: | - echo "::set-output name=lang-list::$(./.github/scripts/list-changed-langs)" + echo "::set-output name=matrix::$(./.github/scripts/list-changed)" - name: Upload diff as artifact uses: actions/upload-artifact@v2 @@ -91,7 +89,7 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix)}} + matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix) }} steps: - name: Checkout repository @@ -120,15 +118,15 @@ jobs: - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v1 + + codeql-changed-dirs: name: Scan all directories with CodeQL needs: generate-changed-dir-list runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - target-dir: ${{fromJson(needs.generate-changed-dir-list.outputs.dir-list)}} - languages: ${{fromJson(needs.generate-changed-dir-list.outputs.lang-list)}} + matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix) }} steps: - name: Checkout repository @@ -151,7 +149,7 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: - config-file: codeql-config.yml + config-file: ${{ matrix.config-file }} languages: ${{ matrix.languages }} - name: Perform CodeQL analysis From 12f8f9707abb05c9e2fe922b07d1642fdd361819 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 10:07:06 -0700 Subject: [PATCH 46/64] Updates to logging --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 5aed029b..5fe5eaf5 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -84,7 +84,7 @@ jobs: codeql-all-dirs: - name: Scan all directories with CodeQL + name: Scanning ${{matrix.target-dir}} with CodeQL needs: generate-dir-list runs-on: ubuntu-latest strategy: @@ -121,7 +121,7 @@ jobs: codeql-changed-dirs: - name: Scan all directories with CodeQL + name: Scanning ${{matrix.target-dir}} with CodeQL needs: generate-changed-dir-list runs-on: ubuntu-latest strategy: From 4926a3e51763edbb2b52693987d3a2dab5ba9b9f Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 10:08:39 -0700 Subject: [PATCH 47/64] More logging updates --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 5fe5eaf5..30c1ed5e 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -84,7 +84,7 @@ jobs: codeql-all-dirs: - name: Scanning ${{matrix.target-dir}} with CodeQL + name: Scanning ${{matrix.target-dir}} (${{ matrix.languages }}) with CodeQL needs: generate-dir-list runs-on: ubuntu-latest strategy: @@ -121,7 +121,7 @@ jobs: codeql-changed-dirs: - name: Scanning ${{matrix.target-dir}} with CodeQL + name: Scanning ${{matrix.target-dir}} (${{ matrix.languages }}) with CodeQL needs: generate-changed-dir-list runs-on: ubuntu-latest strategy: From 8ae1a2799883ec5adf268dd99d98c2629351f007 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 10:14:19 -0700 Subject: [PATCH 48/64] Fix for workflow --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 30c1ed5e..8eff9e3c 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -126,7 +126,7 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix) }} + matrix: ${{ fromJson(needs.generate-changed-dir-list.outputs.matrix) }} steps: - name: Checkout repository From b7ed1549c489c990a59a850264084c6f6b19c51e Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 10:21:11 -0700 Subject: [PATCH 49/64] Fixes for changed workflow --- .github/workflows/code-scanning.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 8eff9e3c..030676c8 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -63,15 +63,10 @@ jobs: with: fetch-depth: 0 - - name: Find changed directories - id: find-dirs + - name: Find changed apps + id: set-matrix run: | git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt - echo "::set-output name=dir-list::$(./.github/scripts/list-changed-dirs)" - - - name: Find changed languages - id: find-langs - run: | echo "::set-output name=matrix::$(./.github/scripts/list-changed)" - name: Upload diff as artifact From 597b8d8c2ed299cd10bde9b261887dfc86d15922 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 13:58:25 -0700 Subject: [PATCH 50/64] Fix analysis categories --- .github/workflows/code-scanning.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 030676c8..40844a0b 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -107,11 +107,13 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: - config-file: ${{ matrix.config-file }} + config-file: codeql-config.yml languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v1 + with: + category: ${{ matrix.target-dir }}-${{ matrix.languages }} @@ -144,8 +146,10 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: - config-file: ${{ matrix.config-file }} + config-file: codeql-config.yml languages: ${{ matrix.languages }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v1 + with: + category: ${{ matrix.target-dir }}-${{ matrix.languages }} From 40419487da91417a23a5458374ac4dc989ad4bd0 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 14:08:32 -0700 Subject: [PATCH 51/64] Clean up workflow file --- .github/workflows/code-scanning.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 40844a0b..e770783c 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -9,7 +9,7 @@ name: Code scanning for all apps # on: push: - branches: [main, dynamic-language-configuration] + branches: [main] paths-ignore: - 'docs/**' - '*' @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -59,7 +59,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -88,7 +88,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # # Build the configuration file for CodeQL to instruct it to only scan the @@ -105,13 +105,13 @@ jobs: sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml languages: ${{ matrix.languages }} - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 with: category: ${{ matrix.target-dir }}-${{ matrix.languages }} @@ -127,7 +127,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # # Build the configuration file for CodeQL to instruct it to only scan the @@ -144,12 +144,12 @@ jobs: sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: config-file: codeql-config.yml languages: ${{ matrix.languages }} - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 with: category: ${{ matrix.target-dir }}-${{ matrix.languages }} From 7cb819e76b8a760236851e20cdcbb995cb89137e Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:00:20 -0700 Subject: [PATCH 52/64] More accurate file extension identification --- .github/scripts/list-all | 3 +-- .github/scripts/list-changed | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/scripts/list-all b/.github/scripts/list-all index 9b8bac09..43a7488b 100755 --- a/.github/scripts/list-all +++ b/.github/scripts/list-all @@ -31,11 +31,10 @@ def serialize_sets(obj): def find_in_list(list, string): for item in list: - if item in string: + if string.strip().endswith(item): return True return False -#only add items that are directories for line in glob.glob('**', recursive=True): path = line.split('/')[0] if find_in_list(javascript, line) and (os.path.isdir(path)): diff --git a/.github/scripts/list-changed b/.github/scripts/list-changed index 4e81e29d..729076f2 100755 --- a/.github/scripts/list-changed +++ b/.github/scripts/list-changed @@ -31,11 +31,10 @@ def serialize_sets(obj): def find_in_list(list, string): for item in list: - if item in string: + if string.strip().endswith(item): return True return False -#only add items that are directories for line in lines: path = line.split('/')[0] if find_in_list(javascript, line) and (os.path.isdir(path)): From e1cbd07683fbf65cc07c75b1cc8def647fbdc624 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:07:08 -0700 Subject: [PATCH 53/64] Clean up comments --- .github/scripts/list-all | 5 +++-- .github/scripts/list-changed | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/scripts/list-all b/.github/scripts/list-all index 43a7488b..4ce945d5 100755 --- a/.github/scripts/list-all +++ b/.github/scripts/list-all @@ -1,9 +1,10 @@ #!/usr/bin/env python3 # -# This script prints a JSON array containing all the supported CodeQL programming languages based on the file extension +# This script prints a JSON representation of an include matrix that will create a job for each top level folder/language combination detected in the repository. # -# ["foo", "bar", "baz"] +# {"include": [{"target-dir": ".github", "languages": "javascript"}, {"target-dir": "project with spaces", "languages": "javascript"}, {"target-dir": "project-1", "languages": "javascript"}, {"target-dir": "python-project", "languages": "python"}]} # + from genericpath import isdir import json import os diff --git a/.github/scripts/list-changed b/.github/scripts/list-changed index 729076f2..a056d375 100755 --- a/.github/scripts/list-changed +++ b/.github/scripts/list-changed @@ -1,9 +1,10 @@ #!/usr/bin/env python3 # -# This script prints a JSON array containing all the supported CodeQL programming languages based on the file extension +# This script prints a JSON representation of an include matrix - based on the output of git diff - that will create a job for each top level folder/language combination detected in the repository. # -# ["foo", "bar", "baz"] +# {"include": [{"target-dir": ".github", "languages": "javascript"}, {"target-dir": "project with spaces", "languages": "javascript"}, {"target-dir": "project-1", "languages": "javascript"}, {"target-dir": "python-project", "languages": "python"}]} # + from genericpath import isdir import json import os From e8a1d6773708bb2dc6add89d67d43520f0650d93 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:35:25 -0700 Subject: [PATCH 54/64] testing streamlined flow --- .github/workflows/code-scanning.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index e770783c..a92d7084 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -28,7 +28,6 @@ jobs: # Find all the top level directories in the repostiory and use them for the scan # when the workflow is not triggered by a pull_request # - if: ${{ github.event_name != 'pull_request'}} name: Generate directory list runs-on: ubuntu-latest outputs: @@ -40,11 +39,34 @@ jobs: with: fetch-depth: 0 + - name: Find all apps + if: ${{ github.event_name != 'pull_request'}} + id: find-all + run: | + echo "::set-output name=all::$(./.github/scripts/list-all)" + + - name: Find changed apps + if: ${{ github.event_name == 'pull_request'}} + id: find-changed + run: | + git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt + echo "::set-output name=changed::$(./.github/scripts/list-changed)" + - name: Find existing directories id: set-matrix + env: + MATRIX: ${{ steps.find-all.all || steps.find-changed.changed }} run: | - echo "::set-output name=matrix::$(./.github/scripts/list-all)" + echo "::set-output name=matrix::$MATRIX" + echo "::debug::Matrix set to $MATRIX" + - name: Upload diff as artifact + if: ${{ github.event_name == 'pull_request'}} + uses: actions/upload-artifact@v2 + with: + name: diff + path: | + ./.github/scripts/diff.txt generate-changed-dir-list: From 52d0c25693621e6b9ec5823e8a6addae4dda45b5 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:38:12 -0700 Subject: [PATCH 55/64] testing streamline --- .github/workflows/code-scanning.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index a92d7084..5873c7b6 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -55,10 +55,12 @@ jobs: - name: Find existing directories id: set-matrix env: - MATRIX: ${{ steps.find-all.all || steps.find-changed.changed }} + ALL: ${{ steps.find-all.all }} + CHANGED: ${{ steps.find-changed.changed }} run: | - echo "::set-output name=matrix::$MATRIX" - echo "::debug::Matrix set to $MATRIX" + echo "::debug::All set to $ALL" + echo "::debug::Changed set to $CHANGED" + # echo "::set-output name=matrix::$MATRIX" - name: Upload diff as artifact if: ${{ github.event_name == 'pull_request'}} From 3c4fafa4fed1ae38330e94f810bae44aba87be6f Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:40:09 -0700 Subject: [PATCH 56/64] test streamline --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 5873c7b6..c57217a0 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -55,8 +55,8 @@ jobs: - name: Find existing directories id: set-matrix env: - ALL: ${{ steps.find-all.all }} - CHANGED: ${{ steps.find-changed.changed }} + ALL: ${{ steps.find-all.output.all }} + CHANGED: ${{ steps.find-changed.output.changed }} run: | echo "::debug::All set to $ALL" echo "::debug::Changed set to $CHANGED" From 6eb8becb4a05ab2289be1d053432d015bdbbcc8f Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:41:07 -0700 Subject: [PATCH 57/64] test streamline --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index c57217a0..1c5abd3d 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -55,8 +55,8 @@ jobs: - name: Find existing directories id: set-matrix env: - ALL: ${{ steps.find-all.output.all }} - CHANGED: ${{ steps.find-changed.output.changed }} + ALL: ${{ steps.find-all.outputs.all }} + CHANGED: ${{ steps.find-changed.outputs.changed }} run: | echo "::debug::All set to $ALL" echo "::debug::Changed set to $CHANGED" From 35879e8862252ad5f64f7d7a5a008d80c5740140 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:44:48 -0700 Subject: [PATCH 58/64] test streamline --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 1c5abd3d..34a69146 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -60,7 +60,7 @@ jobs: run: | echo "::debug::All set to $ALL" echo "::debug::Changed set to $CHANGED" - # echo "::set-output name=matrix::$MATRIX" + echo "::set-output name=matrix::$ALL$CHANGED" - name: Upload diff as artifact if: ${{ github.event_name == 'pull_request'}} From 3563401e3efc2e2f232d1b09e09a0d1ea487327e Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:48:08 -0700 Subject: [PATCH 59/64] testing streamlining --- .github/workflows/code-scanning.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 34a69146..d99a6f53 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -58,8 +58,8 @@ jobs: ALL: ${{ steps.find-all.outputs.all }} CHANGED: ${{ steps.find-changed.outputs.changed }} run: | - echo "::debug::All set to $ALL" - echo "::debug::Changed set to $CHANGED" + echo "::notice::All set to $ALL" + echo "::notice::Changed set to $CHANGED" echo "::set-output name=matrix::$ALL$CHANGED" - name: Upload diff as artifact From ca862baf7320aa765dea5a5522ee0dcd391d0460 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:55:48 -0700 Subject: [PATCH 60/64] more streamlining --- .github/workflows/code-scanning.yml | 85 ++++------------------------- 1 file changed, 10 insertions(+), 75 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index d99a6f53..7f1a8bcf 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -24,7 +24,7 @@ on: jobs: - generate-dir-list: + generate-scan-list: # Find all the top level directories in the repostiory and use them for the scan # when the workflow is not triggered by a pull_request # @@ -52,15 +52,15 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt echo "::set-output name=changed::$(./.github/scripts/list-changed)" - - name: Find existing directories + - name: Setup scanning matrix id: set-matrix env: ALL: ${{ steps.find-all.outputs.all }} CHANGED: ${{ steps.find-changed.outputs.changed }} run: | + echo "::set-output name=matrix::$ALL$CHANGED" echo "::notice::All set to $ALL" echo "::notice::Changed set to $CHANGED" - echo "::set-output name=matrix::$ALL$CHANGED" - name: Upload diff as artifact if: ${{ github.event_name == 'pull_request'}} @@ -71,44 +71,14 @@ jobs: ./.github/scripts/diff.txt - generate-changed-dir-list: - # Find only the changed top level directories and use them for the scan - # when the workflow is triggered by a pull_request - # - if: ${{ github.event_name == 'pull_request'}} - name: Generate changed directory list - runs-on: ubuntu-latest - outputs: - matrix: ${{steps.set-matrix.outputs.matrix}} - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Find changed apps - id: set-matrix - run: | - git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA >./.github/scripts/diff.txt - echo "::set-output name=matrix::$(./.github/scripts/list-changed)" - - - name: Upload diff as artifact - uses: actions/upload-artifact@v2 - with: - name: diff - path: | - ./.github/scripts/diff.txt - - - - codeql-all-dirs: + codeql-scan: name: Scanning ${{matrix.target-dir}} (${{ matrix.languages }}) with CodeQL - needs: generate-dir-list + needs: generate-scan-list runs-on: ubuntu-latest strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-dir-list.outputs.matrix) }} + matrix: ${{ fromJson(needs.generate-scan-list.outputs.matrix) }} steps: - name: Checkout repository @@ -134,46 +104,11 @@ jobs: config-file: codeql-config.yml languages: ${{ matrix.languages }} - - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@v2 - with: - category: ${{ matrix.target-dir }}-${{ matrix.languages }} - - - - codeql-changed-dirs: - name: Scanning ${{matrix.target-dir}} (${{ matrix.languages }}) with CodeQL - needs: generate-changed-dir-list - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.generate-changed-dir-list.outputs.matrix) }} - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # - # Build the configuration file for CodeQL to instruct it to only scan the - # contents of a single directory. For more information about the - # configuration options available, visit: - # - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#using-a-custom-configuration-file - # - - name: Build CodeQL config file - env: - TARGET_DIR: ${{matrix.target-dir}} - run: | - cp .github/codeql/codeql-config-template.yml codeql-config.yml - sed -i 's@__TARGET_DIR__@'"$TARGET_DIR"'@' codeql-config.yml - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - config-file: codeql-config.yml - languages: ${{ matrix.languages }} + - name: Attempting build + if: ${{ matrix.languages == "cpp" || matrix.languages == "csharp" || matrix.languages == "java" }} + uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 with: - category: ${{ matrix.target-dir }}-${{ matrix.languages }} + category: ${{ matrix.target-dir }}-${{ matrix.languages }} \ No newline at end of file From 295b9d38f2065554fbebacfd9760c924eb612098 Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:57:48 -0700 Subject: [PATCH 61/64] more streamlining --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 7f1a8bcf..173b6cec 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -105,7 +105,7 @@ jobs: languages: ${{ matrix.languages }} - name: Attempting build - if: ${{ matrix.languages == "cpp" || matrix.languages == "csharp" || matrix.languages == "java" }} + if: ${{ matrix.languages == 'cpp' }} || ${{ matrix.languages == 'csharp' }} || ${{ matrix.languages == 'java' }} uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL analysis From 9e02eb1885cde0da73dcd7e0e711f3262941143d Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 17:59:42 -0700 Subject: [PATCH 62/64] testing autobuild streamline --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 173b6cec..917fcbca 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -105,7 +105,7 @@ jobs: languages: ${{ matrix.languages }} - name: Attempting build - if: ${{ matrix.languages == 'cpp' }} || ${{ matrix.languages == 'csharp' }} || ${{ matrix.languages == 'java' }} + if: ${{ matrix.languages == 'cpp' }} uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL analysis From f6deb54ae5cbdc32ea285d15407c8df0466626df Mon Sep 17 00:00:00 2001 From: Dave Bloom Date: Thu, 18 Aug 2022 18:01:23 -0700 Subject: [PATCH 63/64] more streamlining of autobuild --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 917fcbca..db1b9772 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -105,7 +105,7 @@ jobs: languages: ${{ matrix.languages }} - name: Attempting build - if: ${{ matrix.languages == 'cpp' }} + if: ${{ (matrix.languages == 'cpp' || matrix.languages == 'csharp' || matrix.languages == 'java') }} uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL analysis From 70afe1d5063b45d47f177bf10263eca197338288 Mon Sep 17 00:00:00 2001 From: Dave <50186003+thedave42@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:08:46 -0800 Subject: [PATCH 64/64] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fb0c232f..a3760892 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Parallel code scanning with CodeQL +https://github.com/thedave42/parallel-code-scanning/labels/documentation + If you have a large repository containing various independent projects (a "monorepo"), the time taken to scan your code with CodeQL can be significantly reduced by splitting the scanning work into various parallel jobs which will