From 2dfe99fb4d7c3c4d532acd780dd18d7f06467391 Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Thu, 4 Dec 2025 08:16:17 +0100 Subject: [PATCH 1/5] Verify analysis-name for security --- .github/workflows/public-analyze-code-graph.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/public-analyze-code-graph.yml b/.github/workflows/public-analyze-code-graph.yml index 659aa04a1..856c96935 100644 --- a/.github/workflows/public-analyze-code-graph.yml +++ b/.github/workflows/public-analyze-code-graph.yml @@ -90,6 +90,12 @@ jobs: - name: Assure that either artifacts-upload-name or sources-upload-name is set if: inputs.artifacts-upload-name == '' && inputs.sources-upload-name == '' run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'sources-upload-name'."; exit 1 + - name: Verify analysis-name only consists of characters safe for folder names + run: | + if [[ ! "${{ inputs.analysis-name }}" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "The analysis-name '${{ inputs.analysis-name }}' contains invalid characters. Only alphanumeric characters, dots (.), underscores (_) and hyphens (-) are allowed." + exit 1 + fi - name: Assemble ENVIRONMENT_INFO run: echo "ENVIRONMENT_INFO=java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}" >> $GITHUB_ENV From b0983a66d001ee29ab864b8995421417914d3d81 Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Sun, 30 Nov 2025 11:39:19 +0100 Subject: [PATCH 2/5] Add input parameter for maven artifacts to public workflow --- .../workflows/internal-java-code-analysis.yml | 9 ++ .../workflows/public-analyze-code-graph.yml | 20 ++- INTEGRATION.md | 1 + scripts/downloadMavenArtifacts.sh | 86 ++++++++++ scripts/testDownloadMavenArtifacts.sh | 150 ++++++++++++++++++ 5 files changed, 262 insertions(+), 4 deletions(-) create mode 100755 scripts/downloadMavenArtifacts.sh create mode 100755 scripts/testDownloadMavenArtifacts.sh diff --git a/.github/workflows/internal-java-code-analysis.yml b/.github/workflows/internal-java-code-analysis.yml index ce7bc2494..f50790d4f 100644 --- a/.github/workflows/internal-java-code-analysis.yml +++ b/.github/workflows/internal-java-code-analysis.yml @@ -48,6 +48,7 @@ jobs: analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }} sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }} artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }} + additional-maven-artifacts: ${{ steps.set-additional-maven-artifacts.outputs.additional-maven-artifacts }} env: PROJECT_NAME: AxonFramework @@ -94,6 +95,10 @@ jobs: id: set-artifacts-upload-name run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT" + - name: (Prepare Code to Analyze) Set output variable 'additional-maven-artifacts' + id: set-additional-maven-artifacts + run: echo "additional-maven-artifacts=org.axonframework:axon-messaging:${{ env.AXON_FRAMEWORK_VERSION }},org.axonframework:axon-modelling:${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT" + - name: (Prepare Code to Analyze) Upload sources to analyze if: success() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -120,6 +125,10 @@ jobs: uses: ./.github/workflows/public-analyze-code-graph.yml with: analysis-name: ${{ needs.prepare-code-to-analyze.outputs.analysis-name }} + # All necessary artifacts are contained in the uploaded artifacts. + # This could easily be replaced by maven-artifact parameter below, but remains here for testing purposes. artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }} + # Additional (duplicate) artifacts are only used here to test maven-artifacts parameter. + maven-artifacts: ${{needs.prepare-code-to-analyze.outputs.additional-maven-artifacts}} sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }} jupyter-pdf: "false" \ No newline at end of file diff --git a/.github/workflows/public-analyze-code-graph.yml b/.github/workflows/public-analyze-code-graph.yml index 856c96935..f6672003d 100644 --- a/.github/workflows/public-analyze-code-graph.yml +++ b/.github/workflows/public-analyze-code-graph.yml @@ -18,6 +18,14 @@ on: required: false type: string default: '' + maven-artifacts: + description: > + Comma-separated list of Maven coordinates (groupId:artifactId:version) + to download from Maven Central for the analysis. + Example: 'org.apache.commons:commons-lang3:3.12.0,com.google.guava:guava:31.1-jre' + required: false + type: string + default: '' sources-upload-name: description: > The name of the sources uploaded with 'actions/upload-artifact' @@ -87,16 +95,15 @@ jobs: python: 3.12 miniforge: 24.9.0-0 steps: - - name: Assure that either artifacts-upload-name or sources-upload-name is set - if: inputs.artifacts-upload-name == '' && inputs.sources-upload-name == '' - run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'sources-upload-name'."; exit 1 + - name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name is set + if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == '' && inputs.sources-upload-name == '' + run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name'."; exit 1 - name: Verify analysis-name only consists of characters safe for folder names run: | if [[ ! "${{ inputs.analysis-name }}" =~ ^[A-Za-z0-9._-]+$ ]]; then echo "The analysis-name '${{ inputs.analysis-name }}' contains invalid characters. Only alphanumeric characters, dots (.), underscores (_) and hyphens (-) are allowed." exit 1 fi - - name: Assemble ENVIRONMENT_INFO run: echo "ENVIRONMENT_INFO=java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}" >> $GITHUB_ENV @@ -176,6 +183,11 @@ jobs: name: ${{ inputs.artifacts-upload-name }} path: temp/${{ inputs.analysis-name }}/artifacts + - name: (Code Analysis Setup) Download Maven artifacts for analysis + if: inputs.maven-artifacts != '' + working-directory: temp/${{ inputs.analysis-name }} + run: ./../../scripts/downloadMavenArtifacts.sh "${{ inputs.maven-artifacts }}" + - name: (Debug) Log folder structure of temp directory if: runner.debug == '1' working-directory: temp diff --git a/INTEGRATION.md b/INTEGRATION.md index 4276aef60..e34bd281d 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -33,6 +33,7 @@ The workflow parameters are as follows: - **analysis-name**: The name of the project to analyze. Example: MyProject-1.0.0. This parameter is required and should be a string. - **artifacts-upload-name**: The name of the artifacts uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'artifacts' directory for the analysis. This is used to analyze Java JARs, WARs, EARs, etc. This parameter is optional and defaults to an empty string. +- **maven-artifacts**: Comma separated list of Maven artifact coordinates (groupId:artifactId:version) to download from Maven Central for the analysis. This is used to analyze Java artifacts without having to upload them as build artifacts. This parameter is optional and defaults to an empty string. - **sources-upload-name**: The name of the sources uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'source' directory for the analysis. It also supports sub-folders for multiple source code bases. This parameter is optional and defaults to an empty string. Please use 'include-hidden-files: true' if you also want to upload the git history. - **ref**: The branch, tag, or SHA of the code-graph-analysis-pipeline to checkout. This parameter is optional and defaults to "main". diff --git a/scripts/downloadMavenArtifacts.sh b/scripts/downloadMavenArtifacts.sh new file mode 100755 index 000000000..996c58379 --- /dev/null +++ b/scripts/downloadMavenArtifacts.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# Uses Maven to download specified Maven artifacts from Maven Central. +# The artifacts are specified in the first argument as comma separated Maven coordinates. +# Details on the Maven coordinates format: https://maven.apache.org/guides/mini/guide-naming-conventions.html +# The downloaded files are written into the "artifacts" directory of the current analysis directory. + +# This script is used inside .github/workflows/public-analyze-code-graph.yml (November 2025) + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +# Overrideable Constants (defaults also defined in sub scripts) +LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} +LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} +ARTIFACTS_DIRECTORY=${ARTIFACTS_DIRECTORY:-"artifacts"} + +# Local constants +SCRIPT_NAME=$(basename "${0}") + +fail() { + local ERROR_COLOR='\033[0;31m' # red + local DEFAULT_COLOR='\033[0m' + local errorMessage="${1}" + echo -e "${ERROR_COLOR}${SCRIPT_NAME}: Error: ${errorMessage}${DEFAULT_COLOR}" >&2 + exit 1 +} + +maven_artifacts="" +dry_run=false + +# Input Arguments: Parse the command-line arguments +while [[ $# -gt 0 ]]; do + arg="$1" + case $arg in + --dry-run) + dry_run=true + shift + ;; + *) + if [ -z "${maven_artifacts}" ]; then + # The first unnamed input argument contains the comma separated Maven artifact coordinates to download + maven_artifacts="${arg}" + #echo "${SCRIPT_NAME}: maven_artifacts: ${maven_artifacts}" + else + fail "Unknown argument: ${arg}" + fi + shift + ;; + esac +done + +if [ -z "${maven_artifacts}" ]; then + fail "No Maven artifacts specified to download. Please provide a comma-separated list of Maven coordinates (groupId:artifactId:version)." +fi + +if [ ! -d "./${ARTIFACTS_DIRECTORY}" ]; then + fail "This script needs to run inside the analysis directory with an already existing artifacts directory in it. Change into that directory or use ./init.sh to set up an analysis." +fi + +if ! command -v "mvn" &> /dev/null ; then + fail "Command mvn (Maven) not found. It's needed to download Maven artifacts from Maven Central." +fi + +dry_run_info="" +if [ "${dry_run}" = true ] ; then + echo "${SCRIPT_NAME}: Info: Dry run mode enabled." + dry_run_info=" (dry run)" +fi + +# Process each Maven artifact coordinate +echo "${maven_artifacts}" | tr ',' '\n' | while read -r maven_artifact; do + maven_artifact=$(echo "$maven_artifact" | xargs) + + # Check if the maven artifact "coordinate" contains exactly two colons + colon_count=$(echo "${maven_artifact}" | tr -cd ':' | wc -c) + if [ "${colon_count}" -ne 2 ]; then + fail "Invalid Maven artifact coordinates: '${maven_artifact}'. It should be in the format 'groupId:artifactId:version'." + fi + + echo "${LOG_GROUP_START}Downloading Maven artifact ${maven_artifact}${dry_run_info}" + if [ "${dry_run}" = false ] ; then + mvn --quiet dependency:copy -Dartifact="${maven_artifact}" -DoutputDirectory="./${ARTIFACTS_DIRECTORY}" -Dtransitive=false -Dsilent=true + fi + echo "${LOG_GROUP_END}" +done \ No newline at end of file diff --git a/scripts/testDownloadMavenArtifacts.sh b/scripts/testDownloadMavenArtifacts.sh new file mode 100755 index 000000000..c923c9ab2 --- /dev/null +++ b/scripts/testDownloadMavenArtifacts.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +# Tests "downloadMavenArtifacts.sh". + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +# Local constants +SCRIPT_NAME=$(basename "${0}") +COLOR_ERROR='\033[0;31m' # red +COLOR_DE_EMPHASIZED='\033[0;90m' # dark gray +COLOR_SUCCESSFUL="\033[0;32m" # green +COLOR_DEFAULT='\033[0m' + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts + +tearDown() { + # echo "${SCRIPT_NAME}: Tear down tests...." + rm -rf "${temporaryTestDirectory}" +} + +successful() { + echo "" + echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}:${COLOR_DEFAULT} ${COLOR_SUCCESSFUL}✅ Tests finished successfully.${COLOR_DEFAULT}" + tearDown +} + +info() { + local infoMessage="${1}" + echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}:${COLOR_DEFAULT} ${infoMessage}" +} + +fail() { + local errorMessage="${1}" + echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}" + tearDown + return 1 +} + +printTestLogFileContent() { + local logFileContent=$( cat "${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log" ) + # Remove color codes from the output for better readability in test logs + logFileContent=$(echo -e "${logFileContent}" | sed -r "s/\x1B\[[0-9;]*[mK]//g") + echo -e "${COLOR_DE_EMPHASIZED}${logFileContent}${COLOR_DEFAULT}" +} + +downloadMavenArtifactsExpectingSuccessUnderTest() { + local COLOR_DE_EMPHASIZED='\033[0;90m' # dark gray + ( + cd "${temporaryTestDirectory}"; + source "${SCRIPTS_DIR}/downloadMavenArtifacts.sh" "$@" >"${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log" 2>&1 + ) + exitCode=$? + if [ ${exitCode} -ne 0 ]; then + fail "❌ Test failed: Script exited with non-zero exit code ${exitCode}." + fi + printTestLogFileContent +} + +downloadMavenArtifactsExpectingFailureUnderTest() { + set +o errexit + ( + cd "${temporaryTestDirectory}"; + source "${SCRIPTS_DIR}/downloadMavenArtifacts.sh" "$@" >"${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log" 2>&1 + exitCode=$? + if [ ${exitCode} -eq 0 ]; then + fail "❌ Test failed: Script exited with zero exit code but was expected to fail." + fi + ) + set -o errexit + printTestLogFileContent +} + +info "Starting tests...." + +# Create testing resources +temporaryTestDirectory=$(mktemp -d 2>/dev/null || mktemp -d -t 'temporaryTestDirectory_${SCRIPT_NAME}') +mkdir -p "${temporaryTestDirectory}/artifacts" + +# ------- Integration Test Case +test_case_number=1 +echo "" +info "${test_case_number}.) Should download existing Maven artifacts with correct coordinates successfully (real-run/integration)." + +output=$(downloadMavenArtifactsExpectingSuccessUnderTest "org.apache.commons:commons-lang3:3.12.0,com.google.guava:guava:31.1-jre") +if [ ! -f "${temporaryTestDirectory}/artifacts/commons-lang3-3.12.0.jar" ]; then + fail "${test_case_number}.) Test failed: Expected artifact 'commons-lang3-3.12.0.jar' not found in artifacts directory." +fi +if [ ! -f "${temporaryTestDirectory}/artifacts/guava-31.1-jre.jar" ]; then + fail "${test_case_number}.) Test failed: Expected artifact 'guava-31.1-jre.jar' not found in artifacts directory." +fi + +# ------- Integration Test Case +test_case_number=2 +echo "" +info "${test_case_number}.) Should download a single Maven artifact successfully even if it had already been downloaded (real-run/integration)." + +output=$(downloadMavenArtifactsExpectingSuccessUnderTest "org.apache.commons:commons-lang3:3.12.0") +if [ ! -f "${temporaryTestDirectory}/artifacts/commons-lang3-3.12.0.jar" ]; then + fail "${test_case_number}.) Test failed: Expected artifact 'commons-lang3-3.12.0.jar' not found in artifacts directory." +fi + +# ------- Integration Test Case +test_case_number=3 +echo "" +info "${test_case_number}.) Should fail when downloading non-existing Maven artifact (real-run/integration)." +downloadMavenArtifactsExpectingFailureUnderTest "org.nonexistent:nonexistent-artifact:0.0.1" + +# ------- Unit Test Case +test_case_number=4 +echo "" +info "${test_case_number}.) Should fail when no input is specified (dry-run)." +downloadMavenArtifactsExpectingFailureUnderTest "--dry-run" + +# ------- Unit Test Case +test_case_number=5 +echo "" +info "${test_case_number}.) Should fail when input is empty (dry-run)." +downloadMavenArtifactsExpectingFailureUnderTest "--dry-run" + +# ------- Unit Test Case +test_case_number=6 +echo "" +info "${test_case_number}.) Should fail on unknown arguments." +downloadMavenArtifactsExpectingFailureUnderTest "--dry-run --unknown-argument" + +# ------- Unit Test Case +test_case_number=7 +echo "" +info "${test_case_number}.) Should fail when artifacts directory is missing (dry-run)." +# Rename artifacts directory to simulate missing directory +mv "${temporaryTestDirectory}/artifacts" "${temporaryTestDirectory}/artifacts_backup" + +downloadMavenArtifactsExpectingFailureUnderTest "--dry-run" + +# Restore artifacts directory +mv "${temporaryTestDirectory}/artifacts_backup" "${temporaryTestDirectory}/artifacts" + +# ------- Unit Test Case +test_case_number=8 +echo "" +info "${test_case_number}.) Should fail when the artifact coordinate has a wrong format (dry-run)." +downloadMavenArtifactsExpectingFailureUnderTest "--dry-run" "org.apache.commons:commons-lang3-3.12.0" + +successful +return 0 \ No newline at end of file From ebb359bf53a657e7a58cc757cfedeeb1cc23d610 Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Wed, 3 Dec 2025 21:50:45 +0100 Subject: [PATCH 3/5] Add input parameter for source repository cloning --- .../workflows/internal-java-code-analysis.yml | 24 +-- .../workflows/public-analyze-code-graph.yml | 33 +++- INTEGRATION.md | 3 + scripts/cloneGitRepository.sh | 127 +++++++++++++++ scripts/downloader/downloadAxonFramework.sh | 12 +- scripts/testCloneGitRepository.sh | 146 ++++++++++++++++++ 6 files changed, 323 insertions(+), 22 deletions(-) create mode 100755 scripts/cloneGitRepository.sh create mode 100755 scripts/testCloneGitRepository.sh diff --git a/.github/workflows/internal-java-code-analysis.yml b/.github/workflows/internal-java-code-analysis.yml index f50790d4f..aaa2bf9ea 100644 --- a/.github/workflows/internal-java-code-analysis.yml +++ b/.github/workflows/internal-java-code-analysis.yml @@ -46,9 +46,9 @@ jobs: runs-on: ubuntu-latest outputs: analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }} - sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }} artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }} additional-maven-artifacts: ${{ steps.set-additional-maven-artifacts.outputs.additional-maven-artifacts }} + source-repository-branch: ${{ steps.set-source-repository-branch.outputs.source-repository-branch }} env: PROJECT_NAME: AxonFramework @@ -76,7 +76,7 @@ jobs: mkdir -p ${{ steps.set-analysis-name.outputs.analysis-name }} cd ${{ steps.set-analysis-name.outputs.analysis-name }} echo "Working directory: $( pwd -P )" - ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} + ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} --skip-clone - name: Debug folder structure in temp directory if: runner.debug == '1' @@ -87,10 +87,6 @@ jobs: - name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV - - name: (Prepare Code to Analyze) Set sources-upload-name - id: set-sources-upload-name - run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT" - - name: (Prepare Code to Analyze) Set output variable 'artifacts-upload-name' id: set-artifacts-upload-name run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT" @@ -99,15 +95,9 @@ jobs: id: set-additional-maven-artifacts run: echo "additional-maven-artifacts=org.axonframework:axon-messaging:${{ env.AXON_FRAMEWORK_VERSION }},org.axonframework:axon-modelling:${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT" - - name: (Prepare Code to Analyze) Upload sources to analyze - if: success() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }} - path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source - include-hidden-files: true - if-no-files-found: error - retention-days: 1 + - name: (Prepare Code to Analyze) Set output variable 'source-repository-branch' + id: set-source-repository-branch + run: echo "source-repository-branch=axon-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT" - name: (Prepare Code to Analyze) Upload artifacts to analyze if: success() @@ -130,5 +120,7 @@ jobs: artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }} # Additional (duplicate) artifacts are only used here to test maven-artifacts parameter. maven-artifacts: ${{needs.prepare-code-to-analyze.outputs.additional-maven-artifacts}} - sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }} + source-repository: https://github.com/AxonFramework/AxonFramework.git + source-repository-branch: ${{ needs.prepare-code-to-analyze.outputs.source-repository-branch}} + source-repository-history-only: true jupyter-pdf: "false" \ No newline at end of file diff --git a/.github/workflows/public-analyze-code-graph.yml b/.github/workflows/public-analyze-code-graph.yml index f6672003d..c23d1233c 100644 --- a/.github/workflows/public-analyze-code-graph.yml +++ b/.github/workflows/public-analyze-code-graph.yml @@ -35,6 +35,28 @@ on: required: false type: string default: '' + source-repository: + description: > + The URL of the source repository to analyze. For now, only GitHub repositories are supported. + This can be used instead of 'sources-upload-name' to directly analyze a repository without uploading artifacts first. + It can also be used in addition to 'sources-upload-name' to analyze both uploaded sources and a repository. + required: false + type: string + default: '' + source-repository-branch: + description: > + The branch, tag or SHA of the source repository to checkout. + Default: default branch of the repository + required: false + type: string + default: '' + source-repository-history-only: + description: > + Whether to clone the source repository as a bare repository ("true") or not ("false", default). + Bare repositories do not have a working directory and are useful for git history analysis when the sources are not needed. + required: false + type: string + default: 'false' ref: description: > The branch, tag or SHA of the code-graph-analysis-pipeline to checkout. @@ -95,9 +117,9 @@ jobs: python: 3.12 miniforge: 24.9.0-0 steps: - - name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name is set - if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == '' && inputs.sources-upload-name == '' - run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name'."; exit 1 + - name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name or source-repository is set + if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == '' && inputs.sources-upload-name == '' && inputs.source-repository == '' + run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name' or 'source-repository'."; exit 1 - name: Verify analysis-name only consists of characters safe for folder names run: | if [[ ! "${{ inputs.analysis-name }}" =~ ^[A-Za-z0-9._-]+$ ]]; then @@ -176,6 +198,11 @@ jobs: name: ${{ inputs.sources-upload-name }} path: temp/${{ inputs.analysis-name }}/source/${{ inputs.analysis-name }} + - name: (Code Analysis Setup) Clone source repository for analysis + if: inputs.source-repository != '' + working-directory: temp/${{ inputs.analysis-name }} + run: ./../../scripts/cloneGitRepository.sh --url "${{ inputs.source-repository }}" --branch "${{ inputs.source-repository-branch }}" --history-only "${{ inputs.source-repository-history-only }}" --target "source/${{ inputs.analysis-name }}" + - name: (Code Analysis Setup) Download artifacts for analysis if: inputs.artifacts-upload-name != '' uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 diff --git a/INTEGRATION.md b/INTEGRATION.md index e34bd281d..ab25df86d 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -37,6 +37,9 @@ The workflow parameters are as follows: - **sources-upload-name**: The name of the sources uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'source' directory for the analysis. It also supports sub-folders for multiple source code bases. This parameter is optional and defaults to an empty string. Please use 'include-hidden-files: true' if you also want to upload the git history. - **ref**: The branch, tag, or SHA of the code-graph-analysis-pipeline to checkout. This parameter is optional and defaults to "main". +- **source-repository**: The URL of the source code repository to analyze. This parameter is optional and defaults to an empty string. If provided, it will be used to clone the repository for analysis instead of using the uploaded source code artifact. Currently. only public GitHub repositories are supported. +- **source-repository-branch**: The branch of the source code repository to analyze. This parameter is optional and defaults to "main". It is only used if the 'source-repository' parameter is provided. +- **source-repository-history-only**: If set to 'true', only the git history of the source code repository will be cloned for analysis. This parameter is optional and defaults to 'false'. It is only used if the 'source-repository' parameter is provided. - **analysis-arguments**: The arguments to pass to the analysis script. This parameter is optional and defaults to '--profile Neo4j-latest-low-memory'. You can find all available options in section [Command Line Options of COMMANDS.md/](./COMMANDS.md#command-line-options). - **typescript-scan-heap-memory**: The heap memory size in MB to use for the TypeScript code scans. This value is only used for the TypeScript code scans and is ignored for other scans. This parameter is optional and defaults to '4096'. It will set the environment variable `TYPESCRIPT_SCAN_HEAP_MEMORY` which leads to `NODE_OPTIONS` set to `--max-old-space-size=4096` for TypeScript scans. See [Questions and Answers of README.md](./README.md#thinking-questions--answers) for more information. diff --git a/scripts/cloneGitRepository.sh b/scripts/cloneGitRepository.sh new file mode 100755 index 000000000..8f119e164 --- /dev/null +++ b/scripts/cloneGitRepository.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +# Provides safe-guarded (security checked parameters) git repository cloning. + +# Note: This script needs the path to target directory to clone the git repository to. It defaults to SOURCE_DIRECTORY ("source"). +# Note: This script needs git to be installed. + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +# Overrideable Defaults +SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source") + +# Local constants +SCRIPT_NAME=$(basename "${0}") + +fail() { + local ERROR_COLOR='\033[0;31m' # red + local DEFAULT_COLOR='\033[0m' + local errorMessage="${1}" + echo -e "${ERROR_COLOR}${SCRIPT_NAME}: Error: ${errorMessage}${DEFAULT_COLOR}" >&2 + exit 1 +} + +# Default and initial values for command line options +url="" +branch="main" +history_only="false" +target="${SOURCE_DIRECTORY}" +dry_run="false" + +# Read command line options +USAGE="${SCRIPT_NAME}: Usage: $0 --url --branch [--history-only ] [--target (default=source)]" + +while [ "$#" -gt "0" ]; do + key="$1" + case ${key} in + --url) + url="$2" + shift + ;; + --branch) + branch="$2" + shift + ;; + --history-only) + history_only="$2" + shift + ;; + --target) + target="$2" + shift + ;; + --dry-run) + dry_run="true" + ;; + *) + fail "Unknown option: ${key}" + echo "${USAGE}" >&2 + exit 1 + esac + shift +done + +# --- Validate URL (mandatory) +if [ -z "${url}" ] ; then + fail "The git repository URL (--url) must be provided." + echo "${USAGE}" >&2 + exit 1 +fi +case "${url}" in + https://github.com/*/*|https://github.com/*/*.git) + ;; + *) + fail "The source repository (--url) must be a valid GitHub repository URL." + ;; +esac + +# --- Validate branch (mandatory, defaults to "main") +if [ -z "${branch}" ] ; then + fail "The git repository branch (--branch) must be provided." + echo "${USAGE}" >&2 + exit 1 +fi +case "${branch}" in + *[\ ~^:?*[\]\\]*) + fail "The source repository branch contains invalid characters." + ;; +esac + +# --- Validate history-only (mandatory, defaults to "false") +case "${history_only}" in + true|false) + ;; + *) + fail "The source repository history-only option must be either 'true' or 'false'." + echo "${USAGE}" >&2 + ;; +esac + +# --- Validate target directory (mandatory, defaults to SOURCE_DIRECTORY) +if [ -z "${target}" ] ; then + fail "The target directory (--target) ${target} must be provided." >&2 + echo "${USAGE}" >&2 + exit 1 +else + mkdir -p "${target}" +fi + +if [ ${dry_run} = "true" ] ; then + echo "Dry run mode enabled. The following command(s) would be executed:" >&2 +fi + +# --- Clone the git repository +bare_option="" +bare_folder="" +if [ "${history_only}" = "true" ]; then + bare_option="--bare" + bare_folder="/.git" # bare clones need the .git folder to be used as target +fi + +if [ ${dry_run} = "true" ] ; then + echo "git clone ${bare_option} --single-branch ${url} --branch ${branch} ${target}${bare_folder}" + exit 0 +else + git clone ${bare_option} --single-branch "${url}" --branch "${branch}" "${target}${bare_folder}" +fi \ No newline at end of file diff --git a/scripts/downloader/downloadAxonFramework.sh b/scripts/downloader/downloadAxonFramework.sh index 63a9ee86a..4239fa249 100755 --- a/scripts/downloader/downloadAxonFramework.sh +++ b/scripts/downloader/downloadAxonFramework.sh @@ -22,13 +22,19 @@ echo "download${ANALYSIS_NAME}: SCRIPT_FILE_NAME_WITHOUT_EXTENSION=${SCRIPT_FILE echo "download${ANALYSIS_NAME}: ANALYSIS_NAME=${ANALYSIS_NAME}" # Read the first input argument containing the version(s) of the artifact(s) -if [ "$#" -ne 1 ]; then - echo "Error (download${ANALYSIS_NAME}): Usage: $0 (e.g. 4.9.3)" >&2 +if [ "$#" -eq 0 ]; then + echo "Error (download${ANALYSIS_NAME}): Usage: $0 (e.g. 4.9.3) [--skip-clone]" >&2 exit 1 fi ARTIFACTS_VERSION=$1 echo "download${ANALYSIS_NAME}: ARTIFACTS_VERSION=${ARTIFACTS_VERSION}" +skipClone=false +if [ "$2" = "--skip-clone" ]; then + skipClone=true + echo "download${ANALYSIS_NAME}: Git clone of source repository will be skipped." +fi + ## Get this "scripts/downloader" directory if not already set # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. @@ -55,7 +61,7 @@ source "${SCRIPTS_DIR}/downloadMavenArtifact.sh" -g "${ARTIFACTS_GROUP}" -a "axo # Download the git history (bare clone without working tree) into the "source" folder. # This makes it possible to additionally import the git log into the graph -if [ ! -d "${SOURCE_DIRECTORY}/AxonFramework-${ARTIFACTS_VERSION}/.git" ]; then +if [ ! -d "${SOURCE_DIRECTORY}/AxonFramework-${ARTIFACTS_VERSION}/.git" ] && [ "${skipClone}" = false ]; then echo "download${ANALYSIS_NAME}: Getting bare git history of source code repository..." git clone --bare https://github.com/AxonFramework/AxonFramework.git --branch "axon-${ARTIFACTS_VERSION}" --single-branch "${SOURCE_DIRECTORY}/AxonFramework-${ARTIFACTS_VERSION}/.git" fi diff --git a/scripts/testCloneGitRepository.sh b/scripts/testCloneGitRepository.sh new file mode 100755 index 000000000..730d0bcb9 --- /dev/null +++ b/scripts/testCloneGitRepository.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +# Tests "cloneGitRepository.sh". + +# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) +set -o errexit -o pipefail + +# Local constants +SCRIPT_NAME=$(basename "${0}") +COLOR_ERROR='\033[0;31m' # red +COLOR_DE_EMPHASIZED='\033[0;90m' # dark gray +COLOR_SUCCESSFUL="\033[0;32m" # green +COLOR_DEFAULT='\033[0m' + +## Get this "scripts" directory if not already set +# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. +# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. +# This way non-standard tools like readlink aren't needed. +SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts + +tearDown() { + # echo "${SCRIPT_NAME}: Tear down tests...." + rm -rf "${temporaryTestDirectory}" +} + +successful() { + echo "" + echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}:${COLOR_DEFAULT} ${COLOR_SUCCESSFUL}✅ Tests finished successfully.${COLOR_DEFAULT}" + tearDown +} + +info() { + local infoMessage="${1}" + echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}:${COLOR_DEFAULT} ${infoMessage}" +} + +fail() { + local errorMessage="${1}" + echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}" + tearDown + return 1 +} + +printTestLogFileContent() { + local logFileName="${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log" + if [ -f "${logFileName}" ]; then + local logFileContent=$( cat "${logFileName}" ) + # Remove color codes from the output for better readability in test logs + logFileContent=$(echo -e "${logFileContent}" | sed -r "s/\x1B\[[0-9;]*[mK]//g") + echo -e "${COLOR_DE_EMPHASIZED}${logFileContent}${COLOR_DEFAULT}" + else + echo -e "${COLOR_ERROR}No log file found at expected location: ${logFileName}${COLOR_DEFAULT}" + fi +} + +cloneGitRepositoryExpectingSuccessUnderTest() { + local COLOR_DE_EMPHASIZED='\033[0;90m' # dark gray + ( + cd "${temporaryTestDirectory}"; + source "${SCRIPTS_DIR}/cloneGitRepository.sh" "$@" > "${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log" + ) + exitCode=$? + if [ ${exitCode} -ne 0 ]; then + fail "❌ Test failed: Script exited with non-zero exit code ${exitCode}." + fi + printTestLogFileContent +} + +cloneGitRepositoryExpectingFailureUnderTest() { + set +o errexit + ( + cd "${temporaryTestDirectory}"; + source "${SCRIPTS_DIR}/cloneGitRepository.sh" "$@" > "${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log" 2>&1 + exitCode=$? + if [ ${exitCode} -eq 0 ]; then + fail "❌ Test failed: Script exited with zero exit code but was expected to fail." + fi + ) + set -o errexit + printTestLogFileContent +} + +info "Starting tests...." + +# Create testing resources +temporaryTestDirectory=$(mktemp -d 2>/dev/null || mktemp -d -t 'temporaryTestDirectory_${SCRIPT_NAME}') +mkdir -p "${temporaryTestDirectory}" + +# ------- Integration Test Case +test_case_number=1 +echo "" +info "${test_case_number}.) Should clone a valid GitHub Repository successfully (real-run/integration)." + +cloneGitRepositoryExpectingSuccessUnderTest --url "https://github.com/JohT/livecoding.git" --branch "main" --target "${temporaryTestDirectory}/livecoding" +if [ ! -f "${temporaryTestDirectory}/livecoding/README.md" ]; then + fail "${test_case_number}.) Test failed: Expected 'README.md' in cloned repository 'livecoding'." +fi + +# ------- Unit Test Case +test_case_number=2 +echo "" +info "${test_case_number}.) Should fail when an unknown option is used (dry-run)." +cloneGitRepositoryExpectingFailureUnderTest --non-existing-parameter --dry-run + +# ------- Unit Test Case +test_case_number=3 +echo "" +info "${test_case_number}.) Should fail when --url is from a different domain than GitHub (dry-run)." +cloneGitRepositoryExpectingFailureUnderTest --url "https://example.com/JohT/livecoding.git" --dry-run + +# ------- Unit Test Case +test_case_number=4 +echo "" +info "${test_case_number}.) Should fail when --branch is empty (dry-run)." +cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --branch "" --dry-run + +# ------- Unit Test Case +test_case_number=5 +echo "" +info "${test_case_number}.) Should fail when --branch contains invalid characters (dry-run)." +cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --branch "main;" --dry-run + +# ------- Unit Test Case +test_case_number=6 +echo "" +info "${test_case_number}.) Should fail when --history-only is neither true nor false (dry-run)." +cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --history-only "invalid" --dry-run + +# ------- Unit Test Case +test_case_number=7 +echo "" +info "${test_case_number}.) Should fail when --target is empty (dry-run)." +cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --target "" --dry-run + +# ------- Unit Test Case +test_case_number=8 +echo "" +info "${test_case_number}.) Should include the bare option in git clone when --history-option is true (dry-run)." +output=$(cloneGitRepositoryExpectingSuccessUnderTest --url "https://github.com/JohT/livecoding.git" --history-only "true" --target "${temporaryTestDirectory}/livecoding" --dry-run) + +if ! echo "${output}" | grep -q "git clone --bare"; then + fail "${test_case_number}.) Test failed: Expected '--bare' option in git clone command." +fi + +successful +return 0 \ No newline at end of file From c9c7c32cafdc35d007811f4cbd426209ae3eba7b Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Sun, 7 Dec 2025 09:51:39 +0100 Subject: [PATCH 4/5] Run tests in a dedicated workflow --- .../workflows/internal-java-code-analysis.yml | 4 --- .github/workflows/internal-run-tests.yml | 31 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/internal-run-tests.yml diff --git a/.github/workflows/internal-java-code-analysis.yml b/.github/workflows/internal-java-code-analysis.yml index aaa2bf9ea..f99e20fc6 100644 --- a/.github/workflows/internal-java-code-analysis.yml +++ b/.github/workflows/internal-java-code-analysis.yml @@ -59,10 +59,6 @@ jobs: - name: Checkout GIT Repository uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - name: Run script tests - id: script-tests - run: ./scripts/runTests.sh - - name: Set Set output variable 'analysis-name' id: set-analysis-name run: echo "analysis-name=${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/internal-run-tests.yml b/.github/workflows/internal-run-tests.yml new file mode 100644 index 000000000..c823015cc --- /dev/null +++ b/.github/workflows/internal-run-tests.yml @@ -0,0 +1,31 @@ +name: Run Tests + +on: + push: + branches: + - main + # Only watch changes in scripts, json files containing test data and this workflow for push events + paths: + - '**/*.sh' + - '**/test*.*' + - '.github/workflows/internal-run-tests.yml' + pull_request: + branches: + - main + # Only watch changes in scripts, json files containing test data and this workflow for push events + paths: + - '**/*.sh' + - '**/test*.*' + - '.github/workflows/internal-run-tests.yml' + +jobs: + run-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout GIT Repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Run script tests + id: script-tests + run: ./scripts/runTests.sh \ No newline at end of file From 3d9d1a66cb3240563e1dce910a7bd2b105dfcaba Mon Sep 17 00:00:00 2001 From: JohT <7671054+JohT@users.noreply.github.com> Date: Sun, 7 Dec 2025 10:49:01 +0100 Subject: [PATCH 5/5] Improve grouped report logging --- scripts/reports/compilations/AllReports.sh | 2 +- scripts/reports/compilations/CsvReports.sh | 19 ++++++++----- .../reports/compilations/JupyterReports.sh | 19 +++++++------ .../reports/compilations/MarkdownReports.sh | 24 +++++++++-------- scripts/reports/compilations/PythonReports.sh | 27 +++++++++++-------- .../compilations/VisualizationReports.sh | 20 +++++++------- 6 files changed, 65 insertions(+), 46 deletions(-) diff --git a/scripts/reports/compilations/AllReports.sh b/scripts/reports/compilations/AllReports.sh index 75e5d3d17..34b1f5f6d 100755 --- a/scripts/reports/compilations/AllReports.sh +++ b/scripts/reports/compilations/AllReports.sh @@ -13,7 +13,7 @@ set -o errexit -o pipefail # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} -echo "AllReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" +#echo "AllReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" # The reports will not be generically searched as files anymore. # Instead, they will be processed in order. Especially the visualization diff --git a/scripts/reports/compilations/CsvReports.sh b/scripts/reports/compilations/CsvReports.sh index 66324bd81..cf49e43e7 100755 --- a/scripts/reports/compilations/CsvReports.sh +++ b/scripts/reports/compilations/CsvReports.sh @@ -12,22 +12,29 @@ set -o errexit -o pipefail LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command. LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command. +# Local constants +SCRIPT_NAME=$(basename "${0}") + ## Get this "scripts/reports/compilations" directory if not already set. # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")} -echo "CsvReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" # Get the "domains" directory that contains analysis and report scripts by functionality. DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY:-"${REPORTS_SCRIPT_DIR}/../../domains"} -echo "CsvReports: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" + +echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Initialize CSV Reports"; +echo "${SCRIPT_NAME}: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" +echo "${LOG_GROUP_END}"; # Run all CSV report scripts (filename ending with Csv.sh) in the REPORTS_SCRIPT_DIR and DOMAINS_DIRECTORY directories. for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do if [ ! -d "${directory}" ]; then - echo "CsvReports: Error: Directory ${directory} does not exist. Please check your REPORTS_SCRIPT_DIR and DOMAIN_DIRECTORY settings." + echo "${SCRIPT_NAME}: Error: Directory ${directory} does not exist. Please check your REPORTS_SCRIPT_DIR and DOMAIN_DIRECTORY settings." exit 1 fi @@ -36,12 +43,12 @@ for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do report_script_filename=$(basename -- "${report_script_file}"); report_script_filename="${report_script_filename%.*}" # Remove file extension - echo "${LOG_GROUP_START}Create CSV Report ${report_script_filename}"; - echo "CsvReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${report_script_filename}..."; + echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Create CSV Report ${report_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Starting ${report_script_filename}..."; source "${report_script_file}" - echo "CsvReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${report_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Finished ${report_script_filename}"; echo "${LOG_GROUP_END}"; done done diff --git a/scripts/reports/compilations/JupyterReports.sh b/scripts/reports/compilations/JupyterReports.sh index 1e345116b..1fb8cd965 100755 --- a/scripts/reports/compilations/JupyterReports.sh +++ b/scripts/reports/compilations/JupyterReports.sh @@ -15,6 +15,9 @@ set -o errexit -o pipefail LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command. LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command. +# Local constants +SCRIPT_NAME=$(basename "${0}") + ## Get this "scripts/reports/compilations" directory if not already set. # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. @@ -26,11 +29,11 @@ SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${REPORTS_SCRIPT_DIR}")} # Get the "jupyter" directory by taking the path of the scripts directory, going up one directory and change then into "jupyter". JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY:-"${SCRIPTS_DIR}/../jupyter"} # Repository directory containing the Jupyter Notebooks -echo "${LOG_GROUP_START}Initialize Jupyter Notebook Reports"; -echo "JupyterReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" -echo "JupyterReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" -echo "JupyterReports: SCRIPTS_DIR=${SCRIPTS_DIR}" -echo "JupyterReports: JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY}" +echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Initialize Jupyter Notebook Reports"; +echo "${SCRIPT_NAME}: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}" +echo "${SCRIPT_NAME}: JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY}" echo "${LOG_GROUP_END}"; # Run all jupiter notebooks @@ -38,11 +41,11 @@ for jupyter_notebook_file in "${JUPYTER_NOTEBOOK_DIRECTORY}"/*.ipynb; do jupyter_notebook_filename=$(basename -- "${jupyter_notebook_file}") jupyter_notebook_filename_without_extension="${jupyter_notebook_filename%.*}" # Remove file extension - echo "${LOG_GROUP_START}Create Jupyter Notebook Report ${jupyter_notebook_filename_without_extension}"; - echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${jupyter_notebook_filename_without_extension}..."; + echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Create Jupyter Notebook Report ${jupyter_notebook_filename_without_extension}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Starting ${jupyter_notebook_filename_without_extension}..."; source "${SCRIPTS_DIR}/executeJupyterNotebookReport.sh" --jupyterNotebook "${jupyter_notebook_filename}" - echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${jupyter_notebook_filename_without_extension}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Finished ${jupyter_notebook_filename_without_extension}"; echo "${LOG_GROUP_END}"; done diff --git a/scripts/reports/compilations/MarkdownReports.sh b/scripts/reports/compilations/MarkdownReports.sh index 2b079258e..812d6d1f6 100755 --- a/scripts/reports/compilations/MarkdownReports.sh +++ b/scripts/reports/compilations/MarkdownReports.sh @@ -12,27 +12,29 @@ set -o errexit -o pipefail LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command. LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command. +# Local constants +SCRIPT_NAME=$(basename "${0}") + ## Get this "scripts/reports/compilations" directory if not already set. # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} -echo "MarkdownReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" - REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")} -echo "MarkdownReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" - -SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${REPORTS_SCRIPT_DIR}")} -echo "MarkdownReports: SCRIPTS_DIR=${SCRIPTS_DIR}" # Get the "domains" directory that contains analysis and report scripts by functionality. DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY:-"${REPORTS_SCRIPT_DIR}/../../domains"} -echo "MarkdownReports: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" + +echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Initialize Markdown Reports"; +echo "${SCRIPT_NAME}: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" +echo "${LOG_GROUP_END}"; # Run all Markdown report scripts (filename ending with Markdown.sh or Summary.sh) in the REPORTS_SCRIPT_DIR and DOMAINS_DIRECTORY directories. for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do if [ ! -d "${directory}" ]; then - echo "MarkdownReports: Error: Directory ${directory} does not exist. Please check your REPORTS_SCRIPT_DIR and DOMAIN_DIRECTORY settings." + echo "${SCRIPT_NAME}: Error: Directory ${directory} does not exist. Please check your REPORTS_SCRIPT_DIR and DOMAIN_DIRECTORY settings." exit 1 fi @@ -41,12 +43,12 @@ for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do report_script_filename=$(basename -- "${report_script_file}"); report_script_filename="${report_script_filename%.*}" # Remove file extension - echo "${LOG_GROUP_START}Create Markdown Report ${report_script_filename}"; - echo "MarkdownReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${report_script_filename}..."; + echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Create Markdown Report ${report_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Starting ${report_script_filename}..."; source "${report_script_file}" - echo "MarkdownReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${report_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Finished ${report_script_filename}"; echo "${LOG_GROUP_END}"; done done diff --git a/scripts/reports/compilations/PythonReports.sh b/scripts/reports/compilations/PythonReports.sh index 730736916..d2e02425a 100755 --- a/scripts/reports/compilations/PythonReports.sh +++ b/scripts/reports/compilations/PythonReports.sh @@ -3,7 +3,7 @@ # Runs all Python report scripts (no Chromium required). # It only considers scripts in the "reports" and "domains" directories and their sub directories (overridable with REPORTS_SCRIPT_DIR and DOMAINS_DIRECTORY). -# Requires reports/*.sh +# Requires activateCondaEnvironment.sh, activatePythonEnvironment.sh, reports/*.sh # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands) set -o errexit -o pipefail @@ -12,22 +12,25 @@ set -o errexit -o pipefail LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command. LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command. +# Local constants +SCRIPT_NAME=$(basename "${0}") + ## Get this "scripts/reports/compilations" directory if not already set. # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} -echo "PythonReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" - REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")} -echo "PythonReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" - SCRIPTS_DIR=${SCRIPTS_DIR:-$(dirname -- "${REPORTS_SCRIPT_DIR}")} -echo "PythonReports: SCRIPTS_DIR=${SCRIPTS_DIR}" # Get the "domains" directory that contains analysis and report scripts by functionality. DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY:-"${REPORTS_SCRIPT_DIR}/../../domains"} -echo "PythonReports: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" + +echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Initialize Python Reports"; +echo "${SCRIPT_NAME}: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}" +echo "${SCRIPT_NAME}: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" # Create and activate (if necessary) a virtual environment (Conda or venv). # For Conda, the environment name is taken from the environment variable CODEGRAPH_CONDA_ENVIRONMENT (default "codegraph") @@ -36,10 +39,12 @@ echo "PythonReports: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" time source "${SCRIPTS_DIR}/activateCondaEnvironment.sh" time source "${SCRIPTS_DIR}/activatePythonEnvironment.sh" +echo "${LOG_GROUP_END}"; + # Run all Python report scripts (filename ending with Csv.sh) in the REPORTS_SCRIPT_DIR and DOMAINS_DIRECTORY directories. for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do if [ ! -d "${directory}" ]; then - echo "PythonReports: Error: Directory ${directory} does not exist. Please check your REPORTS_SCRIPT_DIR and DOMAIN_DIRECTORY settings." + echo "${SCRIPT_NAME}: Error: Directory ${directory} does not exist. Please check your REPORTS_SCRIPT_DIR and DOMAIN_DIRECTORY settings." exit 1 fi @@ -48,12 +53,12 @@ for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do report_script_filename=$(basename -- "${report_script_file}"); report_script_filename="${report_script_filename%.*}" # Remove file extension - echo "${LOG_GROUP_START}Create Python Report ${report_script_filename}"; - echo "PythonReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${report_script_filename}..."; + echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Create Python Report ${report_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Starting ${report_script_filename}..."; source "${report_script_file}" - echo "PythonReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${report_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Finished ${report_script_filename}"; echo "${LOG_GROUP_END}"; done done diff --git a/scripts/reports/compilations/VisualizationReports.sh b/scripts/reports/compilations/VisualizationReports.sh index 4996a457e..dcb5ada27 100755 --- a/scripts/reports/compilations/VisualizationReports.sh +++ b/scripts/reports/compilations/VisualizationReports.sh @@ -15,6 +15,9 @@ set -o errexit -o pipefail LOG_GROUP_START=${LOG_GROUP_START:-"::group::"} # Prefix to start a log group. Defaults to GitHub Actions log group start command. LOG_GROUP_END=${LOG_GROUP_END:-"::endgroup::"} # Prefix to end a log group. Defaults to GitHub Actions log group end command. +# Local constants +SCRIPT_NAME=$(basename "${0}") + ## Get this "scripts/reports/compilations" directory if not already set. # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. @@ -23,12 +26,11 @@ REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR:-$( CDPATH=. cd REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$(dirname -- "${REPORT_COMPILATIONS_SCRIPT_DIR}")} DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY:-"${REPORTS_SCRIPT_DIR}/../../domains"} -# For detailed debug output uncomment the following lines: -#echo "${LOG_GROUP_START}Initialize Visualization Reports"; -#echo "VisualizationReports: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" -#echo "VisualizationReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" -#echo "VisualizationReports: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" -#echo "${LOG_GROUP_END}"; +echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Initialize Visualization Reports"; +echo "${SCRIPT_NAME}: REPORT_COMPILATIONS_SCRIPT_DIR=${REPORT_COMPILATIONS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: DOMAINS_DIRECTORY=${DOMAINS_DIRECTORY}" +echo "${LOG_GROUP_END}"; # Run all visualization scripts (filename ending with Visualization.sh) in the REPORTS_SCRIPT_DIR and DOMAINS_DIRECTORY directories. for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do @@ -42,12 +44,12 @@ for directory in "${REPORTS_SCRIPT_DIR}" "${DOMAINS_DIRECTORY}"; do visualization_script_filename=$(basename -- "${visualization_script_file}") visualization_script_filename="${visualization_script_filename%.*}" # Remove file extension - echo "${LOG_GROUP_START}Create Visualization Report ${visualization_script_filename}"; - echo "VisualizationReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${visualization_script_filename}..."; + echo "${LOG_GROUP_START}$(date +'%Y-%m-%dT%H:%M:%S') Create Visualization Report ${visualization_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Starting ${visualization_script_filename}..."; source "${visualization_script_file}" - echo "VisualizationReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${visualization_script_filename}"; + echo "${SCRIPT_NAME}: $(date +'%Y-%m-%dT%H:%M:%S') Finished ${visualization_script_filename}"; echo "${LOG_GROUP_END}"; done done