Merge pull request #477 from JohT/feature/add-public-workflow-inputs-for-maven-artifacts-and-source-repository

Add input parameter for maven artifacts and git source repository to public workflow

Author: JohT

.github/workflows/internal-java-code-analysis.yml

@@ -46,8 +46,9 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }}
-      sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
       artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }}
+      additional-maven-artifacts: ${{ steps.set-additional-maven-artifacts.outputs.additional-maven-artifacts }}
+      source-repository-branch: ${{ steps.set-source-repository-branch.outputs.source-repository-branch }}
 
     env: 
       PROJECT_NAME: AxonFramework
@@ -58,10 +59,6 @@ jobs:
     - name: Checkout GIT Repository
       uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
 
-    - name: Run script tests
-      id: script-tests
-      run: ./scripts/runTests.sh
-
     - name: Set Set output variable 'analysis-name'
       id: set-analysis-name
       run: echo "analysis-name=${{ env.PROJECT_NAME }}-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
@@ -75,7 +72,7 @@ jobs:
         mkdir -p ${{ steps.set-analysis-name.outputs.analysis-name }}
         cd ${{ steps.set-analysis-name.outputs.analysis-name }}
         echo "Working directory: $( pwd -P )"
-        ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }}
+        ./../../scripts/downloader/downloadAxonFramework.sh ${{ env.AXON_FRAMEWORK_VERSION }} --skip-clone
 
     - name: Debug folder structure in temp directory
       if: runner.debug == '1'
@@ -86,23 +83,17 @@ jobs:
     - name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID
       run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV
 
-    - name: (Prepare Code to Analyze) Set sources-upload-name
-      id: set-sources-upload-name
-      run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
- 
     - name: (Prepare Code to Analyze) Set output variable 'artifacts-upload-name'
       id: set-artifacts-upload-name
       run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
 
-    - name: (Prepare Code to Analyze) Upload sources to analyze
-      if: success()
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-      with:
-        name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
-        path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source
-        include-hidden-files: true
-        if-no-files-found: error
-        retention-days: 1
+    - name: (Prepare Code to Analyze) Set output variable 'additional-maven-artifacts'
+      id: set-additional-maven-artifacts
+      run: echo "additional-maven-artifacts=org.axonframework:axon-messaging:${{ env.AXON_FRAMEWORK_VERSION }},org.axonframework:axon-modelling:${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
+
+    - name: (Prepare Code to Analyze) Set output variable 'source-repository-branch'
+      id: set-source-repository-branch
+      run: echo "source-repository-branch=axon-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
 
     - name: (Prepare Code to Analyze) Upload artifacts to analyze
       if: success()
@@ -120,6 +111,12 @@ jobs:
     uses: ./.github/workflows/public-analyze-code-graph.yml
     with:
       analysis-name: ${{ needs.prepare-code-to-analyze.outputs.analysis-name }}
+      # All necessary artifacts are contained in the uploaded artifacts. 
+      # This could easily be replaced by maven-artifact parameter below, but remains here for testing purposes.
       artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }}
-      sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }}
+      # Additional (duplicate) artifacts are only used here to test maven-artifacts parameter.
+      maven-artifacts: ${{needs.prepare-code-to-analyze.outputs.additional-maven-artifacts}}
+      source-repository: https://github.com/AxonFramework/AxonFramework.git
+      source-repository-branch: ${{ needs.prepare-code-to-analyze.outputs.source-repository-branch}}
+      source-repository-history-only: true
       jupyter-pdf: "false"

.github/workflows/internal-run-tests.yml

@@ -0,0 +1,31 @@
+name: Run Tests
+
+on:
+  push:
+    branches:
+      - main
+    # Only watch changes in scripts, json files containing test data and this workflow for push events
+    paths: 
+      - '**/*.sh'
+      - '**/test*.*'
+      - '.github/workflows/internal-run-tests.yml'
+  pull_request:
+    branches:
+      - main
+    # Only watch changes in scripts, json files containing test data and this workflow for push events
+    paths: 
+      - '**/*.sh'
+      - '**/test*.*'
+      - '.github/workflows/internal-run-tests.yml'
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout GIT Repository
+      uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
+    
+    - name: Run script tests
+      id: script-tests
+      run: ./scripts/runTests.sh

.github/workflows/public-analyze-code-graph.yml

@@ -18,6 +18,14 @@ on:
         required: false
         type: string
         default: ''
+      maven-artifacts:
+        description: >
+          Comma-separated list of Maven coordinates (groupId:artifactId:version) 
+          to download from Maven Central for the analysis.
+          Example: 'org.apache.commons:commons-lang3:3.12.0,com.google.guava:guava:31.1-jre'
+        required: false
+        type: string
+        default: ''
       sources-upload-name:
         description: >
           The name of the sources uploaded with 'actions/upload-artifact' 
@@ -27,6 +35,28 @@ on:
         required: false
         type: string
         default: ''
+      source-repository:
+        description: >
+          The URL of the source repository to analyze. For now, only GitHub repositories are supported.
+          This can be used instead of 'sources-upload-name' to directly analyze a repository without uploading artifacts first.
+          It can also be used in addition to 'sources-upload-name' to analyze both uploaded sources and a repository.
+        required: false
+        type: string
+        default: ''  
+      source-repository-branch:
+        description: >
+          The branch, tag or SHA of the source repository to checkout.
+          Default: default branch of the repository
+        required: false
+        type: string
+        default: ''
+      source-repository-history-only:
+        description: >
+          Whether to clone the source repository as a bare repository ("true") or not ("false", default).
+          Bare repositories do not have a working directory and are useful for git history analysis when the sources are not needed.
+        required: false
+        type: string
+        default: 'false'      
       ref:
         description: >
           The branch, tag or SHA of the code-graph-analysis-pipeline to checkout. 
@@ -87,10 +117,15 @@ jobs:
           python: 3.12
           miniforge: 24.9.0-0
     steps:
-      - name: Assure that either artifacts-upload-name or sources-upload-name is set
-        if: inputs.artifacts-upload-name == '' && inputs.sources-upload-name == ''
-        run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'sources-upload-name'."; exit 1
-
+      - name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name or source-repository is set
+        if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == ''  && inputs.sources-upload-name == '' && inputs.source-repository == ''
+        run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name' or 'source-repository'."; exit 1
+      - name: Verify analysis-name only consists of characters safe for folder names
+        run: |
+          if [[ ! "${{ inputs.analysis-name }}" =~ ^[A-Za-z0-9._-]+$ ]]; then
+            echo "The analysis-name '${{ inputs.analysis-name }}' contains invalid characters. Only alphanumeric characters, dots (.), underscores (_) and hyphens (-) are allowed."
+            exit 1
+          fi
       - name: Assemble ENVIRONMENT_INFO
         run: echo "ENVIRONMENT_INFO=java-${{ matrix.java }}-python-${{ matrix.python }}-miniforge-${{ matrix.miniforge }}" >> $GITHUB_ENV
 
@@ -163,13 +198,23 @@ jobs:
           name: ${{ inputs.sources-upload-name }}
           path: temp/${{ inputs.analysis-name }}/source/${{ inputs.analysis-name }}
 
+      - name: (Code Analysis Setup) Clone source repository for analysis
+        if: inputs.source-repository != ''
+        working-directory: temp/${{ inputs.analysis-name }}
+        run: ./../../scripts/cloneGitRepository.sh --url "${{ inputs.source-repository }}" --branch "${{ inputs.source-repository-branch }}" --history-only "${{ inputs.source-repository-history-only }}" --target "source/${{ inputs.analysis-name }}"
+
       - name: (Code Analysis Setup) Download artifacts for analysis
         if: inputs.artifacts-upload-name != ''
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
         with:
           name: ${{ inputs.artifacts-upload-name }}
           path: temp/${{ inputs.analysis-name }}/artifacts
 
+      - name: (Code Analysis Setup) Download Maven artifacts for analysis
+        if: inputs.maven-artifacts != ''
+        working-directory: temp/${{ inputs.analysis-name }}
+        run: ./../../scripts/downloadMavenArtifacts.sh "${{ inputs.maven-artifacts }}"
+
       - name: (Debug) Log folder structure of temp directory
         if: runner.debug == '1'
         working-directory: temp

INTEGRATION.md

@@ -33,9 +33,13 @@ The workflow parameters are as follows:
 
 - **analysis-name**: The name of the project to analyze. Example: MyProject-1.0.0. This parameter is required and should be a string.
 - **artifacts-upload-name**: The name of the artifacts uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'artifacts' directory for the analysis. This is used to analyze Java JARs, WARs, EARs, etc. This parameter is optional and defaults to an empty string.
+- **maven-artifacts**: Comma separated list of Maven artifact coordinates (groupId:artifactId:version) to download from Maven Central for the analysis. This is used to analyze Java artifacts without having to upload them as build artifacts. This parameter is optional and defaults to an empty string.
 - **sources-upload-name**: The name of the sources uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'source' directory for the analysis. It also supports sub-folders for multiple source code bases. This parameter is optional and defaults to an empty string.
 Please use 'include-hidden-files: true' if you also want to upload the git history.
 - **ref**: The branch, tag, or SHA of the code-graph-analysis-pipeline to checkout. This parameter is optional and defaults to "main".
+- **source-repository**: The URL of the source code repository to analyze. This parameter is optional and defaults to an empty string. If provided, it will be used to clone the repository for analysis instead of using the uploaded source code artifact. Currently. only public GitHub repositories are supported.
+- **source-repository-branch**: The branch of the source code repository to analyze. This parameter is optional and defaults to "main". It is only used if the 'source-repository' parameter is provided.
+- **source-repository-history-only**: If set to 'true', only the git history of the source code repository will be cloned for analysis. This parameter is optional and defaults to 'false'. It is only used if the 'source-repository' parameter is provided.
 - **analysis-arguments**: The arguments to pass to the analysis script. This parameter is optional and defaults to '--profile Neo4j-latest-low-memory'. You can find all available options in section [Command Line Options of COMMANDS.md/](./COMMANDS.md#command-line-options).
 - **typescript-scan-heap-memory**: The heap memory size in MB to use for the TypeScript code scans. This value is only used for the TypeScript code scans and is ignored for other scans. This parameter is optional and defaults to '4096'. It will set the environment variable `TYPESCRIPT_SCAN_HEAP_MEMORY` which leads to `NODE_OPTIONS` set to `--max-old-space-size=4096` for TypeScript scans. See [Questions and Answers of README.md](./README.md#thinking-questions--answers) for more information.
 

scripts/cloneGitRepository.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+
+# Provides safe-guarded (security checked parameters) git repository cloning.
+
+# Note: This script needs the path to target directory to clone the git repository to. It defaults to SOURCE_DIRECTORY ("source"). 
+# Note: This script needs git to be installed.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+# Overrideable Defaults
+SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
+
+# Local constants
+SCRIPT_NAME=$(basename "${0}")
+
+fail() {
+  local ERROR_COLOR='\033[0;31m' # red
+  local DEFAULT_COLOR='\033[0m'
+  local errorMessage="${1}"
+  echo -e "${ERROR_COLOR}${SCRIPT_NAME}: Error: ${errorMessage}${DEFAULT_COLOR}" >&2
+  exit 1
+}
+
+# Default and initial values for command line options
+url=""
+branch="main"
+history_only="false"
+target="${SOURCE_DIRECTORY}"
+dry_run="false"
+
+# Read command line options
+USAGE="${SCRIPT_NAME}: Usage: $0 --url <github-repository-url> --branch <branch-name> [--history-only <true|false>] [--target <clone directory>(default=source)]"
+
+while [ "$#" -gt "0" ]; do
+  key="$1"
+  case ${key} in
+    --url)
+      url="$2"
+      shift
+      ;;
+    --branch)
+      branch="$2"
+      shift
+      ;;
+    --history-only)
+      history_only="$2"
+      shift
+      ;;
+    --target)
+      target="$2"
+      shift
+      ;;
+    --dry-run)
+      dry_run="true"
+      ;;
+    *)
+      fail "Unknown option: ${key}"
+      echo "${USAGE}" >&2
+      exit 1
+  esac
+  shift
+done
+
+# --- Validate URL (mandatory)
+if [ -z "${url}" ] ; then
+  fail "The git repository URL (--url) must be provided."
+  echo "${USAGE}" >&2
+  exit 1
+fi
+case "${url}" in
+  https://github.com/*/*|https://github.com/*/*.git)
+    ;;
+  *)
+    fail "The source repository (--url) must be a valid GitHub repository URL."
+    ;;
+esac
+
+# --- Validate branch (mandatory, defaults to "main")
+if [ -z "${branch}" ] ; then
+  fail "The git repository branch (--branch) must be provided."
+  echo "${USAGE}" >&2
+  exit 1
+fi
+case "${branch}" in
+  *[\ ~^:?*[\]\\]*)
+    fail "The source repository branch contains invalid characters."
+    ;;
+esac
+
+# --- Validate history-only (mandatory, defaults to "false")
+case "${history_only}" in
+  true|false)
+    ;;
+  *)
+    fail "The source repository history-only option must be either 'true' or 'false'."
+    echo "${USAGE}" >&2
+    ;;
+esac
+
+# --- Validate target directory (mandatory, defaults to SOURCE_DIRECTORY)
+if [ -z "${target}" ] ; then
+  fail "The target directory (--target) ${target} must be provided." >&2
+  echo "${USAGE}" >&2
+  exit 1
+else
+  mkdir -p "${target}"
+fi
+
+if [ ${dry_run} = "true" ] ; then
+  echo "Dry run mode enabled. The following command(s) would be executed:" >&2
+fi
+
+# --- Clone the git repository
+bare_option=""
+bare_folder=""
+if [ "${history_only}" = "true" ]; then
+  bare_option="--bare"
+  bare_folder="/.git" # bare clones need the .git folder to be used as target
+fi
+
+if [ ${dry_run} = "true" ] ; then
+  echo "git clone ${bare_option} --single-branch ${url} --branch ${branch} ${target}${bare_folder}"
+  exit 0
+else
+  git clone ${bare_option} --single-branch  "${url}" --branch "${branch}" "${target}${bare_folder}"
+fi