test(act): add plugin-validator tests

.github/workflows/ci.yml

@@ -489,6 +489,7 @@ jobs:
         shell: bash
 
       - name: Replace plugin version
+        id: replace-plugin-version
         if: ${{ inputs.plugin-version-suffix != '' }}
         run: |
           package_json_path="$PLUGIN_DIRECTORY/package.json"
@@ -572,8 +573,8 @@ jobs:
           if [ -n "${PLUGIN_VALIDATOR_CONFIG}" ]; then
             # User-provided configuration content
             echo "Using provided plugin-validator configuration content."
-            PLUGIN_VALIDATOR_CONFIG_PATH=".plugin-validator.yaml"
-            echo "${PLUGIN_VALIDATOR_CONFIG}" > ${PLUGIN_VALIDATOR_CONFIG_PATH}
+            PLUGIN_VALIDATOR_CONFIG_PATH=$(mktemp)
+            echo "${PLUGIN_VALIDATOR_CONFIG}" > "${PLUGIN_VALIDATOR_CONFIG_PATH}"
           elif [ -n "${PLUGIN_VALIDATOR_CONFIG_PATH}" ]; then
             # User-provided configuration file path
             echo "Using plugin-validator configuration file at path: ${PLUGIN_VALIDATOR_CONFIG_PATH}"
@@ -583,8 +584,8 @@ jobs:
             fi
           else
             # Default hardcoded configuration
-            PLUGIN_VALIDATOR_CONFIG_PATH=".plugin-validator.yaml"
             echo "${PLUGIN_VALIDATOR_CONFIG_PATH} configuration file is missing. Providing a default one as fallback."
+            PLUGIN_VALIDATOR_CONFIG_PATH=$(mktemp)
             cat <<EOF > "${PLUGIN_VALIDATOR_CONFIG_PATH}"
           global:
             enabled: true
@@ -598,19 +599,28 @@ jobs:
           # will recurse into the plugin's directory (including node_modules).
           mkdir -p /tmp/empty
 
+          # Copy workspace to a new tempdir to avoid issues when mounting the folder when running with act
+          TEMP_WORKSPACE_DIR=$(mktemp -d)
+          cp -r "${PLUGIN_DIRECTORY}"/* "$TEMP_WORKSPACE_DIR/"
+
+          # Give a UUID to each container so multiple can be run in parallel when running with act
+          UUID=$(cat /proc/sys/kernel/random/uuid)
+
           # Do not run clamav because it takes too long
-          docker run --name=plugin-validator --pull=always \
-            -v "$PWD/${PLUGIN_VALIDATOR_CONFIG_PATH}:/workspace/.plugin-validator.yaml:ro" \
-            -v "$PWD/${PLUGIN_DIRECTORY}/dist-artifacts:/workspace/dist-artifacts:ro" \
-            -v "$PWD/${PLUGIN_DIRECTORY}:/workspace" \
+          docker run --name="plugin-validator-${UUID}" --pull=always --platform=linux/amd64 \
+            -v "${PLUGIN_VALIDATOR_CONFIG_PATH}:/workspace/.plugin-validator.yaml:ro" \
+            -v "${TEMP_WORKSPACE_DIR}/dist-artifacts:/workspace/dist-artifacts:ro" \
+            -v "${TEMP_WORKSPACE_DIR}:/workspace" \
             -v "/tmp/empty:/workspace/node_modules" \
             -e SKIP_CLAMAV=1 \
             grafana/plugin-validator-cli \
             -ghaOutput \
             -config=/workspace/.plugin-validator.yaml \
             -sourceCodeUri=file:///workspace \
             "/workspace/dist-artifacts/${UNIVERSAL_ZIP}"
-          exit "$(docker inspect plugin-validator --format='{{.State.ExitCode}}')"
+          EXIT_CODE=$(docker inspect "plugin-validator-${UUID}" --format='{{.State.ExitCode}}')
+          docker rm "plugin-validator-${UUID}"
+          exit "$EXIT_CODE"
         env:
           UNIVERSAL_ZIP: ${{ steps.universal-zip.outputs.zip }}
           PLUGIN_VALIDATOR_CONFIG: ${{ inputs.plugin-validator-config }}

Makefile

@@ -1,8 +1,11 @@
-.PHONY: clean-mockdata clean-node-modules clean-act-tmp mockdata-dist
+.PHONY: reset-mockdata clean-node-modules clean-act-tmp clean-lfs mockdata-dist mockdata-dist-artifacts mockdata
 
 clean-node-modules:
 	find tests -name node_modules -type d -prune -exec rm -rf '{}' +
 
+clean-dist:
+	find tests ! -path '*/act/*' -name dist -type d -prune -exec rm -rf '{}' +
+
 clean-act-tmp:
 	rm -rf /tmp/act-artifacts
 	rm -rf /tmp/act-cache
@@ -11,16 +14,25 @@ clean-act-tmp:
 clean-act-toolcache-volumes:
 	docker volume ls -q | grep "^act-toolcache-" | xargs docker volume rm
 
-clean: clean-node-modules clean-act-tmp clean-act-toolcache-volumes
+clean-lfs:
+	git lfs prune
+
+clean: clean-node-modules clean-dist clean-act-tmp clean-act-toolcache-volumes clean-lfs
 
-clean-mockdata:
+reset-mockdata:
 	rm -rf tests/act/mockdata/dist/*
+	rm -rf tests/act/mockdata/dist-artifacts-unsigned/*
 
-mockdata-dist: clean-mockdata
-	./scripts/mockdata-dist.sh simple-frontend
-	./scripts/mockdata-dist.sh simple-frontend-yarn
-	./scripts/mockdata-dist.sh simple-frontend-pnpm
-	./scripts/mockdata-dist.sh simple-backend
+mockdata-dist: reset-mockdata
+	for tc in $$(./scripts/find-tests.sh); do \
+		./scripts/mockdata-dist.sh $$tc; \
+	done
 	@echo All done!
 
+mockdata-dist-artifacts: mockdata-dist
+	for tc in $$(./scripts/find-tests.sh); do \
+		./scripts/mockdata-dist-artifacts.sh $$tc; \
+	done
+	@echo All done!
 
+mockdata: mockdata-dist-artifacts

actions/internal/plugins/package/package.sh

@@ -101,7 +101,7 @@ fi
 exe_basename=$(basename $exe)
 for file in $(find "$backend_folder" -type f -name "${exe_basename}_*"); do
     # Extract os+arch from the file name
-    os_arch=$(echo $(basename $file) | sed -E "s|${exe_basename}_(\w+)(.exe)?|\1|")
+    os_arch=$(echo $(basename $file) | sed -E "s|${exe_basename}_([a-zA-Z0-9_]+)(.exe)?|\1|")
 
     # Temporary folder for the zip file
     tmp=$(mktemp -d)

scripts/find-tests.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -euo pipefail
+
+# Prints all testdata plugins found in the tests/ folder to stdout, one per line.
+# The "act" folder is excluded since it contains the tests themselves.
+# Can be used to loop over all test plugins, like this:
+#
+#   for tc in $(./scripts/find-tests.sh); do
+#     echo $tc
+#   done
+
+cd tests
+find . -maxdepth 1 -type d ! -name '.' ! -name 'act' -print

scripts/mockdata-dist-artifacts.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <test-plugin-folder-name>"
+    exit 1
+fi
+
+echo "[$1] Preparing mockdata (dist-artifacts)"
+cd "$(dirname "$0")/.."
+
+
+mkdir -p "tests/act/mockdata/dist-artifacts-unsigned/$1"
+
+echo "[$1] Packaging os/arch ZIPs"
+# Will exit with 0 if the plugin has no backend
+# (in that case, there's no need for os/arch ZIPs, just universal)
+./actions/internal/plugins/package/package.sh "tests/act/mockdata/dist/$1" "tests/act/mockdata/dist-artifacts-unsigned/$1"
+
+echo "[$1] Packaging universal ZIPs"
+./actions/internal/plugins/package/package.sh -u "tests/act/mockdata/dist/$1" "tests/act/mockdata/dist-artifacts-unsigned/$1"

scripts/mockdata-dist.sh

@@ -22,7 +22,7 @@ if [ "$#" -ne 1 ]; then
     exit 1
 fi
 
-echo "[$1] Preparing mockdata"
+echo "[$1] Preparing mockdata (dist)"
 cd "$(dirname "$0")/.."
 cd tests/$1
 

tests/act/internal/act/act.go

@@ -58,10 +58,38 @@ type Runner struct {
 
 	// Verbose enables logging of JSON output from act back to stdout.
 	Verbose bool
+
+	// ContainerArchitecture is the architecture to use for act containers.
+	// By default, act uses the architecture of the host machine.
+	// This can be useful to force a specific platform when running on ARM Macs.
+	ContainerArchitecture string
+}
+
+// RunnerOption is a function that configures a Runner.
+type RunnerOption func(r *Runner)
+
+// WithVerbose enables or disables verbose logging of act output.
+func WithVerbose(verbose bool) RunnerOption {
+	return func(r *Runner) {
+		r.Verbose = verbose
+	}
+}
+
+// WithContainerArchitecture sets the container architecture to use for act.
+func WithContainerArchitecture(architecture string) RunnerOption {
+	return func(r *Runner) {
+		r.ContainerArchitecture = architecture
+	}
+}
+
+// WithLinuxAMD64ContainerArchitecture sets the container architecture to linux/amd64.
+// This is useful when running on ARM Macs to ensure compatibility with x64 images.
+func WithLinuxAMD64ContainerArchitecture() RunnerOption {
+	return WithContainerArchitecture("linux/amd64")
 }
 
 // NewRunner creates a new Runner instance.
-func NewRunner(t *testing.T) (*Runner, error) {
+func NewRunner(t *testing.T, opts ...RunnerOption) (*Runner, error) {
 	// Get GitHub token from environment (GHA) or gh CLI (local)
 	ghToken, ok := os.LookupEnv("GITHUB_TOKEN")
 	if !ok || ghToken == "" {
@@ -86,6 +114,12 @@ func NewRunner(t *testing.T) (*Runner, error) {
 	if err != nil {
 		return nil, fmt.Errorf("new gcs: %w", err)
 	}
+
+	// Apply options
+	for _, opt := range opts {
+		opt(r)
+	}
+
 	return r, nil
 }
 
@@ -113,8 +147,11 @@ func (r *Runner) args(eventKind EventKind, workflowFile string, payloadFile stri
 		// Required for cloning private repos
 		"--secret", "GITHUB_TOKEN=" + r.gitHubToken,
 
-		// Mount mockdata (for mocked testdata, dist artifacts) and GCS (for mocked GCS)
-		"--container-options", `"-v $PWD/tests/act/mockdata:/mockdata -v ` + r.GCS.basePath + `:/gcs"`,
+		// Mounts:
+		// - mockdata: for mocked testdata, dist artifacts
+		// - GCS: for mocked GCS
+		// - /tmp: for temporary files, so the host's /tmp is used
+		"--container-options", `"-v $PWD/tests/act/mockdata:/mockdata -v ` + r.GCS.basePath + `:/gcs -v /tmp:/tmp"`,
 	}
 
 	// Map local all possible references of plugin-ci-workflows to the local repository
@@ -127,6 +164,9 @@ func (r *Runner) args(eventKind EventKind, workflowFile string, payloadFile stri
 	if r.ConcurrentJobs > 0 {
 		args = append(args, "--concurrent-jobs", fmt.Sprint(r.ConcurrentJobs))
 	}
+	if r.ContainerArchitecture != "" {
+		args = append(args, "--container-architecture", r.ContainerArchitecture)
+	}
 	// Map all self-hosted runners otherwise they don't run in act.
 	for _, label := range selfHostedRunnerLabels {
 		args = append(args, "-P", label+"="+nektosActRunnerImage)
@@ -296,9 +336,18 @@ func (r *Runner) parseGHACommand(data logLine, runResult *RunResult) {
 		// Store the output value. StepID can be an array in case of composite actions,
 		// group all composite action outputs under the first step ID for simplicity.
 		runResult.Outputs.Set(data.JobID, data.StepID[0], data.Name, data.Arg)
+	case "debug", "notice", "warning", "error":
+		// Annotations
+		runResult.Annotations = append(runResult.Annotations, Annotation{
+			Level:   AnnotationLevel(data.Command),
+			Title:   data.KvPairs["title"],
+			Message: data.Arg,
+		})
 	default:
-		// Nothing special to do, ignore silently
-		break
+		// Nothing special to do
+		if r.Verbose && data.Command != "" {
+			fmt.Printf("%s: [%s]: unhandled GHA command %q, ignoring", r.t.Name(), data.Job, data.Command)
+		}
 	}
 }
 
@@ -366,6 +415,32 @@ type RunResult struct {
 
 	// Outputs contains the outputs for each job + step of the workflow run.
 	Outputs Outputs
+
+	// Annotations contains the GitHub Actions annotations generated during the workflow run.
+	Annotations []Annotation
+}
+
+// AnnotationLevel represents the level of a GitHub Actions annotation.
+type AnnotationLevel string
+
+// Annotation levels
+const (
+	AnnotationLevelDebug   AnnotationLevel = "debug"
+	AnnotationLevelNotice  AnnotationLevel = "notice"
+	AnnotationLevelWarning AnnotationLevel = "warning"
+	AnnotationLevelError   AnnotationLevel = "error"
+)
+
+// Annotation represents a single GitHub Actions annotation.
+type Annotation struct {
+	// Level is the level of the annotation.
+	Level AnnotationLevel
+
+	// Title is the optional title of the annotation.
+	Title string
+
+	// Message is the message of the annotation itself.
+	Message string
 }
 
 // newRunResult creates a new empty RunResult instance.

tests/act/internal/act/logger.go

@@ -18,7 +18,8 @@ type logLine struct {
 
 	// Intercepted GHA commands
 
-	Command string `json:"command,omitempty"`
-	Name    string `json:"name,omitempty"`
-	Arg     string `json:"arg,omitempty"`
+	Command string            `json:"command,omitempty"`
+	Name    string            `json:"name,omitempty"`
+	Arg     string            `json:"arg,omitempty"`
+	KvPairs map[string]string `json:"kvPairs,omitempty"`
 }