Add deferred Orion summary report step for consolidated regression reporting

ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml

@@ -30,7 +30,7 @@ tests:
       EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s --service-latency
       IGNORE_JOB_ITERATIONS: "true"
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.21"
       ZONES_COUNT: "3"
@@ -46,7 +46,7 @@ tests:
     env:
       ES_TYPE: qe
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.21"
     test:
     - chain: openshift-qe-orion-consolidated
@@ -55,7 +55,7 @@ tests:
   steps:
     cluster_profile: aws-perfscale-qe
     env:
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.19"
     test:
     - chain: openshift-qe-orion-consolidated-virt

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml

@@ -41,7 +41,7 @@ tests:
       ES_TYPE: qe
       IGNORE_JOB_ITERATIONS: "true"
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.21"
       ZONES_COUNT: "3"
@@ -177,6 +177,7 @@ tests:
       BASE_DOMAIN: qe.devcluster.openshift.com
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.21"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml

@@ -41,7 +41,7 @@ tests:
       ES_TYPE: qe
       IGNORE_JOB_ITERATIONS: "true"
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.22"
       ZONES_COUNT: "3"
@@ -281,6 +281,7 @@ tests:
       BASE_DOMAIN: qe.devcluster.openshift.com
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.22"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml

@@ -42,7 +42,7 @@ tests:
       ES_TYPE: qe
       IGNORE_JOB_ITERATIONS: "true"
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "5.0"
       ZONES_COUNT: "3"
@@ -178,6 +178,7 @@ tests:
       BASE_DOMAIN: qe.devcluster.openshift.com
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "5.0"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml

@@ -38,7 +38,7 @@ tests:
       CONFIG: config/standard-scalelab.yml
       INFRA: "true"
       PUBLIC_VLAN: "false"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.19"
       VM: "true"
     post:

ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml

@@ -256,7 +256,7 @@ tests:
   steps:
     cluster_profile: metal-perfscale-cpt
     env:
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       VERSION: "4.19"
     test:
     - chain: openshift-qe-orion-consolidated-virt

ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml

@@ -408,6 +408,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       EXTRA_MG_ARGS: --host-network
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.18"
     post:
@@ -428,6 +429,7 @@ tests:
       ENABLE_LAYER_3: "false"
       EXTRA_MG_ARGS: --host-network
       KUBE_BURNER_VERSION: 1.5.0
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
     post:
     - chain: ipi-aws-post

ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml

@@ -333,7 +333,7 @@ tests:
       ES_TYPE: qe
       EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.19"
       ZONES_COUNT: "3"
@@ -350,7 +350,7 @@ tests:
       BASE_DOMAIN: qe.devcluster.openshift.com
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.19"
     post:
@@ -369,7 +369,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "false"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.19"
     post:

ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml

@@ -334,7 +334,7 @@ tests:
       ES_TYPE: qe
       EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.20"
       ZONES_COUNT: "3"
@@ -352,7 +352,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.20"
     post:
@@ -371,7 +371,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "false"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.20"
     post:

ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml

@@ -424,6 +424,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       EXTRA_MG_ARGS: --host-network
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.18"
     post:
@@ -444,6 +445,7 @@ tests:
       ENABLE_LAYER_3: "false"
       EXTRA_MG_ARGS: --host-network
       KUBE_BURNER_VERSION: 1.5.0
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
     post:
     - chain: ipi-aws-post

ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml

@@ -354,7 +354,7 @@ tests:
       ES_TYPE: qe
       EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.19"
       ZONES_COUNT: "3"
@@ -371,7 +371,7 @@ tests:
       BASE_DOMAIN: qe.devcluster.openshift.com
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.19"
     post:
@@ -390,7 +390,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "false"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.19"
     post:

ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml

@@ -355,7 +355,7 @@ tests:
       ES_TYPE: qe
       EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s
       OUTPUT_FORMAT: JUNIT
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "12"
       VERSION: "4.20"
       ZONES_COUNT: "3"
@@ -373,7 +373,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "true"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.20"
     post:
@@ -392,7 +392,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "24"
       COMPUTE_NODE_TYPE: m5.xlarge
       ENABLE_LAYER_3: "false"
-      RUN_ORION: "true"
+      RUN_ORION: deferred
       UDN_ITERATION_MULTIPLIER: "3"
       VERSION: "4.20"
     post:

ci-operator/step-registry/openshift-qe/control-plane/openshift-qe-control-plane-chain.yaml

@@ -9,3 +9,4 @@ chain:
   - chain: openshift-qe-udn-density-pods
   documentation: |-
     This chain executes cluster density v2, node density-cni, crd-scale, node-density, and udn density workloads.
+

ci-operator/step-registry/openshift-qe/orion/consolidated-virt/openshift-qe-orion-consolidated-virt-chain.yaml

@@ -4,5 +4,6 @@ chain:
   - ref: openshift-qe-orion-data-path
   - ref: openshift-qe-orion-virt-density
   - ref: openshift-qe-orion-virt-udn-density
+  - ref: openshift-qe-orion-report
   documentation: |-
-    Chain of all orion virtualization workloads. 
+    Chain of all orion virt workloads with aggregated summary reporting.

ci-operator/step-registry/openshift-qe/orion/consolidated/openshift-qe-orion-consolidated-chain.yaml

@@ -5,5 +5,6 @@ chain:
   - ref: openshift-qe-orion-node-density
   - ref: openshift-qe-orion-node-density-cni
   - ref: openshift-qe-orion-crd-scale
+  - ref: openshift-qe-orion-report
   documentation: |-
-    Chain of all orion workloads. 
+    Chain of all orion workloads with aggregated summary reporting.

ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-commands.sh

@@ -263,10 +263,22 @@ process_change_point
 
 cp *.csv *.xml *.json *.txt "${ARTIFACT_DIR}/" 2>/dev/null || true
 
+# Write deferred JSON results to SHARED_DIR for report step aggregation
+if [ "${RUN_ORION}" == "deferred" ]; then
+    for f in junit*.json; do
+        [ -e "$f" ] && cp "$f" "${SHARED_DIR}/orion-${FILENAME}-$(basename "$f")" 2>/dev/null || true
+    done
+fi
+
 if [ $orion_exit_status -eq 3 ]; then
   echo "Orion returned exit code 3, which means there are no results to analyze."
   echo "Exiting zero since there were no regressions found."
   exit 0
 fi
 
+if [ "${RUN_ORION}" == "deferred" ]; then
+  echo "RUN_ORION=deferred. Exit status $orion_exit_status deferred to report step."
+  exit 0
+fi
+
 exit $orion_exit_status

ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-ref.yaml

@@ -80,6 +80,7 @@ ref:
     default: ""
     documentation:
       Comma separated list of repositories to monitor before and after change points.
+
   commands: openshift-qe-orion-commands.sh
   timeout: 6h
   credentials:

ci-operator/step-registry/openshift-qe/orion/report/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- perfscale-ocp-approvers
+reviewers:
+- perfscale-ocp-reviewers

ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -x
+set -euo pipefail
+
+if [ "${RUN_ORION}" == "false" ]; then
+  echo "RUN_ORION is false, skipping report."
+  exit 0
+fi
+
+# Check for deferred JSON results
+shopt -s nullglob
+json_files=("${SHARED_DIR}"/orion-*.json)
+shopt -u nullglob
+
+if [ ${#json_files[@]} -eq 0 ]; then
+  echo "No deferred orion JSON results found in SHARED_DIR."
+  echo "This is expected when RUN_ORION is not set to deferred."
+  exit 0
+fi
+
+# Copy JSONs to ARTIFACT_DIR for archival
+cp "${json_files[@]}" "${ARTIFACT_DIR}/" 2>/dev/null || true
+
+# Set up Python environment and install orion
+python --version
+pushd /tmp
+python -m virtualenv ./venv_report
+source ./venv_report/bin/activate
+
+## TODO: revert before merge - using fork for testing
+git clone --branch summary-analysis https://github.com/mohit-sheth/orion.git --depth 1
+pushd orion
+pip install -r requirements.txt
+pip install .
+popd && popd
+
+# Build comma-separated file list for orion --report
+json_file_list=""
+for f in "${json_files[@]}"; do
+  json_file_list="${json_file_list:+${json_file_list},}${f}"
+done
+
+# Run orion report on all deferred JSONs
+# orion --report exits 2 if regressions found, 0 otherwise
+set +e
+orion --report "$json_file_list"
+report_exit=$?
+set -e
+
+if [ "$report_exit" -eq 2 ]; then
+  echo "Orion report detected regressions."
+  exit 1
+fi
+
+exit "$report_exit"

ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.metadata.json

@@ -0,0 +1,11 @@
+{
+	"path": "openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml",
+	"owners": {
+		"approvers": [
+			"perfscale-ocp-approvers"
+		],
+		"reviewers": [
+			"perfscale-ocp-reviewers"
+		]
+	}
+}

ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml

@@ -0,0 +1,29 @@
+ref:
+  as: openshift-qe-orion-report
+  from_image:
+    namespace: ci
+    name: ocp-qe-perfscale-ci
+    tag: latest
+  env:
+  - name: ORION_REPO
+    default: "https://github.com/cloud-bulldozer/orion.git"
+    documentation:
+      Orion github repository to clone for report generation.
+  - name: TAG
+    default: "latest"
+    documentation:
+      Orion version/tag to clone.
+  - name: RUN_ORION
+    default: "false"
+    documentation:
+      When false, the report step is skipped. Set to deferred at test level to enable deferred failure reporting.
+  commands: openshift-qe-orion-report-commands.sh
+  timeout: 30m
+  resources:
+    requests:
+      cpu: 100m
+      memory: 100Mi
+  documentation: >-
+    Aggregates deferred orion regression results from SHARED_DIR using
+    orion --report to produce a consolidated regression report. Must be
+    used as the last step in an orion chain when RUN_ORION is set to deferred.