From 82f1aa35d24783a5c6e75b3546429d22662daeda Mon Sep 17 00:00:00 2001 From: Mohit Sheth Date: Mon, 2 Mar 2026 15:11:23 -0500 Subject: [PATCH 1/3] Add deferred Orion summary report step for consolidated regression reporting Signed-off-by: Mohit Sheth --- .../orion/cloud-bulldozer-orion-main.yaml | 6 +-- ...rfscale-ci-main__aws-4.21-nightly-x86.yaml | 3 +- ...rfscale-ci-main__aws-4.22-nightly-x86.yaml | 3 +- ...erfscale-ci-main__aws-5.0-nightly-x86.yaml | 3 +- ...scale-ci-main__metal-4.19-nightly-x86.yaml | 2 +- ...g-ocp-qe-perfscale-ci-main__metal-x86.yaml | 2 +- ...hift-priv-ovn-kubernetes-release-4.18.yaml | 2 + ...hift-priv-ovn-kubernetes-release-4.19.yaml | 6 +-- ...hift-priv-ovn-kubernetes-release-4.20.yaml | 6 +-- ...openshift-ovn-kubernetes-release-4.18.yaml | 2 + ...openshift-ovn-kubernetes-release-4.19.yaml | 6 +-- ...openshift-ovn-kubernetes-release-4.20.yaml | 6 +-- .../openshift-qe-control-plane-chain.yaml | 1 + ...hift-qe-orion-consolidated-virt-chain.yaml | 3 +- ...openshift-qe-orion-consolidated-chain.yaml | 3 +- .../orion/openshift-qe-orion-commands.sh | 12 +++++ .../orion/openshift-qe-orion-ref.yaml | 1 + .../openshift-qe/orion/report/OWNERS | 4 ++ .../openshift-qe-orion-report-commands.sh | 54 +++++++++++++++++++ ...penshift-qe-orion-report-ref.metadata.json | 11 ++++ .../report/openshift-qe-orion-report-ref.yaml | 29 ++++++++++ ...penshift-qe-udn-density-pods-workflow.yaml | 3 +- 22 files changed, 145 insertions(+), 23 deletions(-) create mode 100644 ci-operator/step-registry/openshift-qe/orion/report/OWNERS create mode 100644 ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh create mode 100644 ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.metadata.json create mode 100644 ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml diff --git a/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml b/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml index abcd3a1731d8b..7150d5dc1b825 100644 --- a/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml +++ b/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml @@ -30,7 +30,7 @@ tests: EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s --service-latency IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.21" ZONES_COUNT: "3" @@ -46,7 +46,7 @@ tests: env: ES_TYPE: qe OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" VERSION: "4.21" test: - chain: openshift-qe-orion-consolidated @@ -55,7 +55,7 @@ tests: steps: cluster_profile: aws-perfscale-qe env: - RUN_ORION: "true" + RUN_ORION: "deferred" VERSION: "4.19" test: - chain: openshift-qe-orion-consolidated-virt diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml index 40b31d714f1d6..88b40d4ac2589 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml @@ -41,7 +41,7 @@ tests: ES_TYPE: qe IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.21" ZONES_COUNT: "3" @@ -177,6 +177,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.21" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml index b28b9b866ad0c..c008929e5200f 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml @@ -41,7 +41,7 @@ tests: ES_TYPE: qe IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.22" ZONES_COUNT: "3" @@ -281,6 +281,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.22" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml index 0918b973564fe..f9215528d2450 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml @@ -42,7 +42,7 @@ tests: ES_TYPE: qe IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "5.0" ZONES_COUNT: "3" @@ -178,6 +178,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "5.0" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml index 20a7082ef8fb4..f2f127d6d987e 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml @@ -38,7 +38,7 @@ tests: CONFIG: config/standard-scalelab.yml INFRA: "true" PUBLIC_VLAN: "false" - RUN_ORION: "true" + RUN_ORION: "deferred" VERSION: "4.19" VM: "true" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml index 105a5e86730a6..d1c1e0fcb916a 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml @@ -256,7 +256,7 @@ tests: steps: cluster_profile: metal-perfscale-cpt env: - RUN_ORION: "true" + RUN_ORION: "deferred" VERSION: "4.19" test: - chain: openshift-qe-orion-consolidated-virt diff --git a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml index 90d792ce51fe5..1eaf8ead2183e 100644 --- a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml +++ b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml @@ -408,6 +408,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge EXTRA_MG_ARGS: --host-network + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.18" post: @@ -428,6 +429,7 @@ tests: ENABLE_LAYER_3: "false" EXTRA_MG_ARGS: --host-network KUBE_BURNER_VERSION: 1.5.0 + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" post: - chain: ipi-aws-post diff --git a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml index 6b851eeec5cc6..1aba91cdc1841 100644 --- a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml +++ b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml @@ -333,7 +333,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.19" ZONES_COUNT: "3" @@ -350,7 +350,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: @@ -369,7 +369,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: diff --git a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml index 6605c2c41a672..af85dbc34298f 100644 --- a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml +++ b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml @@ -334,7 +334,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.20" ZONES_COUNT: "3" @@ -352,7 +352,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: @@ -371,7 +371,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: diff --git a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml index 83a3d869cec9b..500612e5b0e99 100644 --- a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml +++ b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml @@ -424,6 +424,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge EXTRA_MG_ARGS: --host-network + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.18" post: @@ -444,6 +445,7 @@ tests: ENABLE_LAYER_3: "false" EXTRA_MG_ARGS: --host-network KUBE_BURNER_VERSION: 1.5.0 + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" post: - chain: ipi-aws-post diff --git a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml index 33612207c1ea9..75289b0f79bc8 100644 --- a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml +++ b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml @@ -354,7 +354,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.19" ZONES_COUNT: "3" @@ -371,7 +371,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: @@ -390,7 +390,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: diff --git a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml index 61c71758b78e1..db89a859bb122 100644 --- a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml +++ b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml @@ -355,7 +355,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.20" ZONES_COUNT: "3" @@ -373,7 +373,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: @@ -392,7 +392,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "true" + RUN_ORION: "deferred" UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: diff --git a/ci-operator/step-registry/openshift-qe/control-plane/openshift-qe-control-plane-chain.yaml b/ci-operator/step-registry/openshift-qe/control-plane/openshift-qe-control-plane-chain.yaml index b690d1f1b61b3..e1a7613a0d8ee 100644 --- a/ci-operator/step-registry/openshift-qe/control-plane/openshift-qe-control-plane-chain.yaml +++ b/ci-operator/step-registry/openshift-qe/control-plane/openshift-qe-control-plane-chain.yaml @@ -9,3 +9,4 @@ chain: - chain: openshift-qe-udn-density-pods documentation: |- This chain executes cluster density v2, node density-cni, crd-scale, node-density, and udn density workloads. + diff --git a/ci-operator/step-registry/openshift-qe/orion/consolidated-virt/openshift-qe-orion-consolidated-virt-chain.yaml b/ci-operator/step-registry/openshift-qe/orion/consolidated-virt/openshift-qe-orion-consolidated-virt-chain.yaml index 8dba35e4bf074..9bfa2230a762a 100644 --- a/ci-operator/step-registry/openshift-qe/orion/consolidated-virt/openshift-qe-orion-consolidated-virt-chain.yaml +++ b/ci-operator/step-registry/openshift-qe/orion/consolidated-virt/openshift-qe-orion-consolidated-virt-chain.yaml @@ -4,5 +4,6 @@ chain: - ref: openshift-qe-orion-data-path - ref: openshift-qe-orion-virt-density - ref: openshift-qe-orion-virt-udn-density + - ref: openshift-qe-orion-report documentation: |- - Chain of all orion virtualization workloads. + Chain of all orion virt workloads with aggregated summary reporting. diff --git a/ci-operator/step-registry/openshift-qe/orion/consolidated/openshift-qe-orion-consolidated-chain.yaml b/ci-operator/step-registry/openshift-qe/orion/consolidated/openshift-qe-orion-consolidated-chain.yaml index 8c9833533700a..4a9eef0cf54c9 100644 --- a/ci-operator/step-registry/openshift-qe/orion/consolidated/openshift-qe-orion-consolidated-chain.yaml +++ b/ci-operator/step-registry/openshift-qe/orion/consolidated/openshift-qe-orion-consolidated-chain.yaml @@ -5,5 +5,6 @@ chain: - ref: openshift-qe-orion-node-density - ref: openshift-qe-orion-node-density-cni - ref: openshift-qe-orion-crd-scale + - ref: openshift-qe-orion-report documentation: |- - Chain of all orion workloads. + Chain of all orion workloads with aggregated summary reporting. diff --git a/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-commands.sh b/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-commands.sh index 992287ce8139b..16f9bbd70acdc 100755 --- a/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-commands.sh +++ b/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-commands.sh @@ -263,10 +263,22 @@ process_change_point cp *.csv *.xml *.json *.txt "${ARTIFACT_DIR}/" 2>/dev/null || true +# Write deferred JSON results to SHARED_DIR for report step aggregation +if [ "${RUN_ORION}" == "deferred" ]; then + for f in junit*.json; do + [ -e "$f" ] && cp "$f" "${SHARED_DIR}/orion-${FILENAME}-$(basename "$f")" 2>/dev/null || true + done +fi + if [ $orion_exit_status -eq 3 ]; then echo "Orion returned exit code 3, which means there are no results to analyze." echo "Exiting zero since there were no regressions found." exit 0 fi +if [ "${RUN_ORION}" == "deferred" ]; then + echo "RUN_ORION=deferred. Exit status $orion_exit_status deferred to report step." + exit 0 +fi + exit $orion_exit_status diff --git a/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-ref.yaml b/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-ref.yaml index 1762d40a8c827..1ab7b585677b2 100644 --- a/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-ref.yaml +++ b/ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-ref.yaml @@ -80,6 +80,7 @@ ref: default: "" documentation: Comma separated list of repositories to monitor before and after change points. + commands: openshift-qe-orion-commands.sh timeout: 6h credentials: diff --git a/ci-operator/step-registry/openshift-qe/orion/report/OWNERS b/ci-operator/step-registry/openshift-qe/orion/report/OWNERS new file mode 100644 index 0000000000000..0b91d9fc0b75f --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/report/OWNERS @@ -0,0 +1,4 @@ +approvers: +- perfscale-ocp-approvers +reviewers: +- perfscale-ocp-reviewers diff --git a/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh new file mode 100644 index 0000000000000..514b911df2d5b --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -euo pipefail + +if [ "${RUN_ORION}" == "false" ]; then + echo "RUN_ORION is false, skipping report." + exit 0 +fi + +# Check for deferred JSON results +shopt -s nullglob +json_files=(${SHARED_DIR}/orion-*.json) +shopt -u nullglob + +if [ ${#json_files[@]} -eq 0 ]; then + echo "No deferred orion JSON results found in SHARED_DIR." + echo "This is expected when RUN_ORION is not set to deferred." + exit 0 +fi + +# Copy JSONs to ARTIFACT_DIR for archival +cp "${json_files[@]}" "${ARTIFACT_DIR}/" 2>/dev/null || true + +# Set up Python environment and install orion +python --version +pushd /tmp +python -m virtualenv ./venv_report +source ./venv_report/bin/activate + +if [[ $TAG == "latest" ]]; then + LATEST_TAG=$(curl -s "https://api.github.com/repos/cloud-bulldozer/orion/releases/latest" | jq -r '.tag_name') +else + LATEST_TAG=$TAG +fi +git clone --branch "$LATEST_TAG" "$ORION_REPO" --depth 1 +pushd orion +pip install . +popd && popd + +# Build comma-separated file list for orion --report +json_file_list=$(IFS=,; echo "${json_files[*]}") + +# Run orion report on all deferred JSONs +# orion --report exits 2 if regressions found, 0 otherwise +set +e +orion --report "$json_file_list" +report_exit=$? +set -e + +if [ $report_exit -eq 2 ]; then + echo "Orion report detected regressions." + exit 1 +fi + +exit $report_exit diff --git a/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.metadata.json b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.metadata.json new file mode 100644 index 0000000000000..2e4a2b24ab928 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml", + "owners": { + "approvers": [ + "perfscale-ocp-approvers" + ], + "reviewers": [ + "perfscale-ocp-reviewers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml new file mode 100644 index 0000000000000..8cfc600d03ce9 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-ref.yaml @@ -0,0 +1,29 @@ +ref: + as: openshift-qe-orion-report + from_image: + namespace: ci + name: ocp-qe-perfscale-ci + tag: latest + env: + - name: ORION_REPO + default: "https://github.com/cloud-bulldozer/orion.git" + documentation: + Orion github repository to clone for report generation. + - name: TAG + default: "latest" + documentation: + Orion version/tag to clone. + - name: RUN_ORION + default: "false" + documentation: + When false, the report step is skipped. Set to deferred at test level to enable deferred failure reporting. + commands: openshift-qe-orion-report-commands.sh + timeout: 30m + resources: + requests: + cpu: 100m + memory: 100Mi + documentation: >- + Aggregates deferred orion regression results from SHARED_DIR using + orion --report to produce a consolidated regression report. Must be + used as the last step in an orion chain when RUN_ORION is set to deferred. diff --git a/ci-operator/step-registry/openshift-qe/udn-density-pods/openshift-qe-udn-density-pods-workflow.yaml b/ci-operator/step-registry/openshift-qe/udn-density-pods/openshift-qe-udn-density-pods-workflow.yaml index 93ec273f68909..36147d5abc6aa 100644 --- a/ci-operator/step-registry/openshift-qe/udn-density-pods/openshift-qe-udn-density-pods-workflow.yaml +++ b/ci-operator/step-registry/openshift-qe/udn-density-pods/openshift-qe-udn-density-pods-workflow.yaml @@ -4,5 +4,6 @@ workflow: steps: test: - chain: openshift-qe-udn-density-pods + - ref: openshift-qe-orion-report documentation: |- - This workflow executes udn-density-pods workload. \ No newline at end of file + This workflow executes udn-density-pods workload and Orion change detection with summary reporting. \ No newline at end of file From 5ab286b82b15e9254282b672fd17b1bc0613483f Mon Sep 17 00:00:00 2001 From: Mohit Sheth Date: Mon, 2 Mar 2026 15:23:06 -0500 Subject: [PATCH 2/3] Temporarily point orion report to fork for testing Signed-off-by: Mohit Sheth --- .../orion/report/openshift-qe-orion-report-commands.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh index 514b911df2d5b..d2bcd61ac19fc 100644 --- a/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh +++ b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh @@ -26,12 +26,8 @@ pushd /tmp python -m virtualenv ./venv_report source ./venv_report/bin/activate -if [[ $TAG == "latest" ]]; then - LATEST_TAG=$(curl -s "https://api.github.com/repos/cloud-bulldozer/orion/releases/latest" | jq -r '.tag_name') -else - LATEST_TAG=$TAG -fi -git clone --branch "$LATEST_TAG" "$ORION_REPO" --depth 1 +## TODO: revert before merge - using fork for testing +git clone --branch summary-analysis https://github.com/mohit-sheth/orion.git --depth 1 pushd orion pip install . popd && popd From 4d7523fdfb81d04edf7873cfb2140de4e3c17803 Mon Sep 17 00:00:00 2001 From: Mohit Sheth Date: Mon, 2 Mar 2026 15:51:39 -0500 Subject: [PATCH 3/3] Fix shellcheck warnings and ci-operator issues Signed-off-by: Mohit Sheth --- .../orion/cloud-bulldozer-orion-main.yaml | 6 +++--- ...-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml | 4 ++-- ...-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml | 4 ++-- ...p-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml | 4 ++-- ...e-perfscale-ci-main__metal-4.19-nightly-x86.yaml | 2 +- ...ift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml | 2 +- .../openshift-priv-ovn-kubernetes-release-4.18.yaml | 4 ++-- .../openshift-priv-ovn-kubernetes-release-4.19.yaml | 6 +++--- .../openshift-priv-ovn-kubernetes-release-4.20.yaml | 6 +++--- .../openshift-ovn-kubernetes-release-4.18.yaml | 4 ++-- .../openshift-ovn-kubernetes-release-4.19.yaml | 6 +++--- .../openshift-ovn-kubernetes-release-4.20.yaml | 6 +++--- .../report/openshift-qe-orion-report-commands.sh | 13 +++++++++---- 13 files changed, 36 insertions(+), 31 deletions(-) diff --git a/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml b/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml index 7150d5dc1b825..7980d0f67920d 100644 --- a/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml +++ b/ci-operator/config/cloud-bulldozer/orion/cloud-bulldozer-orion-main.yaml @@ -30,7 +30,7 @@ tests: EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s --service-latency IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.21" ZONES_COUNT: "3" @@ -46,7 +46,7 @@ tests: env: ES_TYPE: qe OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred VERSION: "4.21" test: - chain: openshift-qe-orion-consolidated @@ -55,7 +55,7 @@ tests: steps: cluster_profile: aws-perfscale-qe env: - RUN_ORION: "deferred" + RUN_ORION: deferred VERSION: "4.19" test: - chain: openshift-qe-orion-consolidated-virt diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml index 88b40d4ac2589..f1e40e5285861 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.21-nightly-x86.yaml @@ -41,7 +41,7 @@ tests: ES_TYPE: qe IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.21" ZONES_COUNT: "3" @@ -177,7 +177,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.21" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml index c008929e5200f..359c6588deb79 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-4.22-nightly-x86.yaml @@ -41,7 +41,7 @@ tests: ES_TYPE: qe IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.22" ZONES_COUNT: "3" @@ -281,7 +281,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.22" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml index f9215528d2450..7627179677bad 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__aws-5.0-nightly-x86.yaml @@ -42,7 +42,7 @@ tests: ES_TYPE: qe IGNORE_JOB_ITERATIONS: "true" OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "5.0" ZONES_COUNT: "3" @@ -178,7 +178,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "5.0" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml index f2f127d6d987e..25033a3233633 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-4.19-nightly-x86.yaml @@ -38,7 +38,7 @@ tests: CONFIG: config/standard-scalelab.yml INFRA: "true" PUBLIC_VLAN: "false" - RUN_ORION: "deferred" + RUN_ORION: deferred VERSION: "4.19" VM: "true" post: diff --git a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml index d1c1e0fcb916a..ee3764cb20eb1 100644 --- a/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml +++ b/ci-operator/config/openshift-eng/ocp-qe-perfscale-ci/openshift-eng-ocp-qe-perfscale-ci-main__metal-x86.yaml @@ -256,7 +256,7 @@ tests: steps: cluster_profile: metal-perfscale-cpt env: - RUN_ORION: "deferred" + RUN_ORION: deferred VERSION: "4.19" test: - chain: openshift-qe-orion-consolidated-virt diff --git a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml index 1eaf8ead2183e..63bf485853267 100644 --- a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml +++ b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.18.yaml @@ -408,7 +408,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge EXTRA_MG_ARGS: --host-network - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.18" post: @@ -429,7 +429,7 @@ tests: ENABLE_LAYER_3: "false" EXTRA_MG_ARGS: --host-network KUBE_BURNER_VERSION: 1.5.0 - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" post: - chain: ipi-aws-post diff --git a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml index 1aba91cdc1841..11c3f997a18fe 100644 --- a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml +++ b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.19.yaml @@ -333,7 +333,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.19" ZONES_COUNT: "3" @@ -350,7 +350,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: @@ -369,7 +369,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: diff --git a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml index af85dbc34298f..fe5efe59c6605 100644 --- a/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml +++ b/ci-operator/config/openshift-priv/ovn-kubernetes/openshift-priv-ovn-kubernetes-release-4.20.yaml @@ -334,7 +334,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.20" ZONES_COUNT: "3" @@ -352,7 +352,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: @@ -371,7 +371,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: diff --git a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml index 500612e5b0e99..db7903c445fb5 100644 --- a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml +++ b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.18.yaml @@ -424,7 +424,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge EXTRA_MG_ARGS: --host-network - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.18" post: @@ -445,7 +445,7 @@ tests: ENABLE_LAYER_3: "false" EXTRA_MG_ARGS: --host-network KUBE_BURNER_VERSION: 1.5.0 - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" post: - chain: ipi-aws-post diff --git a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml index 75289b0f79bc8..c3453803edb58 100644 --- a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml +++ b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.19.yaml @@ -354,7 +354,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.19" ZONES_COUNT: "3" @@ -371,7 +371,7 @@ tests: BASE_DOMAIN: qe.devcluster.openshift.com COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: @@ -390,7 +390,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.19" post: diff --git a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml index db89a859bb122..7fe6c6ab322dc 100644 --- a/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml +++ b/ci-operator/config/openshift/ovn-kubernetes/openshift-ovn-kubernetes-release-4.20.yaml @@ -355,7 +355,7 @@ tests: ES_TYPE: qe EXTRA_FLAGS: --churn-duration=20m --pod-ready-threshold=20s OUTPUT_FORMAT: JUNIT - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "12" VERSION: "4.20" ZONES_COUNT: "3" @@ -373,7 +373,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "true" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: @@ -392,7 +392,7 @@ tests: COMPUTE_NODE_REPLICAS: "24" COMPUTE_NODE_TYPE: m5.xlarge ENABLE_LAYER_3: "false" - RUN_ORION: "deferred" + RUN_ORION: deferred UDN_ITERATION_MULTIPLIER: "3" VERSION: "4.20" post: diff --git a/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh index d2bcd61ac19fc..671c7f74e6091 100644 --- a/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh +++ b/ci-operator/step-registry/openshift-qe/orion/report/openshift-qe-orion-report-commands.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -x set -euo pipefail if [ "${RUN_ORION}" == "false" ]; then @@ -8,7 +9,7 @@ fi # Check for deferred JSON results shopt -s nullglob -json_files=(${SHARED_DIR}/orion-*.json) +json_files=("${SHARED_DIR}"/orion-*.json) shopt -u nullglob if [ ${#json_files[@]} -eq 0 ]; then @@ -29,11 +30,15 @@ source ./venv_report/bin/activate ## TODO: revert before merge - using fork for testing git clone --branch summary-analysis https://github.com/mohit-sheth/orion.git --depth 1 pushd orion +pip install -r requirements.txt pip install . popd && popd # Build comma-separated file list for orion --report -json_file_list=$(IFS=,; echo "${json_files[*]}") +json_file_list="" +for f in "${json_files[@]}"; do + json_file_list="${json_file_list:+${json_file_list},}${f}" +done # Run orion report on all deferred JSONs # orion --report exits 2 if regressions found, 0 otherwise @@ -42,9 +47,9 @@ orion --report "$json_file_list" report_exit=$? set -e -if [ $report_exit -eq 2 ]; then +if [ "$report_exit" -eq 2 ]; then echo "Orion report detected regressions." exit 1 fi -exit $report_exit +exit "$report_exit"