Comprehensive e2e testing for Component Readiness with abstracted data backends

cmd/sippy/serve.go

@@ -19,6 +19,7 @@ import (
 	resources "github.com/openshift/sippy"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider"
 	bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery"
+	pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres"
 	"github.com/openshift/sippy/pkg/apis/cache"
 	"github.com/openshift/sippy/pkg/bigquery"
 	"github.com/openshift/sippy/pkg/bigquery/bqlabel"
@@ -69,10 +70,13 @@ func (f *ServerFlags) BindFlags(flagSet *pflag.FlagSet) {
 	f.ConfigFlags.BindFlags(flagSet)
 	f.APIFlags.BindFlags(flagSet)
 	f.JiraFlags.BindFlags(flagSet)
-	flagSet.StringVar(&f.DataProvider, "data-provider", "bigquery", "Data provider for component readiness: bigquery")
+	flagSet.StringVar(&f.DataProvider, "data-provider", "bigquery", "Data provider for component readiness: bigquery, postgres")
 }
 
 func (f *ServerFlags) Validate() error {
+	if f.DataProvider == "postgres" {
+		return nil
+	}
 	return f.GoogleCloudFlags.Validate()
 }
 
@@ -132,8 +136,12 @@ func NewServeCommand() *cobra.Command {
 					crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient, config.ComponentReadinessConfig.VariantJunitTableOverrides)
 				}
 
+			case "postgres":
+				crDataProvider = pgprovider.NewPostgresProvider(dbc, cacheClient)
+				log.Info("Using Postgres data provider for component readiness")
+
 			default:
-				return fmt.Errorf("unknown --data-provider %q, must be bigquery", f.DataProvider)
+				return fmt.Errorf("unknown --data-provider %q, must be bigquery or postgres", f.DataProvider)
 			}
 
 			gcsClient, err = gcs.NewGCSClient(context.TODO(),

config/e2e-openshift.yaml

@@ -1,8 +1,8 @@
 prow:
   url: https://prow.ci.openshift.org/prowjobs.js
 releases:
-  "4.20":
+  "4.22":
     jobs:
-      periodic-ci-openshift-release-master-nightly-4.20-e2e-aws-ovn-serial-2of2: true
-      periodic-ci-openshift-release-main-ci-4.20-e2e-azure-ovn-upgrade: true
-      periodic-ci-openshift-release-master-nightly-4.20-e2e-gcp-ovn-csi: true
+      periodic-ci-openshift-release-master-nightly-4.22-e2e-aws-ovn-serial-2of2: true
+      periodic-ci-openshift-release-main-ci-4.22-e2e-azure-ovn-upgrade: true
+      periodic-ci-openshift-release-master-nightly-4.22-e2e-gcp-ovn-csi: true

config/e2e-views.yaml

@@ -1,81 +1,71 @@
----
 component_readiness:
-- name: 4.20-main
-  base_release:
-    release: "4.19"
-    relative_start: ga-30d
-    relative_end: ga
-  sample_release:
-    release: "4.20"
-    relative_start: now-7d
-    relative_end: now
-  variant_options:
-    column_group_by:
-      Architecture: {}
-      Network: {}
-      Platform: {}
-      Topology: {}
-    db_group_by:
-      Architecture: {}
-      FeatureSet: {}
-      Installer: {}
-      Network: {}
-      Platform: {}
-      Suite: {}
-      Topology: {}
-      Upgrade: {}
-    include_variants:
-      Architecture:
-      - amd64
-      FeatureSet:
-      - default
-      - techpreview
-      Installer:
-      - ipi
-      - upi
-      - hypershift
-      JobTier:
-      - blocking
-      - informing
-      - standard
-      LayeredProduct:
-      - none
-      - virt
-      Network:
-      - ovn
-      Owner:
-      - eng
-      - service-delivery
-      Platform:
-      - aws
-      - azure
-      - gcp
-      - metal
-      - rosa
-      - vsphere
-      Topology:
-      - ha
-      - microshift
-      - external
-      CGroupMode:
-      - v2
-      ContainerRuntime:
-      - runc
-      - crun
-  advanced_options:
-    minimum_failure: 3
-    confidence: 95
-    pity_factor: 5
-    ignore_missing: false
-    ignore_disruption: true
-    flake_as_failure: false
-    pass_rate_required_new_tests: 95
-    include_multi_release_analysis: true
-  metrics:
-    enabled: true
-  regression_tracking:
-    enabled: true
-  prime_cache:
-    enabled: true
-  automate_jira:
-    enabled: true
+    - name: 4.22-main
+      base_release:
+        release: "4.21"
+        relative_start: now-60d
+        relative_end: now-30d
+      sample_release:
+        release: "4.22"
+        relative_start: now-3d
+        relative_end: now
+      test_id_options: {}
+      test_filters: {}
+      variant_options:
+        column_group_by:
+            Network: {}
+            Platform: {}
+            Topology: {}
+        db_group_by:
+            Architecture: {}
+            FeatureSet: {}
+            Installer: {}
+            LayeredProduct: {}
+            Network: {}
+            Platform: {}
+            Suite: {}
+            Topology: {}
+            Upgrade: {}
+        include_variants:
+            Architecture:
+                - amd64
+                - arm64
+            FeatureSet:
+                - default
+                - techpreview
+            Installer:
+                - ipi
+            LayeredProduct:
+                - none
+            Network:
+                - ovn
+            Platform:
+                - aws
+                - gcp
+            Suite:
+                - parallel
+                - serial
+                - unknown
+            Topology:
+                - ha
+            Upgrade:
+                - micro
+                - minor
+                - none
+      advanced_options:
+        minimum_failure: 3
+        confidence: 95
+        pity_factor: 5
+        pass_rate_required_new_tests: 90
+        pass_rate_required_all_tests: 0
+        ignore_missing: false
+        ignore_disruption: false
+        flake_as_failure: false
+        include_multi_release_analysis: true
+      metrics:
+        enabled: false
+      regression_tracking:
+        enabled: true
+      automate_jira:
+        enabled: false
+      prime_cache:
+        enabled: true

e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh

@@ -6,11 +6,6 @@
 # When running locally, the user has to define SIPPY_IMAGE.
 echo "The sippy CI image: ${SIPPY_IMAGE}"
 
-# The GCS_CRED allows us to pull artifacts from GCS when importing prow jobs.
-# Redefine GCS_CRED to use your own.
-GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}"
-echo "The GCS cred is: ${GCS_CRED}"
-
 # If you're using Openshift, we use oc, if you're using plain Kubernetes,
 # we use kubectl.
 #
@@ -61,14 +56,6 @@ fi
 
 e2e_pause
 
-echo "Checking for presense of GCS credentials ..."
-if [ -f ${GCS_CRED} ]; then
-  ls -l ${GCS_CRED}
-else
-  echo "Aborting: GCS credential file ${GCS_CRED} not found"
-  exit 1
-fi
-
 echo "Starting postgres on cluster-pool cluster..."
 
 # Make the "postgres" namespace and pod.
@@ -261,9 +248,12 @@ ${KUBECTL_CMD} -n sippy-e2e get svc,ep
 # Get the gcs credentials out to the cluster-pool cluster.
 # These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations).
 # See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa
-#
-
-${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred=$GCS_CRED -n sippy-e2e
+GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}"
+if [ -f "${GCS_CRED}" ]; then
+  ${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred="${GCS_CRED}" -n sippy-e2e
+else
+  echo "WARNING: GCS credential file ${GCS_CRED} not found, BigQuery tests will fail"
+fi
 
 # Get the registry credentials for all build farm clusters out to the cluster-pool cluster.
 ${KUBECTL_CMD} -n sippy-e2e create secret generic regcred --from-file=.dockerconfigjson=${DOCKERCONFIGJSON} --type=kubernetes.io/dockerconfigjson
@@ -283,12 +273,12 @@ spec:
       storage: 100Mi
 END
 
-# Make the "sippy loader" pod.
+# Seed synthetic data into postgres.
 cat << END | ${KUBECTL_CMD} apply -f -
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: sippy-load-job
+  name: sippy-seed-job
   namespace: sippy-e2e
 spec:
   template:
@@ -304,20 +294,19 @@ spec:
         terminationMessagePolicy: File
         command:  ["/bin/sh", "-c"]
         args:
-          - /bin/sippy load --init-database --log-level=debug --release 4.20 --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres --redis-url=redis://redis.sippy-e2e.svc.cluster.local:6379 --mode=ocp --config ./config/e2e-openshift.yaml --google-service-account-credential-file /tmp/secrets/gcs-cred
+          - /bin/sippy-cover seed-data --init-database --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres
         env:
-        - name: GCS_SA_JSON_PATH
-          value: /tmp/secrets/gcs-cred
+        - name: GOCOVERDIR
+          value: /tmp/coverage
         volumeMounts:
-        - mountPath: /tmp/secrets
-          name: gcs-cred
-          readOnly: true
+        - mountPath: /tmp/coverage
+          name: coverage
       imagePullSecrets:
       - name: regcred
       volumes:
-        - name: gcs-cred
-          secret:
-            secretName: gcs-cred
+      - name: coverage
+        persistentVolumeClaim:
+          claimName: sippy-coverage
       dnsPolicy: ClusterFirst
       restartPolicy: Never
       schedulerName: default-scheduler
@@ -327,33 +316,33 @@ spec:
 END
 
 date
-echo "Waiting for sippy loader job to finish ..."
-${KUBECTL_CMD} -n sippy-e2e get job sippy-load-job
-${KUBECTL_CMD} -n sippy-e2e describe job sippy-load-job
+echo "Waiting for sippy seed job to finish ..."
+${KUBECTL_CMD} -n sippy-e2e get job sippy-seed-job
+${KUBECTL_CMD} -n sippy-e2e describe job sippy-seed-job
 
 # We set +e to avoid the script aborting before we can retrieve logs.
 set +e
 
-echo "Waiting up to ${SIPPY_LOAD_TIMEOUT:=1200s} for the sippy-load-job to complete..."
-${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-load-job --timeout ${SIPPY_LOAD_TIMEOUT}
+echo "Waiting up to ${SIPPY_LOAD_TIMEOUT:=1200s} for the sippy-seed-job to complete..."
+${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-seed-job --timeout "${SIPPY_LOAD_TIMEOUT}"
 retVal=$?
 set -e
 
-job_pod=$(${KUBECTL_CMD} -n sippy-e2e get pod --selector=job-name=sippy-load-job --output=jsonpath='{.items[0].metadata.name}')
-${KUBECTL_CMD} -n sippy-e2e logs ${job_pod} > ${ARTIFACT_DIR}/sippy-load.log 2>&1
+job_pod=$(${KUBECTL_CMD} -n sippy-e2e get pod --selector=job-name=sippy-seed-job --output=jsonpath='{.items[0].metadata.name}')
+${KUBECTL_CMD} -n sippy-e2e logs "${job_pod}" > "${ARTIFACT_DIR}/sippy-seed.log" 2>&1
 
 if [ ${retVal} -ne 0 ]; then
   echo
-  echo "=== SIPPY LOAD JOB FAILURE DIAGNOSTICS ==="
+  echo "=== SIPPY SEED JOB FAILURE DIAGNOSTICS ==="
   echo "=== Job status ==="
-  ${KUBECTL_CMD} -n sippy-e2e describe job sippy-load-job
+  ${KUBECTL_CMD} -n sippy-e2e describe job sippy-seed-job
   echo "=== Job pod status ==="
   ${KUBECTL_CMD} -n sippy-e2e describe pod ${job_pod}
   echo "=== Recent namespace events ==="
   ${KUBECTL_CMD} -n sippy-e2e get events --sort-by='.lastTimestamp'
-  echo "=== END SIPPY LOAD JOB FAILURE DIAGNOSTICS ==="
+  echo "=== END SIPPY SEED JOB FAILURE DIAGNOSTICS ==="
   echo
-  echo "ERROR: sippy-load-job did not complete within ${SIPPY_LOAD_TIMEOUT}"
+  echo "ERROR: sippy-seed-job did not complete within ${SIPPY_LOAD_TIMEOUT}"
   exit 1
 fi