Reconsider use of never-stable, add canary and techpreview variants

- I haven't found being more liberal with adding jobs to the
  never-stable list to particularly be beneficial, so I've removed most
  and adjusted the comment about criteria.

- Add a canary variant, so it is more prominently displayed

- Adds a new techpreview variant, for TPNU jobs

Author: stbenjam

pkg/testgridanalysis/testidentification/ocp_variants.go

@@ -12,6 +12,7 @@ var (
 	// variant regexes
 	awsRegex     = regexp.MustCompile(`(?i)-aws`)
 	azureRegex   = regexp.MustCompile(`(?i)-azure`)
+	canaryRegex  = regexp.MustCompile(`(?i)-canary`)
 	compactRegex = regexp.MustCompile(`(?i)-compact`)
 	fipsRegex    = regexp.MustCompile(`(?i)-fips`)
 	metalRegex   = regexp.MustCompile(`(?i)-metal`)
@@ -32,6 +33,7 @@ var (
 	rtRegex      = regexp.MustCompile(`(?i)-rt`)
 	s390xRegex   = regexp.MustCompile(`(?i)-s390x`)
 	serialRegex  = regexp.MustCompile(`(?i)-serial`)
+	techpreview  = regexp.MustCompile(`(?i)-techpreview`)
 	upgradeRegex = regexp.MustCompile(`(?i)-upgrade`)
 	// some vsphere jobs do not have a trailing -version segment
 	vsphereRegex    = regexp.MustCompile(`(?i)-vsphere`)
@@ -41,6 +43,7 @@ var (
 	allOpenshiftVariants = sets.NewString(
 		"aws",
 		"azure",
+		"canary",
 		"compact",
 		"fips",
 		"gcp",
@@ -58,64 +61,30 @@ var (
 		"realtime",
 		"s390x",
 		"serial",
+		"techpreview",
 		"upgrade",
 		"vsphere-ipi",
 		"vsphere-upi",
 		"single-node",
 	)
 
-	// openshiftJobsNeverStableForVariants is a list of jobs that are
-	// below 40% passing rate. As we phase these jobs in, they should be
-	// excluded from "normal" variants and once they are passing above 40%
-	// can "graduated" from never-stable.
+	// openshiftJobsNeverStableForVariants is a list of unproven new jobs or
+	// jobs that are near permafail (i.e. < 40%) for an extended period of time.
+	// They are excluded from "normal" variants and once they are passing above
+	// 40% can "graduated" from never-stable.
 	//
 	// These jobs are still listed as jobs in total and when individual
 	// tests fail, they will still be listed with these jobs as causes.
 	openshiftJobsNeverStableForVariants = sets.NewString(
+		// Experimental jobs that are under active development
 		"periodic-ci-openshift-release-master-ci-4.9-e2e-aws-hypershift",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-aws-ovn",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-aws-upgrade-single-node",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-gcp-ovn",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-openstack-ovn",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-e2e-aws-ovn-upgrade",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-e2e-aws-ovn-upgrade-rollback",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-e2e-aws-upgrade",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-e2e-aws-upgrade-rollback",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-e2e-azure-ovn-upgrade",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-e2e-gcp-ovn-upgrade",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-e2e-openstack-upgrade",
-		"periodic-ci-openshift-release-master-ci-4.9-upgrade-from-stable-4.8-from-stable-4.7-e2e-aws-upgrade",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-aws-arm64",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-aws-csi-migration",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-aws-proxy",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-aws-workers-rhel7",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-dualstack",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-dualstack-local-gateway",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-remote-libvirt-ppc64le",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-remote-libvirt-s390x",
-		"periodic-ci-openshift-release-master-nightly-4.9-openshift-ipi-azure-arcconformance",
-		"periodic-ci-openshift-release-master-nightly-4.9-upgrade-from-stable-4.7-e2e-aws-upgrade-paused",
-		"periodic-ci-openshift-release-master-nightly-4.9-upgrade-from-stable-4.8-e2e-aws-upgrade",
-		"release-openshift-ocp-installer-e2e-azure-ovn-4.9",
-		"release-openshift-ocp-installer-e2e-gcp-ovn-4.9",
+
+		// https://issues.redhat.com/browse/OSD-8071
 		"release-openshift-ocp-osd-aws-nightly-4.9",
 		"release-openshift-ocp-osd-gcp-nightly-4.9",
-		"release-openshift-origin-installer-e2e-aws-disruptive-4.9",
+
+		// https://bugzilla.redhat.com/show_bug.cgi?id=1979962
 		"release-openshift-origin-installer-e2e-aws-sdn-network-stress-4.9",
-		"release-openshift-origin-installer-e2e-aws-upgrade-4.6-to-4.7-to-4.8-to-4.9-ci",
-		"release-openshift-origin-installer-old-rhcos-e2e-aws-4.9",
-
-		// Compact jobs alerting on etcdGRPCRequestsSlow
-		// BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1986119
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-aws-compact",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-aws-compact-upgrade",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-azure-compact",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-azure-compact-upgrade",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-gcp-compact",
-		"periodic-ci-openshift-release-master-ci-4.9-e2e-gcp-compact-upgrade",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-compact-remote-libvirt-ppc64le",
-		"periodic-ci-openshift-release-master-nightly-4.9-e2e-compact-remote-libvirt-s390x",
 	)
 )
 
@@ -140,10 +109,16 @@ func (v openshiftVariants) IdentifyVariants(jobName string) []string {
 		}
 	}()
 
+	// If a job is in never-stable, it is excluded from other possible variant aggregations
 	if v.IsJobNeverStable(jobName) {
 		return []string{"never-stable"}
 	}
 
+	// Tech preview jobs are excluded from other possible variant aggregations
+	if techpreview.MatchString(jobName) {
+		return []string{"techpreview"}
+	}
+
 	// if it's a promotion job, it can't be a part of any other variant aggregation
 	if promoteRegex.MatchString(jobName) {
 		variants = append(variants, "promote")
@@ -156,6 +131,11 @@ func (v openshiftVariants) IdentifyVariants(jobName string) []string {
 	if azureRegex.MatchString(jobName) {
 		variants = append(variants, "azure")
 	}
+
+	if canaryRegex.MatchString(jobName) {
+		variants = append(variants, "canary")
+	}
+
 	if compactRegex.MatchString(jobName) {
 		variants = append(variants, "compact")
 	}