[release-4.20] OCPBUGS-76856: cert tests: mark TLS registry test as informing

hack/verify-jsonformat.sh

@@ -20,6 +20,7 @@ found=0
 excluded_files=(
   "test/extended/testdata/cmd/test/cmd/testdata/new-app/bc-from-imagestreamimage.json"
   "test/extended/testdata/cmd/test/cmd/testdata/new-app/invalid.json"
+  "test/extended/util/compat_otp/testdata/opm/render/validate/catalog-error/operator-2/index.json"
 )
 
 set +e

pkg/cmd/openshift-tests/run-upgrade/options.go

@@ -72,7 +72,10 @@ func (o *RunUpgradeSuiteOptions) UpgradeTestPreSuite() error {
 	if err := clusterdiscovery.InitializeTestFramework(exutil.TestContext, config, o.GinkgoRunSuiteOptions.DryRun); err != nil {
 		return err
 	}
-	klog.V(4).Infof("Loaded test configuration: %#v", exutil.TestContext)
+	// Redact the bearer token exposure
+	testContextString := fmt.Sprintf("%#v", exutil.TestContext)
+	redactedTestContext := exutil.RedactBearerToken(testContextString)
+	klog.V(4).Infof("Loaded test configuration: %s", redactedTestContext)
 
 	return nil
 }

pkg/monitortests/kubeapiserver/auditloganalyzer/handle_operator_watch_count_tracking.go

@@ -311,7 +311,7 @@ func (s *watchCountTracking) CreateJunits() ([]*junitapi.JUnitTestCase, error) {
 			"kube-controller-manager-operator":       175.0,
 			"kube-storage-version-migrator-operator": 70.0,
 			"machine-api-operator":                   50.0,
-			"marketplace-operator":                   17.0,
+			"marketplace-operator":                   43.0,
 			"openshift-apiserver-operator":           244.0,
 			"openshift-config-operator":              52.0,
 			"openshift-controller-manager-operator":  200.0,

pkg/monitortests/kubeapiserver/disruptioninclusterapiserver/monitortest.go

@@ -67,16 +67,30 @@ var (
 	rbacMonitorCRBName         string
 )
 
+// HostedClusterType represents the type of cluster hosting model
+type HostedClusterType string
+
+const (
+	// HostedClusterTypeStandalone represents a standard OpenShift cluster with self-hosted control plane
+	HostedClusterTypeStandalone HostedClusterType = "Standalone"
+	// HostedClusterTypeAROHCP represents an ARO HCP (Azure Red Hat OpenShift Hosted Control Plane) cluster
+	HostedClusterTypeAROHCP HostedClusterType = "AROHCP"
+	// HostedClusterTypeBareMetal represents a bare metal HyperShift hosted cluster
+	HostedClusterTypeBareMetal HostedClusterType = "BareMetal"
+	// HostedClusterTypeOther represents other HyperShift hosted cluster types
+	HostedClusterTypeOther HostedClusterType = "Other"
+)
+
 type InvariantInClusterDisruption struct {
 	namespaceName               string
 	openshiftTestsImagePullSpec string
 	payloadImagePullSpec        string
 	notSupportedReason          string
 	replicas                    int32
 	controlPlaneNodes           int32
-
-	adminRESTConfig *rest.Config
-	kubeClient      kubernetes.Interface
+	hostedClusterType           HostedClusterType
+	adminRESTConfig             *rest.Config
+	kubeClient                  kubernetes.Interface
 }
 
 func NewInvariantInClusterDisruption(info monitortestframework.MonitorTestInitializationInfo) monitortestframework.MonitorTest {
@@ -85,6 +99,69 @@ func NewInvariantInClusterDisruption(info monitortestframework.MonitorTestInitia
 	}
 }
 
+// parseAdminRESTConfigHost parses the adminRESTConfig.Host URL and returns hostname and port
+func (i *InvariantInClusterDisruption) parseAdminRESTConfigHost() (hostname, port string, err error) {
+	parsedURL, err := url.Parse(i.adminRESTConfig.Host)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to parse adminRESTConfig.Host %q: %v", i.adminRESTConfig.Host, err)
+	}
+
+	hostname = parsedURL.Hostname()
+	if hostname == "" {
+		return "", "", fmt.Errorf("no hostname found in adminRESTConfig.Host %q", i.adminRESTConfig.Host)
+	}
+
+	port = parsedURL.Port()
+	if port == "" {
+		port = "6443" // default port
+	}
+
+	return hostname, port, nil
+}
+
+// setKubernetesServiceEnvVars sets the KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT environment variables
+// based on the hosted cluster type
+func (i *InvariantInClusterDisruption) setKubernetesServiceEnvVars(envVars []corev1.EnvVar, apiIntHost, apiIntPort string) []corev1.EnvVar {
+	// Parse adminRESTConfig.Host once for bare metal HyperShift
+	var bareMetalHost, bareMetalPort string
+	var bareMetalErr error
+	if i.hostedClusterType == HostedClusterTypeBareMetal {
+		bareMetalHost, bareMetalPort, bareMetalErr = i.parseAdminRESTConfigHost()
+		if bareMetalErr != nil {
+			logrus.WithError(bareMetalErr).Errorf("Failed to parse adminRESTConfig.Host for bare metal HyperShift")
+		}
+	}
+
+	for j, env := range envVars {
+		switch env.Name {
+		case "KUBERNETES_SERVICE_HOST":
+			if i.hostedClusterType == HostedClusterTypeBareMetal {
+				if bareMetalErr != nil {
+					envVars[j].Value = apiIntHost
+				} else {
+					envVars[j].Value = bareMetalHost
+				}
+			} else {
+				envVars[j].Value = apiIntHost
+			}
+		case "KUBERNETES_SERVICE_PORT":
+			if i.hostedClusterType == HostedClusterTypeAROHCP {
+				// ARO HCP uses port 7443 for the internal API server load balancer
+				envVars[j].Value = "7443"
+			} else if i.hostedClusterType == HostedClusterTypeBareMetal {
+				if bareMetalErr != nil {
+					envVars[j].Value = apiIntPort
+				} else {
+					envVars[j].Value = bareMetalPort
+				}
+			} else {
+				envVars[j].Value = apiIntPort
+			}
+		}
+	}
+	return envVars
+}
+
 func (i *InvariantInClusterDisruption) createDeploymentAndWaitToRollout(ctx context.Context, deploymentObj *appsv1.Deployment) error {
 	deploymentID := uuid.New().String()
 	deploymentObj = disruptionlibrary.UpdateDeploymentENVs(deploymentObj, deploymentID, "")
@@ -113,15 +190,18 @@ func (i *InvariantInClusterDisruption) createDeploymentAndWaitToRollout(ctx cont
 	return nil
 }
 
-func (i *InvariantInClusterDisruption) createInternalLBDeployment(ctx context.Context, apiIntHost string) error {
+func (i *InvariantInClusterDisruption) createInternalLBDeployment(ctx context.Context, apiIntHost, apiIntPort string) error {
 	deploymentObj := resourceread.ReadDeploymentV1OrDie(internalLBDeploymentYaml)
 	deploymentObj.SetNamespace(i.namespaceName)
-	deploymentObj.Spec.Template.Spec.Containers[0].Env[0].Value = apiIntHost
 	// set amount of deployment replicas to make sure it runs on all nodes
 	deploymentObj.Spec.Replicas = &i.replicas
 	// we need to use the openshift-tests image of the destination during an upgrade.
 	deploymentObj.Spec.Template.Spec.Containers[0].Image = i.openshiftTestsImagePullSpec
 
+	// Set the correct host and port for internal API server based on cluster type
+	deploymentObj.Spec.Template.Spec.Containers[0].Env = i.setKubernetesServiceEnvVars(
+		deploymentObj.Spec.Template.Spec.Containers[0].Env, apiIntHost, apiIntPort)
+
 	err := i.createDeploymentAndWaitToRollout(ctx, deploymentObj)
 	if err != nil {
 		return err
@@ -304,9 +384,10 @@ func (i *InvariantInClusterDisruption) StartCollection(ctx context.Context, admi
 	var err error
 	log := logrus.WithField("monitorTest", "apiserver-incluster-availability").WithField("namespace", i.namespaceName).WithField("func", "StartCollection")
 
-	// Check for ARO HCP and skip if detected
+	// Determine hosted cluster type
 	oc := exutil.NewCLI("apiserver-incluster-availability").AsAdmin()
-	var isAROHCPcluster bool
+	i.hostedClusterType = HostedClusterTypeStandalone // Default to standalone
+
 	isHypershift, _ := exutil.IsHypershift(ctx, oc.AdminConfigClient())
 	if isHypershift {
 		_, hcpNamespace, err := exutil.GetHypershiftManagementClusterConfigAndNamespace()
@@ -318,12 +399,29 @@ func (i *InvariantInClusterDisruption) StartCollection(ctx context.Context, admi
 		// For Hypershift, only skip if it's specifically ARO HCP
 		// Use management cluster client to check the control-plane-operator deployment
 		managementOC := exutil.NewHypershiftManagementCLI(hcpNamespace)
+		var isAROHCPcluster bool
 		if isAROHCPcluster, err = exutil.IsAroHCP(ctx, hcpNamespace, managementOC.AdminKubeClient()); err != nil {
 			logrus.WithError(err).Warning("Failed to check if ARO HCP, assuming it's not")
 		} else if isAROHCPcluster {
 			i.notSupportedReason = "platform Hypershift - ARO HCP not supported"
 			return nil
 		}
+
+		// Determine the specific HyperShift variant
+		if isAROHCPcluster {
+			i.hostedClusterType = HostedClusterTypeAROHCP
+		} else {
+			// Check if this is a bare metal HyperShift cluster
+			isBareMetalHypershift, err := exutil.IsBareMetalHyperShiftCluster(ctx, managementOC)
+			if err != nil {
+				logrus.WithError(err).Warning("Failed to check if bare metal HyperShift, assuming other HyperShift type")
+				i.hostedClusterType = HostedClusterTypeOther
+			} else if isBareMetalHypershift {
+				i.hostedClusterType = HostedClusterTypeBareMetal
+			} else {
+				i.hostedClusterType = HostedClusterTypeOther
+			}
+		}
 	}
 
 	if len(i.payloadImagePullSpec) == 0 {
@@ -378,11 +476,30 @@ func (i *InvariantInClusterDisruption) StartCollection(ctx context.Context, admi
 		return fmt.Errorf("error getting openshift infrastructure: %v", err)
 	}
 
-	internalAPI, err := url.Parse(infra.Status.APIServerInternalURL)
-	if err != nil {
-		return fmt.Errorf("error parsing api int url: %v", err)
+	var apiIntHost string
+	var apiIntPort string
+	// Hosted clusters use adminRESTConfig.Host, standalone clusters use APIServerInternalURL
+	isHostedCluster := i.hostedClusterType == HostedClusterTypeAROHCP ||
+		i.hostedClusterType == HostedClusterTypeBareMetal ||
+		i.hostedClusterType == HostedClusterTypeOther
+
+	if isHostedCluster {
+		apiIntHost, apiIntPort, err = i.parseAdminRESTConfigHost()
+		if err != nil {
+			return fmt.Errorf("failed to parse adminRESTConfig.Host: %v", err)
+		}
+	} else {
+		internalAPI, err := url.Parse(infra.Status.APIServerInternalURL)
+		if err != nil {
+			return fmt.Errorf("error parsing api int url: %v", err)
+		}
+		apiIntHost = internalAPI.Hostname()
+		if internalAPI.Port() != "" {
+			apiIntPort = internalAPI.Port()
+		} else {
+			apiIntPort = "6443" // default port
+		}
 	}
-	apiIntHost := internalAPI.Hostname()
 
 	allNodes, err := i.kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
 	if err != nil {
@@ -438,7 +555,7 @@ func (i *InvariantInClusterDisruption) StartCollection(ctx context.Context, admi
 	if err != nil {
 		return fmt.Errorf("error creating localhost: %v", err)
 	}
-	err = i.createInternalLBDeployment(ctx, apiIntHost)
+	err = i.createInternalLBDeployment(ctx, apiIntHost, apiIntPort)
 	if err != nil {
 		return fmt.Errorf("error creating internal LB: %v", err)
 	}

pkg/test/extensions/binary.go

@@ -105,7 +105,10 @@ func InitializeOpenShiftTestsExtensionFramework() (*extension.Registry, *extensi
 		if err := clusterdiscovery.InitializeTestFramework(exutil.TestContext, config, false); err != nil {
 			panic(err)
 		}
-		klog.V(4).Infof("Loaded test configuration: %#v", exutil.TestContext)
+		// Redact the bearer token exposure
+		testContextString := fmt.Sprintf("%#v", exutil.TestContext)
+		redactedTestContext := exutil.RedactBearerToken(testContextString)
+		klog.V(4).Infof("Loaded test configuration: %s", redactedTestContext)
 
 		exutil.TestContext.ReportDir = os.Getenv("TEST_JUNIT_DIR")
 

pkg/test/extensions/util.go

@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	kapierrs "k8s.io/apimachinery/pkg/api/errors"
 	"os"
 	"os/exec"
 	"path"
@@ -18,6 +17,8 @@ import (
 	"strings"
 	"time"
 
+	kapierrs "k8s.io/apimachinery/pkg/api/errors"
+
 	"github.com/openshift-eng/openshift-tests-extension/pkg/dbtime"
 	"github.com/sirupsen/logrus"
 
@@ -182,7 +183,7 @@ func DetermineReleasePayloadImage() (string, error) {
 
 	if len(releaseImage) == 0 {
 		// Note that MicroShift does not have this resource. The test driver must use ENV vars.
-		oc := util.NewCLIWithoutNamespace("default")
+		oc := util.NewCLIForMonitorTest("default")
 		cv, err := oc.AdminConfigClient().ConfigV1().ClusterVersions().Get(context.TODO(), "version",
 			metav1.GetOptions{})
 		if err != nil {

pkg/test/ginkgo/cmd_runsuite.go

@@ -447,6 +447,10 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 		return strings.Contains(t.name, "[sig-network]")
 	})
 
+	netpolTests, networkK8sTests := splitTests(networkK8sTests, func(t *testCase) bool {
+		return strings.Contains(t.name, "Netpol")
+	})
+
 	buildsTests, openshiftTests := splitTests(openshiftTests, func(t *testCase) bool {
 		return strings.Contains(t.name, "[sig-builds]")
 	})
@@ -460,6 +464,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 	logrus.Infof("Found %d storage tests", len(storageTests))
 	logrus.Infof("Found %d network k8s tests", len(networkK8sTests))
 	logrus.Infof("Found %d network tests", len(networkTests))
+	logrus.Infof("Found %d netpol tests", len(netpolTests))
 	logrus.Infof("Found %d builds tests", len(buildsTests))
 	logrus.Infof("Found %d must-gather tests", len(mustGatherTests))
 
@@ -471,6 +476,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 		originalStorage := storageTests
 		originalNetworkK8s := networkK8sTests
 		originalNetwork := networkTests
+		originalNetpol := netpolTests
 		originalBuilds := buildsTests
 		originalMustGather := mustGatherTests
 
@@ -480,11 +486,12 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 			storageTests = append(storageTests, copyTests(originalStorage)...)
 			networkK8sTests = append(networkK8sTests, copyTests(originalNetworkK8s)...)
 			networkTests = append(networkTests, copyTests(originalNetwork)...)
+			netpolTests = append(netpolTests, copyTests(originalNetpol)...)
 			buildsTests = append(buildsTests, copyTests(originalBuilds)...)
 			mustGatherTests = append(mustGatherTests, copyTests(originalMustGather)...)
 		}
 	}
-	expectedTestCount += len(openshiftTests) + len(kubeTests) + len(storageTests) + len(networkK8sTests) + len(networkTests) + len(buildsTests) + len(mustGatherTests)
+	expectedTestCount += len(openshiftTests) + len(kubeTests) + len(storageTests) + len(networkK8sTests) + len(networkTests) + len(netpolTests) + len(buildsTests) + len(mustGatherTests)
 
 	abortFn := neverAbort
 	testCtx := ctx
@@ -522,6 +529,14 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 		q.Execute(testCtx, networkTestsCopy, max(1, parallelism/2), testOutputConfig, abortFn) // run network tests separately.
 		tests = append(tests, networkTestsCopy...)
 
+		// k8s netpol tests are known to be heavy and there are cases when run in parallel it can overload
+		// a cluster causing negative side effects (e.g., https://issues.redhat.com/browse/OCPBUGS-57665)
+		// https://github.com/openshift/origin/pull/26775/changes#diff-998be43366fe821c61ca242aa34949870c9c6df2572cc060000e4cd990a72bebL58-L62
+		// this will only run 2 in parallel at once
+		netpolTestsCopy := copyTests(netpolTests)
+		q.Execute(testCtx, netpolTestsCopy, 2, testOutputConfig, abortFn)
+		tests = append(tests, netpolTestsCopy...)
+
 		buildsTestsCopy := copyTests(buildsTests)
 		q.Execute(testCtx, buildsTestsCopy, max(1, parallelism/2), testOutputConfig, abortFn) // builds tests only run at half the parallelism, so we can avoid high cpu problems.
 		tests = append(tests, buildsTestsCopy...)

pkg/test/ginkgo/status.go

@@ -3,6 +3,7 @@ package ginkgo
 import (
 	"fmt"
 	"io"
+	"os"
 	"sort"
 	"sync"
 )
@@ -38,6 +39,11 @@ func (s *testSuiteProgress) TestEnded(testName string, testRunResult *testRunRes
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
+	if testRunResult == nil {
+		fmt.Fprintln(os.Stderr, "testRunResult is nil")
+		return
+	}
+
 	if isTestFailed(testRunResult.testState) {
 		s.failures++
 	}

pkg/test/ginkgo/test_runner.go

@@ -221,6 +221,11 @@ func recordTestResultInLogWithoutOverlap(testRunResult *testRunResultHandle, tes
 }
 
 func recordTestResultInLog(testRunResult *testRunResultHandle, out io.Writer, includeSuccessfulOutput bool) {
+	if testRunResult == nil {
+		fmt.Fprintln(os.Stderr, "testRunResult is nil")
+		return
+	}
+
 	// output the status of the test
 	switch testRunResult.testState {
 	case TestFlaked:
@@ -257,6 +262,11 @@ func recordTestResultInLog(testRunResult *testRunResultHandle, out io.Writer, in
 }
 
 func recordTestResultInMonitor(testRunResult *testRunResultHandle, monitorRecorder monitorapi.Recorder) {
+	if testRunResult == nil {
+		fmt.Fprintln(os.Stderr, "testRunResult is nil")
+		return
+	}
+
 	eventLevel := monitorapi.Warning
 
 	msg := monitorapi.NewMessage().HumanMessage("e2e test finished")

test/e2e/upgrade/upgrade.go

@@ -403,9 +403,8 @@ func clusterUpgrade(f *framework.Framework, c configv1client.Interface, dc dynam
 		// to be 30 minutes longer. https://issues.redhat.com/browse/OCPBUGS-13059
 		upgradeDurationLimit = 130 * time.Minute
 	case platformType.Architecture == platformidentification.ArchitecturePPC64le:
-		// ppc appears to take just over 75 minutes, let's lock it into that value, so we don't
-		// get worse.
-		upgradeDurationLimit = 80 * time.Minute
+		// ppc takes slightly more than 80 minutes. https://issues.redhat.com/browse/OCPBUGS-63201
+		upgradeDurationLimit = 100 * time.Minute
 	case infra.Status.InfrastructureTopology == configv1.SingleReplicaTopologyMode:
 		// single node takes a lot less since there's one node
 		upgradeDurationLimit = 65 * time.Minute