diff --git a/tests/e2e/framework/common.go b/tests/e2e/framework/common.go index 6fcbbadbe2..80d6743ff5 100644 --- a/tests/e2e/framework/common.go +++ b/tests/e2e/framework/common.go @@ -3,6 +3,7 @@ package framework import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -3249,3 +3250,175 @@ func (f *Framework) AssertNodeNameIsInTargetAndFactIdentifierInCM(nodes []core.N } return nil } + +// GetClusterArchitecture returns cluster architecture string (or "multi" when mixed). +func (f *Framework) GetClusterArchitecture() (string, error) { + nodeList := &corev1.NodeList{} + if err := f.Client.List(context.TODO(), nodeList); err != nil { + return "", err + } + seen := map[string]struct{}{} + for i := range nodeList.Items { + arch := strings.ToLower(nodeList.Items[i].Status.NodeInfo.Architecture) + if arch != "" { + seen[arch] = struct{}{} + } + } + if len(seen) == 0 { + return "", fmt.Errorf("no nodes with architecture found") + } + if len(seen) > 1 { + return "multi", nil + } + for arch := range seen { + return arch, nil + } + return "", fmt.Errorf("cannot determine architecture") +} + +func compareVersion(v1, v2 string) int { + s1 := strings.Split(v1, ".") + s2 := strings.Split(v2, ".") + maxLen := len(s1) + if len(s2) > maxLen { + maxLen = len(s2) + } + for i := 0; i < maxLen; i++ { + a := 0 + b := 0 + if i < len(s1) { + part := regexp.MustCompile(`[^0-9]`).ReplaceAllString(s1[i], "") + if part != "" { + a, _ = strconv.Atoi(part) + } + } + if i < len(s2) { + part := regexp.MustCompile(`[^0-9]`).ReplaceAllString(s2[i], "") + if part != "" { + b, _ = strconv.Atoi(part) + } + } + if a > b { + return 1 + } + if a < b { + return -1 + } + } + return 0 +} + +func (f *Framework) GetInstalledComplianceOperatorCSV() (string, error) { + out, err := runOCandGetOutput([]string{ + "get", "csv", "-n", f.OperatorNamespace, + "-l", "operators.coreos.com/compliance-operator." + f.OperatorNamespace + "=", + "-o", "jsonpath={.items[0].metadata.name}", + }) + if err != nil { + return "", err + } + name := strings.TrimSpace(strings.Trim(out, "'")) + if name == "" { + return "", fmt.Errorf("installed CSV not found in namespace %s", f.OperatorNamespace) + } + return name, nil +} + +// IsComplianceOperatorUpgradable checks if catalog/channel advertises newer CSV than installed. +func (f *Framework) IsComplianceOperatorUpgradable(catalogSource, channel string) (bool, error) { + oldCSV, err := f.GetInstalledComplianceOperatorCSV() + if err != nil { + return false, err + } + oldVersion := strings.TrimPrefix(oldCSV, "compliance-operator.v") + + out, err := runOCandGetOutput([]string{ + "get", "packagemanifest", "-n", "openshift-marketplace", + "-l", fmt.Sprintf("catalog=%s", catalogSource), + "-o", "json", + }) + if err != nil { + return false, err + } + var pm struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Channels []struct { + Name string `json:"name"` + CurrentCSV string `json:"currentCSV"` + } `json:"channels"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal([]byte(out), &pm); err != nil { + return false, err + } + for _, item := range pm.Items { + if item.Metadata.Name != "compliance-operator" { + continue + } + for _, ch := range item.Status.Channels { + if ch.Name != channel { + continue + } + curVersion := strings.TrimPrefix(ch.CurrentCSV, "compliance-operator.v") + return compareVersion(curVersion, oldVersion) == 1, nil + } + } + return false, nil +} + +func (f *Framework) PatchComplianceOperatorSubscriptionSource(source string) error { + _, err := runOCandGetOutput([]string{ + "patch", "subscription", "compliance-operator", "-n", f.OperatorNamespace, + "--type", "merge", "-p", fmt.Sprintf(`{"spec":{"source":"%s"}}`, source), + }) + return err +} + +func (f *Framework) WaitForComplianceOperatorCSVUpgrade(oldCSV string) error { + return wait.PollImmediate(10*time.Second, Timeout, func() (bool, error) { + curCSVOut, err := runOCandGetOutput([]string{ + "get", "subscription", "compliance-operator", "-n", f.OperatorNamespace, + "-o", "jsonpath={.status.currentCSV}", + }) + if err != nil { + return false, nil + } + curCSV := strings.TrimSpace(curCSVOut) + if curCSV == "" || curCSV == oldCSV { + return false, nil + } + phaseOut, err := runOCandGetOutput([]string{ + "get", "csv", curCSV, "-n", f.OperatorNamespace, + "-o", "jsonpath={.status.phase}", + }) + if err != nil { + return false, nil + } + return strings.Contains(strings.TrimSpace(phaseOut), "Succeeded"), nil + }) +} + +func (f *Framework) AssertComplianceOperatorPodRunning() error { + pods, err := f.KubeClient.CoreV1().Pods(f.OperatorNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "name=compliance-operator", + }) + if err != nil { + return err + } + if len(pods.Items) == 0 { + return fmt.Errorf("no compliance-operator pod found in %s", f.OperatorNamespace) + } + if pods.Items[0].Status.Phase != corev1.PodRunning { + return fmt.Errorf("compliance-operator pod phase is %s", pods.Items[0].Status.Phase) + } + return nil +} + +func (f *Framework) EnsureE2ESchemes() error { + return f.addFrameworks() +} \ No newline at end of file diff --git a/tests/e2e/framework/utils.go b/tests/e2e/framework/utils.go index 06c937c806..38d748eade 100644 --- a/tests/e2e/framework/utils.go +++ b/tests/e2e/framework/utils.go @@ -21,6 +21,7 @@ import ( "github.com/ComplianceAsCode/compliance-operator/pkg/utils" imagev1 "github.com/openshift/api/image/v1" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -401,6 +402,48 @@ func (f *Framework) AssertMetricsEndpointUsesHTTPVersion(endpoint, version strin return nil } +func (f *Framework) SetScanSettingSuspend(scanSettingName string, suspend bool) error { + ss := &compv1alpha1.ScanSetting{} + key := types.NamespacedName{Name: scanSettingName, Namespace: f.OperatorNamespace} + if err := f.Client.Get(context.TODO(), key, ss); err != nil { + return err + } + updated := ss.DeepCopy() + updated.Suspend = suspend + return f.Client.Update(context.TODO(), updated) +} + +func (f *Framework) GetCronJobLastSuccessfulTime(cronJobName string) (string, error) { + job := &batchv1.CronJob{} + key := types.NamespacedName{Name: cronJobName, Namespace: f.OperatorNamespace} + if err := f.Client.Get(context.TODO(), key, job); err != nil { + return "", err + } + if job.Status.LastSuccessfulTime == nil { + return "", nil + } + return job.Status.LastSuccessfulTime.UTC().Format(time.RFC3339Nano), nil +} + +func (f *Framework) WaitForCronJobLastSuccessfulTime(cronJobName string, timeout time.Duration) (string, error) { + lastSuccessfulTime := "" + err := wait.Poll(RetryInterval, timeout, func() (bool, error) { + current, err := f.GetCronJobLastSuccessfulTime(cronJobName) + if err != nil { + return false, err + } + if current == "" { + return false, nil + } + lastSuccessfulTime = current + return true, nil + }) + if err != nil { + return "", err + } + return lastSuccessfulTime, nil +} + func runOCandGetOutput(arg []string) (string, error) { ocPath, err := exec.LookPath("oc") if err != nil { diff --git a/tests/e2e/upgrade/main_test.go b/tests/e2e/upgrade/main_test.go new file mode 100644 index 0000000000..05eee728a9 --- /dev/null +++ b/tests/e2e/upgrade/main_test.go @@ -0,0 +1,225 @@ +package upgrade_e2e + +import ( + "context" + "log" + "os" + "testing" + "time" + + compv1alpha1 "github.com/ComplianceAsCode/compliance-operator/pkg/apis/compliance/v1alpha1" + compsuitectrl "github.com/ComplianceAsCode/compliance-operator/pkg/controller/compliancesuite" + "github.com/ComplianceAsCode/compliance-operator/tests/e2e/framework" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMain(m *testing.M) { + f := framework.NewFramework() + + // Use with TEST_OPERATOR_NAMESPACE=openshift-compliance (or your install ns) when Compliance + // Operator is already installed. Skips manifest deploy, CRD apply, e2e MCP, and TearDown. + skipSetup := os.Getenv("E2E_SKIP_FRAMEWORK_SETUP") == "true" + if skipSetup { + if err := f.EnsureE2ESchemes(); err != nil { + log.Fatalf("EnsureE2ESchemes: %v", err) + } + } else { + if err := f.SetUp(); err != nil { + log.Fatal(err) + } + } + + exitCode := m.Run() + if !skipSetup && (exitCode == 0 || (exitCode > 0 && f.CleanUpOnError())) { + if err := f.TearDown(); err != nil { + log.Fatal(err) + } + } + os.Exit(exitCode) +} + +// TestUpgradeScanSuspendResumeRerunner ports test case 37721/56351: +// - run ocp4-cis through custom ScanSetting/ScanSettingBinding, +// - upgrade compliance-operator to catalog source "compliance-operator", +// - verify rerunner CronJob behavior while suspending/resuming ScanSetting. +func TestUpgradeScanSuspendResumeRerunner(t *testing.T) { + f := framework.Global + + arch, err := f.GetClusterArchitecture() + if err != nil { + t.Fatalf("cluster architecture: %v", err) + } + if arch == "arm64" || arch == "multi" { + t.Skipf("skipping on architecture %s (upstream parity)", arch) + } + + upgradable, err := f.IsComplianceOperatorUpgradable("compliance-operator", "stable") + if err != nil { + t.Fatalf("check upgradable: %v", err) + } + if !upgradable { + t.Skip("compliance-operator stable channel has no newer CSV than installed") + } + + oldCSV, err := f.GetInstalledComplianceOperatorCSV() + if err != nil { + t.Fatalf("get installed CSV: %v", err) + } + t.Logf("Old CSV version: %v", oldCSV) + + scanSettingName := framework.GetObjNameFromTest(t) + "-scansetting" + scanSettingSchedule := "*/3 * * * *" + scanSetting := &compv1alpha1.ScanSetting{ + ObjectMeta: metav1.ObjectMeta{ + Name: scanSettingName, + Namespace: f.OperatorNamespace, + }, + ComplianceSuiteSettings: compv1alpha1.ComplianceSuiteSettings{ + AutoApplyRemediations: false, + AutoUpdateRemediations: false, + Schedule: scanSettingSchedule, + Suspend: false, + }, + ComplianceScanSettings: compv1alpha1.ComplianceScanSettings{ + RawResultStorage: compv1alpha1.RawResultStorageSettings{ + Size: "2Gi", + Rotation: 3, + }, + Debug: true, + }, + Roles: []string{"master", "worker"}, + } + if err := f.Client.Create(context.TODO(), scanSetting, nil); err != nil { + t.Fatalf("create ScanSetting %s: %v", scanSettingName, err) + } + defer func() { + if err := f.Client.Delete(context.TODO(), scanSetting); err != nil && !apierrors.IsNotFound(err) { + t.Logf("cleanup ScanSetting %s: %v", scanSettingName, err) + } + }() + + ssbName := framework.GetObjNameFromTest(t) + "-binding" + ssb := &compv1alpha1.ScanSettingBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: ssbName, + Namespace: f.OperatorNamespace, + }, + Profiles: []compv1alpha1.NamedObjectReference{ + { + Name: "ocp4-cis", + Kind: "Profile", + APIGroup: "compliance.openshift.io/v1alpha1", + }, + }, + SettingsRef: &compv1alpha1.NamedObjectReference{ + Name: scanSettingName, + Kind: "ScanSetting", + APIGroup: "compliance.openshift.io/v1alpha1", + }, + } + if err := f.Client.Create(context.TODO(), ssb, nil); err != nil { + t.Fatalf("create ScanSettingBinding %s: %v", ssbName, err) + } + defer func() { + if err := f.DeleteScanSettingBindingAndWaitForCleanup(ssb); err != nil { + t.Logf("cleanup ScanSettingBinding %s: %v", ssbName, err) + } + }() + + if err := f.WaitForSuiteScansStatus(f.OperatorNamespace, ssbName, compv1alpha1.PhaseRunning, compv1alpha1.ResultNotAvailable); err != nil { + t.Fatalf("suite %s did not enter RUNNING before upgrade: %v", ssbName, err) + } + if err := f.WaitForSuiteScansStatus(f.OperatorNamespace, ssbName, compv1alpha1.PhaseDone, compv1alpha1.ResultNonCompliant); err != nil { + t.Fatalf("suite %s not DONE/NON-COMPLIANT before upgrade: %v", ssbName, err) + } + if err := f.WaitForCronJobWithSchedule(f.OperatorNamespace, ssbName, scanSettingSchedule); err != nil { + t.Fatalf("rerunner cronjob %s schedule mismatch before upgrade: %v", ssbName, err) + } + + if err := f.PatchComplianceOperatorSubscriptionSource("compliance-operator"); err != nil { + t.Fatalf("patch subscription source: %v", err) + } + time.Sleep(10 * time.Second) + if err := f.WaitForComplianceOperatorCSVUpgrade(oldCSV); err != nil { + t.Fatalf("wait for upgraded CSV: %v", err) + } + if err := f.AssertComplianceOperatorPodRunning(); err != nil { + t.Fatalf("compliance-operator pod is not running: %v", err) + } + if err := f.WaitForProfileBundleStatus("ocp4", compv1alpha1.DataStreamValid); err != nil { + t.Fatalf("ocp4 profile bundle is not VALID: %v", err) + } + if arch == "amd64" { + if err := f.WaitForProfileBundleStatus("rhcos4", compv1alpha1.DataStreamValid); err != nil { + t.Fatalf("rhcos4 profile bundle is not VALID: %v", err) + } + } + + if err := f.WaitForSuiteScansStatus(f.OperatorNamespace, ssbName, compv1alpha1.PhaseRunning, compv1alpha1.ResultNotAvailable); err != nil { + t.Fatalf("suite %s did not enter RUNNING after upgrade: %v", ssbName, err) + } + if err := f.WaitForSuiteScansStatus(f.OperatorNamespace, ssbName, compv1alpha1.PhaseDone, compv1alpha1.ResultNonCompliant); err != nil { + t.Fatalf("suite %s not DONE/NON-COMPLIANT after first post-upgrade run: %v", ssbName, err) + } + + rerunnerName := compsuitectrl.GetRerunnerName(ssbName) + lastSuccessfulTime, err := f.WaitForCronJobLastSuccessfulTime(rerunnerName, 6*time.Minute) + if err != nil { + t.Fatalf("get baseline cronjob lastSuccessfulTime for %s: %v", rerunnerName, err) + } + + if err := f.SetScanSettingSuspend(scanSettingName, true); err != nil { + t.Fatalf("suspend ScanSetting %s: %v", scanSettingName, err) + } + if err := f.WaitForScanSettingBindingStatus(f.OperatorNamespace, ssbName, compv1alpha1.ScanSettingBindingPhaseSuspended); err != nil { + t.Fatalf("ScanSettingBinding %s failed to suspend: %v", ssbName, err) + } + if err := f.AssertScanSettingBindingConditionIsSuspended(ssbName, f.OperatorNamespace); err != nil { + t.Fatalf("ScanSettingBinding %s suspended condition mismatch: %v", ssbName, err) + } + if err := f.AssertCronJobIsSuspended(rerunnerName); err != nil { + t.Fatalf("CronJob %s should be suspended: %v", rerunnerName, err) + } + lastSuccessfulTimeSuspended, err := f.GetCronJobLastSuccessfulTime(rerunnerName) + if err != nil { + t.Fatalf("get suspended cronjob lastSuccessfulTime for %s: %v", rerunnerName, err) + } + if lastSuccessfulTimeSuspended != lastSuccessfulTime { + t.Fatalf("expected suspended lastSuccessfulTime (%q) to equal baseline (%q)", lastSuccessfulTimeSuspended, lastSuccessfulTime) + } + + if err := f.SetScanSettingSuspend(scanSettingName, false); err != nil { + t.Fatalf("resume ScanSetting %s: %v", scanSettingName, err) + } + if err := f.WaitForScanSettingBindingStatus(f.OperatorNamespace, ssbName, compv1alpha1.ScanSettingBindingPhaseReady); err != nil { + t.Fatalf("ScanSettingBinding %s failed to resume: %v", ssbName, err) + } + if err := f.AssertScanSettingBindingConditionIsReady(ssbName, f.OperatorNamespace); err != nil { + t.Fatalf("ScanSettingBinding %s ready condition mismatch after resume: %v", ssbName, err) + } + if err := f.AssertCronJobIsNotSuspended(rerunnerName); err != nil { + t.Fatalf("CronJob %s should be active after resume: %v", rerunnerName, err) + } + + if err := f.WaitForSuiteScansStatus(f.OperatorNamespace, ssbName, compv1alpha1.PhaseRunning, compv1alpha1.ResultNotAvailable); err != nil { + t.Fatalf("suite %s did not enter RUNNING after resume: %v", ssbName, err) + } + if err := f.WaitForSuiteScansStatusAnyResult( + f.OperatorNamespace, + ssbName, + compv1alpha1.PhaseDone, + compv1alpha1.ResultNonCompliant, + compv1alpha1.ResultInconsistent, + ); err != nil { + t.Fatalf("suite %s not DONE with allowed result after resume: %v", ssbName, err) + } + + lastSuccessfulTimeResumed, err := f.WaitForCronJobLastSuccessfulTime(rerunnerName, 6*time.Minute) + if err != nil { + t.Fatalf("get resumed cronjob lastSuccessfulTime for %s: %v", rerunnerName, err) + } + if lastSuccessfulTimeResumed == lastSuccessfulTime { + t.Fatalf("expected resumed lastSuccessfulTime to differ from baseline; both were %q", lastSuccessfulTime) + } +}