diff --git a/tests/e2e/framework/common.go b/tests/e2e/framework/common.go index 6fcbbadbe2..95b9c63f4c 100644 --- a/tests/e2e/framework/common.go +++ b/tests/e2e/framework/common.go @@ -3,6 +3,7 @@ package framework import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -3249,3 +3250,42 @@ func (f *Framework) AssertNodeNameIsInTargetAndFactIdentifierInCM(nodes []core.N } return nil } + +// AssertPrometheusRuleComplianceNonCompliantAlert checks that some PrometheusRule in the operator +// namespace has a name containing "compliance" and an alert name containing "NonCompliant" +// (TC 43072 / observability parity with openshift-tests). +func (f *Framework) AssertPrometheusRuleComplianceNonCompliantAlert() error { + out, err := runOCandGetOutput([]string{"get", "prometheusrule", "-n", f.OperatorNamespace, "-o", "json"}) + if err != nil { + return fmt.Errorf("list prometheusrule: %w", err) + } + var pr struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Groups []struct { + Rules []struct { + Alert string `json:"alert"` + } `json:"rules"` + } `json:"groups"` + } `json:"spec"` + } `json:"items"` + } + if err := json.Unmarshal([]byte(out), &pr); err != nil { + return fmt.Errorf("parse prometheusrule list: %w", err) + } + for _, item := range pr.Items { + if !strings.Contains(strings.ToLower(item.Metadata.Name), "compliance") { + continue + } + if len(item.Spec.Groups) == 0 || len(item.Spec.Groups[0].Rules) == 0 { + continue + } + if strings.Contains(item.Spec.Groups[0].Rules[0].Alert, "NonCompliant") { + return nil + } + } + return fmt.Errorf("no compliance PrometheusRule with NonCompliant alert found in %s", f.OperatorNamespace) +} diff --git a/tests/e2e/framework/utils.go b/tests/e2e/framework/utils.go index 06c937c806..bdb68a3019 100644 --- a/tests/e2e/framework/utils.go +++ b/tests/e2e/framework/utils.go @@ -48,6 +48,67 @@ type PrometheusTarget struct { ScrapeTimeout string `json:"scrapeTimeout"` } +// labelSetJSON unmarshals Prometheus /api/v1/targets "labels" as either a JSON object (common) +// or an array of {name,value} pairs, so we do not depend on vendoring prometheus/prometheus. +type labelSetJSON struct { + m map[string]string +} + +func (l *labelSetJSON) UnmarshalJSON(data []byte) error { + l.m = nil + s := strings.TrimSpace(string(data)) + if s == "" || s == "null" { + return nil + } + if strings.HasPrefix(s, "{") { + if err := json.Unmarshal(data, &l.m); err != nil { + return err + } + return nil + } + if strings.HasPrefix(s, "[") { + var pairs []struct { + Name string `json:"name"` + Value string `json:"value"` + } + if err := json.Unmarshal(data, &pairs); err != nil { + return err + } + l.m = make(map[string]string, len(pairs)) + for _, p := range pairs { + if p.Name != "" { + l.m[p.Name] = p.Value + } + } + return nil + } + return fmt.Errorf("unexpected Prometheus target labels JSON (prefix %.80q)", s) +} + +func (l labelSetJSON) labelEquals(name, value string) bool { + if l.m == nil { + return false + } + v, ok := l.m[name] + return ok && v == value +} + +func (l labelSetJSON) String() string { + if l.m == nil { + return "{}" + } + return fmt.Sprintf("%v", l.m) +} + +// prometheusScrapeTarget mirrors the subset of fields we need from Prometheus targets API JSON +// without importing github.com/prometheus/prometheus (not vendored; breaks -mod=vendor builds). +type prometheusScrapeTarget struct { + Labels labelSetJSON `json:"labels"` + Health string `json:"health"` + LastScrape string `json:"lastScrape"` + LastError string `json:"lastError"` +} + func (f *Framework) AssertMustHaveParsedProfiles(pbName, productType, productName string) error { var l compv1alpha1.ProfileList o := &client.ListOptions{ @@ -409,10 +470,11 @@ func runOCandGetOutput(arg []string) (string, error) { cmd := exec.Command(ocPath, arg...) out, err := cmd.CombinedOutput() + outStr := string(out) if err != nil { - return "", fmt.Errorf("Failed to run oc command: %v", err) + return outStr, fmt.Errorf("Failed to run oc command: %v", err) } - return string(out), nil + return outStr, nil } // createServiceAccount creates a service account @@ -616,6 +678,30 @@ func getMetricResults(namespace string) (string, error) { return string(out), nil } +// WaitForMetricOutputContainsAll polls the operator metrics-co scrape (same path as getMetricResults) +// until the response body contains every substring (TC 43072 style substring checks). +func WaitForMetricOutputContainsAll(namespace string, substrings []string, timeout, interval time.Duration) error { + var lastMiss string + err := wait.Poll(interval, timeout, func() (bool, error) { + out, gerr := getMetricResults(namespace) + if gerr != nil { + lastMiss = gerr.Error() + return false, nil + } + for _, s := range substrings { + if !strings.Contains(out, s) { + lastMiss = fmt.Sprintf("missing substring %q", s) + return false, nil + } + } + return true, nil + }) + if err != nil { + return fmt.Errorf("wait for metric substrings: %w (last: %s)", err, lastMiss) + } + return nil +} + func getTestMetricsCMD(namespace string) string { var curlCMD = "curl -ks -H \"Authorization: Bearer `cat /var/run/secrets/kubernetes.io/serviceaccount/token`\" " return curlCMD + fmt.Sprintf("https://metrics.%s.svc:8585/metrics-co", namespace) diff --git a/tests/e2e/serial/main_test.go b/tests/e2e/serial/main_test.go index 6de021fc38..89bc2e9d04 100644 --- a/tests/e2e/serial/main_test.go +++ b/tests/e2e/serial/main_test.go @@ -2877,3 +2877,76 @@ func TestOpenSCAPRuleMetadataPropagation(t *testing.T) { t.Errorf("operator-managed scan label should not be overridden, got %q", checkResult.Labels[compv1alpha1.ComplianceScanLabel]) } } + +// TestErrorMetricsPrometheusRule uses a suite intended to reach phase DONE, result ERROR: +// real content image with a non-existent content path so OpenSCAP cannot load the data stream +// and the scan ends DONE/ERROR. +func TestErrorMetricsPrometheusRule(t *testing.T) { + f := framework.Global + + ns := &corev1.Namespace{} + if err := f.Client.Get(context.TODO(), types.NamespacedName{Name: f.OperatorNamespace}, ns); err != nil { + t.Fatal(err) + } + if ns.Labels == nil || ns.Labels["openshift.io/cluster-monitoring"] != "true" { + t.Fatalf("namespace %s does not have openshift.io/cluster-monitoring=true", f.OperatorNamespace) + } + metricsSvc := &corev1.Service{} + if err := f.Client.Get(context.TODO(), types.NamespacedName{Name: "metrics", Namespace: f.OperatorNamespace}, metricsSvc); err != nil { + t.Fatal(err) + } + + base := framework.GetObjNameFromTest(t) + suiteName := base + "-test-suite" + scanName := base + "-scan" + selectWorkers := map[string]string{"node-role.kubernetes.io/worker": ""} + + suite := &compv1alpha1.ComplianceSuite{ + ObjectMeta: metav1.ObjectMeta{ + Name: suiteName, + Namespace: f.OperatorNamespace, + }, + Spec: compv1alpha1.ComplianceSuiteSpec{ + ComplianceSuiteSettings: compv1alpha1.ComplianceSuiteSettings{ + AutoApplyRemediations: false, + }, + Scans: []compv1alpha1.ComplianceScanSpecWrapper{ + { + Name: scanName, + ComplianceScanSpec: compv1alpha1.ComplianceScanSpec{ + Profile: "xccdf_org.ssgproject.content_profile_moderate", + Content: "this-file-does-not-exist.xml", + ContentImage: "quay.io/compliance-operator/compliance-operator-content:latest", + NodeSelector: selectWorkers, + }, + }, + }, + }, + } + + if err := f.Client.Create(context.TODO(), suite, nil); err != nil { + t.Fatal(err) + } + defer func() { + _ = f.Client.Delete(context.TODO(), suite) + }() + + if err := f.WaitForSuiteScansStatus(f.OperatorNamespace, suiteName, compv1alpha1.PhaseDone, compv1alpha1.ResultError); err != nil { + t.Fatalf("suite %s did not reach DONE/ERROR: %v", suiteName, err) + } + + wantScanMetric := fmt.Sprintf(`compliance_operator_compliance_scan_status_total{name="%s",phase="DONE",result="ERROR"`, scanName) + wantSuiteMetric := fmt.Sprintf(`compliance_operator_compliance_state{name="%s"}`, suiteName) + if err := framework.WaitForMetricOutputContainsAll( + f.OperatorNamespace, + []string{wantScanMetric, wantSuiteMetric}, + 3*time.Minute, + framework.RetryInterval, + ); err != nil { + t.Fatalf("metrics scrape (same path as getMetricResults / TestSingleScanSucceeds style): %v", err) + } + + if err := f.AssertPrometheusRuleComplianceNonCompliantAlert(); err != nil { + t.Fatalf("PrometheusRule NonCompliant alert: %v", err) + } +}