CMP-3829: The metrics and alerts are getting reported for Compliance …

tests/e2e/framework/common.go

@@ -3,6 +3,7 @@ package framework
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -3249,3 +3250,42 @@ func (f *Framework) AssertNodeNameIsInTargetAndFactIdentifierInCM(nodes []core.N
 	}
 	return nil
 }
+
+// AssertPrometheusRuleComplianceNonCompliantAlert checks that some PrometheusRule in the operator
+// namespace has a name containing "compliance" and an alert name containing "NonCompliant"
+// (TC 43072 / observability parity with openshift-tests).
+func (f *Framework) AssertPrometheusRuleComplianceNonCompliantAlert() error {
+	out, err := runOCandGetOutput([]string{"get", "prometheusrule", "-n", f.OperatorNamespace, "-o", "json"})
+	if err != nil {
+		return fmt.Errorf("list prometheusrule: %w", err)
+	}
+	var pr struct {
+		Items []struct {
+			Metadata struct {
+				Name string `json:"name"`
+			} `json:"metadata"`
+			Spec struct {
+				Groups []struct {
+					Rules []struct {
+						Alert string `json:"alert"`
+					} `json:"rules"`
+				} `json:"groups"`
+			} `json:"spec"`
+		} `json:"items"`
+	}
+	if err := json.Unmarshal([]byte(out), &pr); err != nil {
+		return fmt.Errorf("parse prometheusrule list: %w", err)
+	}
+	for _, item := range pr.Items {
+		if !strings.Contains(strings.ToLower(item.Metadata.Name), "compliance") {
+			continue
+		}
+		if len(item.Spec.Groups) == 0 || len(item.Spec.Groups[0].Rules) == 0 {
+			continue
+		}
+		if strings.Contains(item.Spec.Groups[0].Rules[0].Alert, "NonCompliant") {
+			return nil
+		}
+	}
+	return fmt.Errorf("no compliance PrometheusRule with NonCompliant alert found in %s", f.OperatorNamespace)
+}

tests/e2e/framework/utils.go

@@ -48,6 +48,67 @@ type PrometheusTarget struct {
 	ScrapeTimeout      string            `json:"scrapeTimeout"`
 }
 
+// labelSetJSON unmarshals Prometheus /api/v1/targets "labels" as either a JSON object (common)
+// or an array of {name,value} pairs, so we do not depend on vendoring prometheus/prometheus.
+type labelSetJSON struct {
+	m map[string]string
+}
+
+func (l *labelSetJSON) UnmarshalJSON(data []byte) error {
+	l.m = nil
+	s := strings.TrimSpace(string(data))
+	if s == "" || s == "null" {
+		return nil
+	}
+	if strings.HasPrefix(s, "{") {
+		if err := json.Unmarshal(data, &l.m); err != nil {
+			return err
+		}
+		return nil
+	}
+	if strings.HasPrefix(s, "[") {
+		var pairs []struct {
+			Name  string `json:"name"`
+			Value string `json:"value"`
+		}
+		if err := json.Unmarshal(data, &pairs); err != nil {
+			return err
+		}
+		l.m = make(map[string]string, len(pairs))
+		for _, p := range pairs {
+			if p.Name != "" {
+				l.m[p.Name] = p.Value
+			}
+		}
+		return nil
+	}
+	return fmt.Errorf("unexpected Prometheus target labels JSON (prefix %.80q)", s)
+}
+
+func (l labelSetJSON) labelEquals(name, value string) bool {
+	if l.m == nil {
+		return false
+	}
+	v, ok := l.m[name]
+	return ok && v == value
+}
+
+func (l labelSetJSON) String() string {
+	if l.m == nil {
+		return "{}"
+	}
+	return fmt.Sprintf("%v", l.m)
+}
+
+// prometheusScrapeTarget mirrors the subset of fields we need from Prometheus targets API JSON
+// without importing github.com/prometheus/prometheus (not vendored; breaks -mod=vendor builds).
+type prometheusScrapeTarget struct {
+	Labels     labelSetJSON `json:"labels"`
+	Health     string       `json:"health"`
+	LastScrape string       `json:"lastScrape"`
+	LastError  string       `json:"lastError"`
+}
+
 func (f *Framework) AssertMustHaveParsedProfiles(pbName, productType, productName string) error {
 	var l compv1alpha1.ProfileList
 	o := &client.ListOptions{
@@ -409,10 +470,11 @@ func runOCandGetOutput(arg []string) (string, error) {
 
 	cmd := exec.Command(ocPath, arg...)
 	out, err := cmd.CombinedOutput()
+	outStr := string(out)
 	if err != nil {
-		return "", fmt.Errorf("Failed to run oc command: %v", err)
+		return outStr, fmt.Errorf("Failed to run oc command: %v", err)
 	}
-	return string(out), nil
+	return outStr, nil
 }
 
 // createServiceAccount creates a service account
@@ -616,6 +678,30 @@ func getMetricResults(namespace string) (string, error) {
 	return string(out), nil
 }
 
+// WaitForMetricOutputContainsAll polls the operator metrics-co scrape (same path as getMetricResults)
+// until the response body contains every substring (TC 43072 style substring checks).
+func WaitForMetricOutputContainsAll(namespace string, substrings []string, timeout, interval time.Duration) error {
+	var lastMiss string
+	err := wait.Poll(interval, timeout, func() (bool, error) {
+		out, gerr := getMetricResults(namespace)
+		if gerr != nil {
+			lastMiss = gerr.Error()
+			return false, nil
+		}
+		for _, s := range substrings {
+			if !strings.Contains(out, s) {
+				lastMiss = fmt.Sprintf("missing substring %q", s)
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("wait for metric substrings: %w (last: %s)", err, lastMiss)
+	}
+	return nil
+}
+
 func getTestMetricsCMD(namespace string) string {
 	var curlCMD = "curl -ks -H \"Authorization: Bearer `cat /var/run/secrets/kubernetes.io/serviceaccount/token`\" "
 	return curlCMD + fmt.Sprintf("https://metrics.%s.svc:8585/metrics-co", namespace)

tests/e2e/serial/main_test.go

@@ -2877,3 +2877,76 @@ func TestOpenSCAPRuleMetadataPropagation(t *testing.T) {
 		t.Errorf("operator-managed scan label should not be overridden, got %q", checkResult.Labels[compv1alpha1.ComplianceScanLabel])
 	}
 }
+
+// TestErrorMetricsPrometheusRule uses a suite intended to reach phase DONE, result ERROR:
+// real content image with a non-existent content path so OpenSCAP cannot load the data stream
+// and the scan ends DONE/ERROR.
+func TestErrorMetricsPrometheusRule(t *testing.T) {
+	f := framework.Global
+
+	ns := &corev1.Namespace{}
+	if err := f.Client.Get(context.TODO(), types.NamespacedName{Name: f.OperatorNamespace}, ns); err != nil {
+		t.Fatal(err)
+	}
+	if ns.Labels == nil || ns.Labels["openshift.io/cluster-monitoring"] != "true" {
+		t.Fatalf("namespace %s does not have openshift.io/cluster-monitoring=true", f.OperatorNamespace)
+	}
+	metricsSvc := &corev1.Service{}
+	if err := f.Client.Get(context.TODO(), types.NamespacedName{Name: "metrics", Namespace: f.OperatorNamespace}, metricsSvc); err != nil {
+		t.Fatal(err)
+	}
+
+	base := framework.GetObjNameFromTest(t)
+	suiteName := base + "-test-suite"
+	scanName := base + "-scan"
+	selectWorkers := map[string]string{"node-role.kubernetes.io/worker": ""}
+
+	suite := &compv1alpha1.ComplianceSuite{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      suiteName,
+			Namespace: f.OperatorNamespace,
+		},
+		Spec: compv1alpha1.ComplianceSuiteSpec{
+			ComplianceSuiteSettings: compv1alpha1.ComplianceSuiteSettings{
+				AutoApplyRemediations: false,
+			},
+			Scans: []compv1alpha1.ComplianceScanSpecWrapper{
+				{
+					Name: scanName,
+					ComplianceScanSpec: compv1alpha1.ComplianceScanSpec{
+						Profile:      "xccdf_org.ssgproject.content_profile_moderate",
+						Content:      "this-file-does-not-exist.xml",
+						ContentImage: "quay.io/compliance-operator/compliance-operator-content:latest",
+						NodeSelector: selectWorkers,
+					},
+				},
+			},
+		},
+	}
+
+	if err := f.Client.Create(context.TODO(), suite, nil); err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = f.Client.Delete(context.TODO(), suite)
+	}()
+
+	if err := f.WaitForSuiteScansStatus(f.OperatorNamespace, suiteName, compv1alpha1.PhaseDone, compv1alpha1.ResultError); err != nil {
+		t.Fatalf("suite %s did not reach DONE/ERROR: %v", suiteName, err)
+	}
+
+	wantScanMetric := fmt.Sprintf(`compliance_operator_compliance_scan_status_total{name="%s",phase="DONE",result="ERROR"`, scanName)
+	wantSuiteMetric := fmt.Sprintf(`compliance_operator_compliance_state{name="%s"}`, suiteName)
+	if err := framework.WaitForMetricOutputContainsAll(
+		f.OperatorNamespace,
+		[]string{wantScanMetric, wantSuiteMetric},
+		3*time.Minute,
+		framework.RetryInterval,
+	); err != nil {
+		t.Fatalf("metrics scrape (same path as getMetricResults / TestSingleScanSucceeds style): %v", err)
+	}
+
+	if err := f.AssertPrometheusRuleComplianceNonCompliantAlert(); err != nil {
+		t.Fatalf("PrometheusRule NonCompliant alert: %v", err)
+	}
+}