From c7533ef524b79e53a65f20bb76675cce0a09702c Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 11:32:01 -0400 Subject: [PATCH 01/25] Add PostgreSQL data provider for Component Readiness and enhance e2e testing When refactoring Component Readiness with AI assistance, the lack of tests made it difficult to verify changes didn't break anything. Adding comprehensive e2e tests made the refactor dramatically more reliable -- the AI could detect and fix regressions as it worked. Component Readiness was tightly coupled to BigQuery. The first step was extracting a DataProvider interface, then adding implementations behind it. A mock provider with static fixtures came first, but was replaced with a PostgreSQL provider because: 1) we'd eventually like to run CR against postgres in production, 2) postgres is already available in e2e infrastructure, and 3) a real database lets us fully test triage, regressions, and other write-path features. The e2e suite now seeds realistic synthetic data (4800 job runs across 4 releases with deterministic pass/fail rates), launches the full API server against it, and runs 70+ tests covering component reports, test details, variants, regressions, triage workflows, and the regression tracker. This also enables future UI testing with tools like Playwright or headless Chromium against a fully functional API. Changes: - Add pgprovider implementing the ComponentRedinessDataProvider interface - Add seed-data command generating synthetic prow job runs and test results - Rewrite e2e tests to use postgres provider exclusively (BigQuery no longer needed for e2e) - Add e2e tests for test_details, variants, regressions, columnGroupBy/ dbGroupBy, and variant cross-compare endpoints - Fix triage e2e tests to work with real seed data regressions instead of fragile cache injection (delete E2ECacheManipulator) - Fix test isolation in regressiontracker tests (scoped cleanup) - Add optional data sync test (when GCS_SA_JSON_PATH is set) to verify prow/GCS loading still works - Add server metrics for postgres query performance Co-Authored-By: Claude Opus 4.6 --- .gitignore | 2 + cmd/sippy/annotatejobruns.go | 11 +- cmd/sippy/automatejira.go | 8 +- cmd/sippy/component_readiness.go | 6 + cmd/sippy/load.go | 3 +- cmd/sippy/seed_data.go | 952 ++++++++++++------ cmd/sippy/serve.go | 88 +- config/e2e-openshift.yaml | 8 +- config/e2e-views.yaml | 150 ++- .../componentreadiness/component_report.go | 238 +---- .../component_report_test.go | 102 +- .../dataprovider/bigquery/provider.go | 412 ++++++++ .../dataprovider/interface.go | 90 ++ .../dataprovider/postgres/provider.go | 790 +++++++++++++++ .../middleware/interface.go | 6 +- .../middleware/linkinjector/linkinjector.go | 6 +- pkg/api/componentreadiness/middleware/list.go | 6 +- .../regressionallowances.go | 6 +- .../regressiontracker/regressiontracker.go | 6 +- .../releasefallback/releasefallback.go | 182 +--- .../releasefallback/releasefallback_test.go | 10 +- .../query/querygenerators.go | 44 +- .../componentreadiness/regressiontracker.go | 12 +- pkg/api/componentreadiness/test_details.go | 125 +-- pkg/api/componentreadiness/utils/utils.go | 4 +- .../api/componentreport/crstatus/types.go | 78 ++ .../api/componentreport/testdetails/types.go | 3 +- pkg/bigquery/bqlabel/labels.go | 1 + .../jiraautomator/jiraautomator.go | 6 +- .../jobrunannotator/jobrunannotator.go | 6 +- pkg/dataloader/crcacheloader/crcacheloader.go | 6 +- pkg/sippyserver/metrics/metrics.go | 32 +- pkg/sippyserver/server.go | 81 +- scripts/e2e.sh | 34 +- .../componentreadiness_test.go | 237 ++++- .../regressiontracker_test.go | 91 +- .../componentreadiness/report/report_test.go | 295 ++++++ .../triage/triageapi_test.go | 262 ++--- test/e2e/datasync/datasync_test.go | 54 + test/e2e/util/cache_manipulator.go | 238 ----- test/e2e/util/e2erequest.go | 4 +- 41 files changed, 3267 insertions(+), 1428 deletions(-) create mode 100644 pkg/api/componentreadiness/dataprovider/bigquery/provider.go create mode 100644 pkg/api/componentreadiness/dataprovider/interface.go create mode 100644 pkg/api/componentreadiness/dataprovider/postgres/provider.go create mode 100644 pkg/apis/api/componentreport/crstatus/types.go create mode 100644 test/e2e/componentreadiness/report/report_test.go create mode 100644 test/e2e/datasync/datasync_test.go delete mode 100644 test/e2e/util/cache_manipulator.go diff --git a/.gitignore b/.gitignore index 1f885da126..0f715cb799 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ report.sh .vscode/ .idea/ /sippy-ng/build/* +.env +*.log diff --git a/cmd/sippy/annotatejobruns.go b/cmd/sippy/annotatejobruns.go index f9375e1da9..b1248d6231 100644 --- a/cmd/sippy/annotatejobruns.go +++ b/cmd/sippy/annotatejobruns.go @@ -8,7 +8,8 @@ import ( "time" "github.com/openshift/sippy/pkg/api/componentreadiness" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/cache" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" @@ -32,7 +33,7 @@ type AnnotateJobRunsFlags struct { ComponentReadinessFlags *flags.ComponentReadinessFlags ConfigFlags *configflags.ConfigFlags VariantStr []string - Variants []bq.Variant + Variants []crstatus.Variant Release string Label string BuildClusters []string @@ -104,7 +105,7 @@ func (f *AnnotateJobRunsFlags) Validate(allVariants crtest.JobVariants) error { if !found { return fmt.Errorf("--variant %s has wrong variant value %s", variantStr, vt[1]) } - f.Variants = append(f.Variants, bq.Variant{Key: vt[0], Value: vt[1]}) + f.Variants = append(f.Variants, crstatus.Variant{Key: vt[0], Value: vt[1]}) } if len(f.Label) == 0 { return fmt.Errorf("--label is required") @@ -179,9 +180,9 @@ Example run: sippy annotate-job-runs --google-service-account-credential-file=f return errors.WithMessage(err, "couldn't get DB client") } - allVariants, errs := componentreadiness.GetJobVariantsFromBigQuery(ctx, bigQueryClient) + allVariants, errs := componentreadiness.GetJobVariants(ctx, bqprovider.NewBigQueryProvider(bigQueryClient)) if len(errs) > 0 { - return fmt.Errorf("failed to get variants from bigquery") + return fmt.Errorf("failed to get job variants") } if err = f.Validate(allVariants); err != nil { return errors.WithMessage(err, "error validating options") diff --git a/cmd/sippy/automatejira.go b/cmd/sippy/automatejira.go index 8ce8684219..bb69d2d1ad 100644 --- a/cmd/sippy/automatejira.go +++ b/cmd/sippy/automatejira.go @@ -15,6 +15,7 @@ import ( "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness" + bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/cache" jiratype "github.com/openshift/sippy/pkg/apis/jira/v1" @@ -169,9 +170,10 @@ func NewAutomateJiraCommand() *cobra.Command { log.WithError(err).Warn("error reading config file") } - allVariants, errs := componentreadiness.GetJobVariantsFromBigQuery(ctx, bigQueryClient) + provider := bqprovider.NewBigQueryProvider(bigQueryClient) + allVariants, errs := componentreadiness.GetJobVariants(ctx, provider) if len(errs) > 0 { - return fmt.Errorf("failed to get variants from bigquery") + return fmt.Errorf("failed to get job variants") } variantToJiraComponents, err := jiraautomator.GetVariantJiraMap(ctx, bigQueryClient) if err != nil { @@ -186,7 +188,7 @@ func NewAutomateJiraCommand() *cobra.Command { log.WithError(err).Fatal("unable to connect to postgres") } j, err := jiraautomator.NewJiraAutomator( - jiraClient, bigQueryClient, dbc, cacheOpts, + jiraClient, bigQueryClient, provider, dbc, cacheOpts, views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount, f.IncludeComponents, f.ColumnThresholds, f.DryRun, variantToJiraComponents, diff --git a/cmd/sippy/component_readiness.go b/cmd/sippy/component_readiness.go index e9154a7645..69f60ec70a 100644 --- a/cmd/sippy/component_readiness.go +++ b/cmd/sippy/component_readiness.go @@ -17,6 +17,7 @@ import ( "gopkg.in/yaml.v3" resources "github.com/openshift/sippy" + bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/config/v1" "github.com/openshift/sippy/pkg/bigquery" @@ -198,6 +199,7 @@ func (f *ComponentReadinessFlags) runServerMode() error { gcsClient, f.GoogleCloudFlags.StorageBucket, bigQueryClient, + bqprovider.NewBigQueryProvider(bigQueryClient), nil, cacheClient, f.ComponentReadinessFlags.CRTimeRoundingFactor, @@ -208,12 +210,15 @@ func (f *ComponentReadinessFlags) runServerMode() error { jiraClient, ) + crDataProvider := bqprovider.NewBigQueryProvider(bigQueryClient) + if f.APIFlags.MetricsAddr != "" { // Do an immediate metrics update err = metrics.RefreshMetricsDB( context.Background(), dbc, bigQueryClient, + crDataProvider, time.Time{}, cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), views.ComponentReadiness, @@ -234,6 +239,7 @@ func (f *ComponentReadinessFlags) runServerMode() error { context.Background(), dbc, bigQueryClient, + crDataProvider, time.Time{}, cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), views.ComponentReadiness, diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 0cc759ec14..861efc07d9 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -11,6 +11,7 @@ import ( "cloud.google.com/go/bigquery" "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness" + bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" "github.com/openshift/sippy/pkg/apis/cache" sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/dataloader/crcacheloader" @@ -318,7 +319,7 @@ func NewLoadCommand() *cobra.Command { } regressionTracker := componentreadiness.NewRegressionTracker( - bqc, dbc, cacheOpts, releases, + bqprovider.NewBigQueryProvider(bqc), dbc, cacheOpts, releases, componentreadiness.NewPostgresRegressionStore(dbc, jiraClient), views.ComponentReadiness, config.ComponentReadinessConfig.VariantJunitTableOverrides, diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index 6b5e3a6329..e9533398b0 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -1,58 +1,49 @@ package main import ( + "context" "fmt" - "math/rand" + "os" + "sort" + "strings" "time" + "github.com/lib/pq" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" - + "gopkg.in/yaml.v3" + + componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" + pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" + apitype "github.com/openshift/sippy/pkg/apis/api" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/sippyprocessing/v1" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/db/models/jobrunscan" "github.com/openshift/sippy/pkg/flags" "github.com/openshift/sippy/pkg/sippyserver" + "github.com/openshift/sippy/pkg/util/sets" ) type SeedDataFlags struct { - DBFlags *flags.PostgresFlags - InitDatabase bool - Releases []string - JobsPerRelease int - TestNames []string - RunsPerJob int + DBFlags *flags.PostgresFlags + InitDatabase bool } func NewSeedDataFlags() *SeedDataFlags { return &SeedDataFlags{ - DBFlags: flags.NewPostgresDatabaseFlags(), - Releases: []string{"5.0", "4.22", "4.21"}, // Default releases - JobsPerRelease: 3, // Default jobs per release - TestNames: []string{ - "install should succeed: infrastructure", - "install should succeed: overall", - "install should succeed: configuration", - "install should succeed: cluster bootstrap", - "install should succeed: other", - "[sig-cluster-lifecycle] Cluster completes upgrade", - "[sig-sippy] upgrade should work", - "[sig-sippy] openshift-tests should work", - }, - RunsPerJob: 20, // Default runs per job + DBFlags: flags.NewPostgresDatabaseFlags(), } } func (f *SeedDataFlags) BindFlags(fs *pflag.FlagSet) { f.DBFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Initialize the DB schema before seeding data") - fs.StringSliceVar(&f.Releases, "release", f.Releases, "Releases to create ProwJobs for (can be specified multiple times)") - fs.IntVar(&f.JobsPerRelease, "jobs", f.JobsPerRelease, "Number of ProwJobs to create for each release") - fs.StringSliceVar(&f.TestNames, "test", f.TestNames, "Test names to create (can be specified multiple times)") - fs.IntVar(&f.RunsPerJob, "runs", f.RunsPerJob, "Number of ProwJobRuns to create for each ProwJob") } func NewSeedDataCommand() *cobra.Command { @@ -62,16 +53,19 @@ func NewSeedDataCommand() *cobra.Command { Use: "seed-data", Short: "Populate test data in the database", Long: `Populate test data in the database for development purposes. -This command creates sample ProwJob and Test records with realistic test data -that can be used for local development and testing. -Test results are randomized with 85% pass rate, 10% flake rate, and 5% failure rate. -All counts, releases, and test names are configurable via command-line flags. +Creates deterministic Component Readiness data covering all CR statuses +(NotSignificant, SignificantRegression, ExtremeRegression, MissingSample, +MissingBasis, BasisOnly, SignificantImprovement, BelowMinFailure) and +fallback scenarios. Use with 'sippy serve --data-provider postgres'. -The command can be re-run as needed to add more runs, or because your old job runs -rolled off the 1 week window. +The command can be re-run to refresh data. `, RunE: func(cmd *cobra.Command, args []string) error { + if strings.Contains(f.DBFlags.DSN, "amazonaws.com") { + return fmt.Errorf("refusing to seed synthetic data into a production database") + } + dbc, err := f.DBFlags.GetDBClient() if err != nil { return errors.WithMessage(err, "could not connect to database") @@ -87,247 +81,703 @@ rolled off the 1 week window. } log.Info("Starting to seed test data...") + return seedSyntheticData(dbc) + }, + } - // Create the test suite - if err := createTestSuite(dbc); err != nil { - return errors.WithMessage(err, "failed to create test suite") - } - log.Info("Created test suite 'ourtests'") - - // Create ProwJobs for each release - for _, release := range f.Releases { - if err := createProwJobsForRelease(dbc, release, f.JobsPerRelease); err != nil { - return errors.WithMessagef(err, "failed to create ProwJobs for release %s", release) - } - log.Infof("Processed %d ProwJobs for release %s", f.JobsPerRelease, release) - } + f.BindFlags(cmd.Flags()) - // Create Test models - if err := createTestModels(dbc, f.TestNames); err != nil { - return errors.WithMessage(err, "failed to create Test models") - } - log.Infof("Processed %d Test models", len(f.TestNames)) + return cmd +} - // Create labels and symptoms - if err := createLabelsAndSymptoms(dbc); err != nil { - return errors.WithMessage(err, "failed to create labels and symptoms") - } - log.Info("Created sample labels and symptoms") +// --- Synthetic data seeding --- - // Create ProwJobRuns for each ProwJob - if err := createProwJobRuns(dbc, f.RunsPerJob); err != nil { - return errors.WithMessage(err, "failed to create ProwJobRuns") - } - log.Info("Created ProwJobRuns and test results for all ProwJobs") +// syntheticJobDef defines a job with its full 9-key variant map. +type syntheticJobDef struct { + nameTemplate string + variants map[string]string +} - // Apply labels to job runs - if err := applyLabelsToJobRuns(dbc); err != nil { - return errors.WithMessage(err, "failed to apply labels to job runs") - } - log.Info("Applied labels to ~25% of job runs") +// syntheticTestSpec defines a test with deterministic pass/fail counts per release per job. +type syntheticTestSpec struct { + testID string + testName string + component string + capabilities []string + // Each entry maps a job name template -> per-release counts. + // The job template determines which variants the test runs with. + jobCounts map[string]map[string]testCount // jobTemplate -> release -> counts +} - totalProwJobs := len(f.Releases) * f.JobsPerRelease - totalRuns := totalProwJobs * f.RunsPerJob - totalTestResults := totalRuns * len(f.TestNames) +type testCount struct { + total int + success int + flake int +} - log.Info("Refreshing materialized views...") - sippyserver.RefreshData(dbc, nil, false) +var syntheticReleases = []string{"4.22", "4.21", "4.20", "4.19"} - log.Infof("Successfully seeded test data! Created %d ProwJobs, %d Tests, %d ProwJobRuns, and %d test results", - totalProwJobs, len(f.TestNames), totalRuns, totalTestResults) - return nil +var syntheticJobs = []syntheticJobDef{ + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-upgrade-from-stable-4.21-e2e-aws-ovn-upgrade", + variants: map[string]string{ + "Platform": "aws", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "unknown", "Upgrade": "minor", "LayeredProduct": "none", }, - } + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-amd64", + variants: map[string]string{ + "Platform": "aws", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "parallel", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-arm64", + variants: map[string]string{ + "Platform": "aws", "Architecture": "arm64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "parallel", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-techpreview-serial", + variants: map[string]string{ + "Platform": "aws", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "techpreview", + "Suite": "serial", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-gcp-ovn-amd64", + variants: map[string]string{ + "Platform": "gcp", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "parallel", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-gcp-ovn-upgrade-micro", + variants: map[string]string{ + "Platform": "gcp", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "unknown", "Upgrade": "micro", "LayeredProduct": "none", + }, + }, +} - f.BindFlags(cmd.Flags()) +// Job template constants for referencing specific jobs in test specs. +const awsAmd64Parallel = "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-amd64" +const awsArm64Parallel = "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-arm64" +const gcpAmd64Parallel = "periodic-ci-openshift-release-master-ci-%s-e2e-gcp-ovn-amd64" + +// allJobTemplates returns name templates from syntheticJobs for use in test specs +// that should run on every job (e.g. install tests). +func allJobTemplates() []string { + templates := make([]string, len(syntheticJobs)) + for i, j := range syntheticJobs { + templates[i] = j.nameTemplate + } + return templates +} - return cmd +// allJobCounts builds a jobCounts map that assigns the given per-release counts +// to every synthetic job. Used for tests like install indicators that run everywhere. +func allJobCounts(releaseCounts map[string]testCount) map[string]map[string]testCount { + result := make(map[string]map[string]testCount, len(syntheticJobs)) + for _, tpl := range allJobTemplates() { + result[tpl] = releaseCounts + } + return result } -func createProwJobsForRelease(dbc *db.DB, release string, jobsPerRelease int) error { - for i := 1; i <= jobsPerRelease; i++ { - // Choose JobTier based on whether i is even or odd - var jobTier = "JobTier:standard" // even number job index = standard - if i%2 != 0 { - jobTier = "JobTier:hidden" // odd = hidden - } +var syntheticTests = []syntheticTestSpec{ + // --- NotSignificant: appears in 3 jobs across 2 platforms --- + { + testID: "test-not-significant", testName: "[sig-arch] Check build pods use all cpu cores", + component: "comp-NotSignificant", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 95, 0}, "4.22": {100, 93, 0}}, + awsArm64Parallel: {"4.21": {80, 76, 0}, "4.22": {80, 75, 0}}, + gcpAmd64Parallel: {"4.21": {100, 97, 0}, "4.22": {100, 95, 0}}, + }, + }, + + // --- SignificantRegression: regressed on aws/amd64, fine elsewhere --- + { + testID: "test-significant-regression", testName: "[sig-network] Services should serve endpoints on same port and different protocol", + component: "comp-SignificantRegression", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 170, 0}}, + awsArm64Parallel: {"4.21": {180, 171, 0}, "4.22": {180, 168, 0}}, + gcpAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 188, 0}}, + }, + }, + + // --- ExtremeRegression: extreme on aws/amd64, significant on others --- + { + testID: "test-extreme-regression", testName: "[sig-etcd] etcd leader changes are not excessive", + component: "comp-ExtremeRegression", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 140, 0}}, + awsArm64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 170, 0}}, + gcpAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 170, 0}}, + }, + }, + + // --- MissingSample: test in base, 0 sample runs --- + { + testID: "test-missing-sample", testName: "[sig-storage] CSI volumes should be mountable", + component: "comp-MissingSample", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 95, 0}, "4.22": {0, 0, 0}}, + }, + }, + + // --- MissingBasis: test only in sample --- + { + testID: "test-missing-basis", testName: "[sig-node] New pod lifecycle test", + component: "comp-MissingBasis", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.22": {100, 95, 0}}, + }, + }, + + // --- BasisOnly: test in base, absent from sample --- + { + testID: "test-basis-only", testName: "[sig-apps] Removed deployment test", + component: "comp-BasisOnly", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 95, 0}}, + }, + }, + + // --- SignificantImprovement: 80% -> 95% --- + { + testID: "test-significant-improvement", testName: "[sig-cli] oc adm should handle upgrades gracefully", + component: "comp-SignificantImprovement", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {200, 160, 0}, "4.22": {200, 190, 0}}, + }, + }, + + // --- BelowMinFailure: only 2 failures, below MinimumFailure=3 --- + { + testID: "test-below-min-failure", testName: "[sig-auth] RBAC should allow access with valid token", + component: "comp-BelowMinFailure", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 100, 0}, "4.22": {100, 98, 0}}, + }, + }, + + // --- Fallback: 4.21 worse, 4.20 better -> swaps to 4.20 --- + { + testID: "test-fallback-improves", testName: "[sig-instrumentation] Metrics should report accurate cpu usage", + component: "comp-FallbackImproves", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: { + "4.21": {200, 180, 0}, + "4.20": {200, 194, 0}, + "4.22": {200, 160, 0}, + }, + }, + }, + + // --- Double fallback: 4.21->4.20->4.19 --- + { + testID: "test-fallback-double", testName: "[sig-scheduling] Scheduler should spread pods evenly", + component: "comp-FallbackDouble", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: { + "4.21": {200, 180, 0}, + "4.20": {200, 186, 0}, + "4.19": {200, 194, 0}, + "4.22": {200, 160, 0}, + }, + }, + }, + + // --- Fallback insufficient runs: 4.20 has <60% of 4.21 count --- + { + testID: "test-fallback-insufficient-runs", testName: "[sig-network] DNS should resolve cluster services", + component: "comp-FallbackInsufficient", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: { + "4.21": {1000, 940, 0}, + "4.20": {100, 99, 0}, + "4.22": {1000, 850, 0}, + }, + }, + }, + + // --- Install / health indicator tests: run on every job, every release --- + { + testID: "test-install-overall", testName: "install should succeed: overall", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 95, 0}, "4.21": {100, 96, 0}, "4.20": {100, 97, 0}, "4.19": {100, 97, 0}, + }), + }, + { + testID: "test-install-config", testName: "install should succeed: configuration", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 97, 0}, "4.21": {100, 98, 0}, "4.20": {100, 98, 0}, "4.19": {100, 98, 0}, + }), + }, + { + testID: "test-install-bootstrap", testName: "install should succeed: cluster bootstrap", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 96, 0}, "4.21": {100, 97, 0}, "4.20": {100, 97, 0}, "4.19": {100, 97, 0}, + }), + }, + { + testID: "test-install-other", testName: "install should succeed: other", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 98, 0}, "4.21": {100, 99, 0}, "4.20": {100, 99, 0}, "4.19": {100, 99, 0}, + }), + }, + { + testID: "test-install-infra", testName: "install should succeed: infrastructure", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 96, 0}, "4.21": {100, 97, 0}, "4.20": {100, 97, 0}, "4.19": {100, 97, 0}, + }), + }, + { + testID: "test-upgrade", testName: "[sig-sippy] upgrade should work", + component: "comp-Install", capabilities: []string{"upgrade"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 94, 0}, "4.21": {100, 95, 0}, "4.20": {100, 96, 0}, "4.19": {100, 96, 0}, + }), + }, + { + testID: "test-openshift-tests", testName: "[sig-sippy] openshift-tests should work", + component: "comp-Install", capabilities: []string{"tests"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 90, 0}, "4.21": {100, 92, 0}, "4.20": {100, 93, 0}, "4.19": {100, 93, 0}, + }), + }, +} - prowJob := models.ProwJob{ - Kind: models.ProwKind("periodic"), - Name: fmt.Sprintf("sippy-test-job-%s-test-%d", release, i), - Release: release, - // TestGridURL, Bugs, and JobRuns are left empty as requested - Variants: []string{"Platform:aws", "Upgrade:none", jobTier}, - } +// releaseTimeWindow returns the start/end times for a release's test data. +func releaseTimeWindow(release string) (start, end time.Time) { + now := time.Now().UTC().Truncate(time.Hour) + switch release { + case "4.22": + return now.Add(-3 * 24 * time.Hour), now + case "4.21": + return now.Add(-60 * 24 * time.Hour), now.Add(-30 * 24 * time.Hour) + case "4.20": + return now.Add(-120 * 24 * time.Hour), now.Add(-90 * 24 * time.Hour) + case "4.19": + return now.Add(-180 * 24 * time.Hour), now.Add(-150 * 24 * time.Hour) + default: + return now.Add(-14 * 24 * time.Hour), now + } +} - // Use FirstOrCreate to avoid duplicates - only creates if a ProwJob with this name doesn't exist - var existingJob models.ProwJob - if err := dbc.DB.Where("name = ?", prowJob.Name).FirstOrCreate(&existingJob, prowJob).Error; err != nil { - return fmt.Errorf("failed to create or find ProwJob %s: %v", prowJob.Name, err) - } +func seedSyntheticData(dbc *db.DB) error { + // Check if data already exists + var count int64 + if err := dbc.DB.Model(&models.ProwJob{}).Count(&count).Error; err != nil { + return fmt.Errorf("failed to check for existing data: %w", err) + } + if count > 0 { + log.Infof("Database already contains %d ProwJobs, skipping seed. Use --init-database to reset.", count) + return nil + } - // Log whether we created a new job or found an existing one - if existingJob.CreatedAt.IsZero() || existingJob.CreatedAt.Equal(existingJob.UpdatedAt) { - log.Debugf("Created new ProwJob: %s", prowJob.Name) - } else { - log.Debugf("ProwJob already exists: %s", prowJob.Name) + // 1. Create test suite + if err := createTestSuite(dbc, "synthetic"); err != nil { + return errors.WithMessage(err, "failed to create test suite") + } + log.Info("Created test suite 'synthetic'") + + // 2. Create ProwJobs for all releases + for _, release := range syntheticReleases { + for _, job := range syntheticJobs { + name := fmt.Sprintf(job.nameTemplate, release) + variants := variantMapToArray(job.variants) + prowJob := models.ProwJob{ + Kind: models.ProwKind("periodic"), + Name: name, + Release: release, + Variants: variants, + } + var existing models.ProwJob + if err := dbc.DB.Where("name = ?", name).FirstOrCreate(&existing, prowJob).Error; err != nil { + return fmt.Errorf("failed to create ProwJob %s: %w", name, err) + } } } + log.Infof("Created ProwJobs for %d releases x %d jobs", len(syntheticReleases), len(syntheticJobs)) - return nil -} + // 3. Create Tests and TestOwnerships + var suite models.Suite + if err := dbc.DB.Where("name = ?", "synthetic").First(&suite).Error; err != nil { + return fmt.Errorf("failed to find suite: %w", err) + } -func createTestModels(dbc *db.DB, testNames []string) error { - for _, testName := range testNames { - testModel := models.Test{ - Name: testName, + // Collect unique tests + type testInfo struct { + name string + uniqueID string + component string + capabilities []string + } + seenTests := map[string]testInfo{} + for _, ts := range syntheticTests { + if _, ok := seenTests[ts.testName]; !ok { + seenTests[ts.testName] = testInfo{ + name: ts.testName, + uniqueID: ts.testID, + component: ts.component, + capabilities: ts.capabilities, + } } + } - // Use FirstOrCreate to avoid duplicates - only creates if a Test with this name doesn't exist + for _, info := range seenTests { + // Create test + testModel := models.Test{Name: info.name} var existingTest models.Test - if err := dbc.DB.Where("name = ?", testModel.Name).FirstOrCreate(&existingTest, testModel).Error; err != nil { - return fmt.Errorf("failed to create or find Test %s: %v", testModel.Name, err) + if err := dbc.DB.Where("name = ?", info.name).FirstOrCreate(&existingTest, testModel).Error; err != nil { + return fmt.Errorf("failed to create Test %s: %w", info.name, err) } - if existingTest.CreatedAt.IsZero() || existingTest.CreatedAt.Equal(existingTest.UpdatedAt) { - log.Debugf("Created new Test: %s", testModel.Name) - } else { - log.Debugf("Test already exists: %s", testModel.Name) + // Create test ownership + ownership := models.TestOwnership{ + UniqueID: info.uniqueID, + Name: info.name, + TestID: existingTest.ID, + Suite: "synthetic", + SuiteID: &suite.ID, + Component: info.component, + Capabilities: info.capabilities, + } + var existingOwnership models.TestOwnership + if err := dbc.DB.Where("name = ? AND suite = ?", info.name, "synthetic").FirstOrCreate(&existingOwnership, ownership).Error; err != nil { + return fmt.Errorf("failed to create TestOwnership for %s: %w", info.name, err) } } - - return nil -} - -func createTestSuite(dbc *db.DB) error { - suite := models.Suite{ - Name: "ourtests", - } - - // Use FirstOrCreate to avoid duplicates - var existingSuite models.Suite - if err := dbc.DB.Where("name = ?", suite.Name).FirstOrCreate(&existingSuite, suite).Error; err != nil { - return fmt.Errorf("failed to create or find Suite %s: %v", suite.Name, err) + log.Infof("Created %d tests with ownership records", len(seenTests)) + + // 4. Create deterministic ProwJobRuns and test results + // + // Strategy: for each (job template, release), determine the max run count needed + // across all tests assigned to that job+release. Create that many shared runs. + // Then assign test results per test with exact counts. + type jobReleaseKey struct { + jobTemplate string + release string } - return nil -} - -func createProwJobRuns(dbc *db.DB, runsPerJob int) error { - var prowJobs []models.ProwJob - if err := dbc.DB.Find(&prowJobs).Error; err != nil { - return fmt.Errorf("failed to fetch existing ProwJobs: %v", err) + // Compute max runs needed per job+release + maxRuns := map[jobReleaseKey]int{} + for _, ts := range syntheticTests { + for jobTpl, releaseCounts := range ts.jobCounts { + for release, counts := range releaseCounts { + key := jobReleaseKey{jobTpl, release} + if counts.total > maxRuns[key] { + maxRuns[key] = counts.total + } + } + } } - var tests []models.Test - if err := dbc.DB.Find(&tests).Error; err != nil { - return fmt.Errorf("failed to fetch existing Tests: %v", err) + // Load all tests by name for ID lookup + testIDsByName := map[string]uint{} + var allTests []models.Test + if err := dbc.DB.Find(&allTests).Error; err != nil { + return fmt.Errorf("failed to fetch tests: %w", err) } - - var suite models.Suite - if err := dbc.DB.Where("name = ?", "ourtests").First(&suite).Error; err != nil { - return fmt.Errorf("failed to find Suite 'ourtests': %v", err) + for _, t := range allTests { + testIDsByName[t.Name] = t.ID } - log.Infof("Found %d ProwJobs, creating %d runs for each", len(prowJobs), runsPerJob) - - // Calculate time range: past 2 weeks from now - now := time.Now() - twoWeeksAgo := now.AddDate(0, 0, -14) - - // Duration for each run: 3 hours - runDuration := 3 * time.Hour - - for _, prowJob := range prowJobs { - log.Infof("Creating %d ProwJobRuns for ProwJob: %s", runsPerJob, prowJob.Name) + // Create runs and test results + totalRuns := 0 + totalResults := 0 + for jrKey, runCount := range maxRuns { + if runCount == 0 { + continue + } - for i := 0; i < runsPerJob; i++ { - // Log progress every 10 runs to show activity - if (i+1)%10 == 0 { - log.Infof(" Progress: %d/%d runs created for %s", i+1, runsPerJob, prowJob.Name) - } + // Look up the ProwJob + jobName := fmt.Sprintf(jrKey.jobTemplate, jrKey.release) + var prowJob models.ProwJob + if err := dbc.DB.Where("name = ?", jobName).First(&prowJob).Error; err != nil { + return fmt.Errorf("failed to find ProwJob %s: %w", jobName, err) + } - // Calculate timestamp: spread evenly over the past 2 weeks - totalDuration := 14 * 24 * time.Hour - // Time between runs = total duration / runs - timeBetweenRuns := totalDuration / time.Duration(runsPerJob) - timestamp := twoWeeksAgo.Add(time.Duration(i) * timeBetweenRuns) + // Create runs spread across the release time window. + // Reserve last 2 runs as infra failures (no test results). + start, end := releaseTimeWindow(jrKey.release) + window := end.Sub(start) + interval := window / time.Duration(runCount) - prowJobRun := models.ProwJobRun{ + runIDs := make([]uint, runCount) + for i := 0; i < runCount; i++ { + timestamp := start.Add(time.Duration(i) * interval) + run := models.ProwJobRun{ ProwJobID: prowJob.ID, Cluster: "build01", Timestamp: timestamp, - Duration: runDuration, - TestCount: len(tests), + Duration: 3 * time.Hour, + } + if err := dbc.DB.Create(&run).Error; err != nil { + return fmt.Errorf("failed to create ProwJobRun: %w", err) + } + runIDs[i] = run.ID + totalRuns++ + } + + // Runs that get test results (all except the last 2) + testableRuns := runCount + if testableRuns > 2 { + testableRuns = runCount - 2 + } + + // Track which runs have at least one test failure + runsWithFailure := map[uint]bool{} + + // Assign test results to testable runs only + for _, ts := range syntheticTests { + releaseCounts, hasJob := ts.jobCounts[jrKey.jobTemplate] + if !hasJob { + continue + } + counts, hasRelease := releaseCounts[jrKey.release] + if !hasRelease || counts.total == 0 { + continue } - if err := dbc.DB.Create(&prowJobRun).Error; err != nil { - return fmt.Errorf("failed to create ProwJobRun for ProwJob %s: %v", prowJob.Name, err) + testID, ok := testIDsByName[ts.testName] + if !ok { + return fmt.Errorf("test %q not found in DB", ts.testName) } - var testFailures int - for _, test := range tests { - // Determine test status based on random chance - // 5% chance of failure, 10% chance of flake, 85% chance of pass - // nolint: gosec - randNum := rand.Float64() + for i := 0; i < counts.total && i < testableRuns; i++ { var status int - if randNum < 0.05 { - status = 12 // failure - testFailures++ - } else if randNum < 0.15 { - status = 13 // flake - } else { + switch { + case i < counts.success-counts.flake: status = 1 // pass + case i < counts.success: + status = 13 // flake (counts as success too) + default: + status = 12 // failure + runsWithFailure[runIDs[i]] = true } - prowJobRunTest := models.ProwJobRunTest{ - ProwJobRunID: prowJobRun.ID, - TestID: test.ID, + result := models.ProwJobRunTest{ + ProwJobRunID: runIDs[i], + TestID: testID, SuiteID: &suite.ID, Status: status, - Duration: 5.0, // 5 seconds - CreatedAt: timestamp, + Duration: 5.0, + CreatedAt: start.Add(time.Duration(i) * interval), } - - if err := dbc.DB.Create(&prowJobRunTest).Error; err != nil { - return fmt.Errorf("failed to create ProwJobRunTest for test %s: %v", test.Name, err) + if err := dbc.DB.Create(&result).Error; err != nil { + return fmt.Errorf("failed to create ProwJobRunTest: %w", err) } + totalResults++ } + } - // Set overall result based on test failures and random factors + // Set OverallResult on all runs + for i, runID := range runIDs { var overallResult v1.JobOverallResult - if testFailures > 0 { - prowJobRun.Failed = true - prowJobRun.Succeeded = false - prowJobRun.TestFailures = testFailures - - // Randomly assign different failure types - // nolint: gosec - failureType := rand.Float64() - if failureType < 0.7 { - overallResult = v1.JobTestFailure // 70% test failures - } else if failureType < 0.85 { - overallResult = v1.JobUpgradeFailure // 15% upgrade failures - } else if failureType < 0.92 { - overallResult = v1.JobInstallFailure // 7% install failures - } else { - overallResult = v1.JobExternalInfrastructureFailure // 8% infrastructure failures - } + var succeeded, failed bool + + if i >= testableRuns { + // Last 2 runs: infra failure, no tests ran + overallResult = v1.JobInternalInfrastructureFailure + failed = true + } else if runsWithFailure[runID] { + overallResult = v1.JobTestFailure + failed = true } else { - prowJobRun.Failed = false - prowJobRun.Succeeded = true - prowJobRun.TestFailures = 0 overallResult = v1.JobSucceeded + succeeded = true } - prowJobRun.OverallResult = overallResult - if err := dbc.DB.Save(&prowJobRun).Error; err != nil { - return fmt.Errorf("failed to update ProwJobRun for ProwJob %s: %v", prowJob.Name, err) + if err := dbc.DB.Model(&models.ProwJobRun{}).Where("id = ?", runID). + Updates(map[string]interface{}{ + "overall_result": overallResult, + "succeeded": succeeded, + "failed": failed, + }).Error; err != nil { + return fmt.Errorf("failed to update ProwJobRun result: %w", err) } } - log.Infof("Completed creating %d ProwJobRuns for ProwJob: %s", runsPerJob, prowJob.Name) + // Update test_failures count + if err := dbc.DB.Exec(` + UPDATE prow_job_runs SET test_failures = COALESCE(( + SELECT COUNT(*) FROM prow_job_run_tests + WHERE prow_job_run_id = prow_job_runs.id AND status = 12 + ), 0) WHERE prow_job_id = ?`, prowJob.ID).Error; err != nil { + log.WithError(err).Warn("failed to update test_failures count") + } + + log.Debugf("Created %d runs for %s", runCount, jobName) + } + + // Create labels and symptoms + if err := createLabelsAndSymptoms(dbc); err != nil { + return errors.WithMessage(err, "failed to create labels and symptoms") + } + + // Generate views file with include_variants matching our seed data + if err := writeSyntheticViewsFile(); err != nil { + return errors.WithMessage(err, "failed to write views file") + } + + log.Info("Refreshing materialized views...") + sippyserver.RefreshData(dbc, nil, false) + + // Run regression tracker to populate test_regressions table + log.Info("Syncing regressions...") + if err := syncRegressions(dbc); err != nil { + log.WithError(err).Warn("failed to sync regressions") + } + + log.Infof("Seeded synthetic data: %d ProwJobRuns, %d test results across %d releases", + totalRuns, totalResults, len(syntheticReleases)) + return nil +} + +func syncRegressions(dbc *db.DB) error { + provider := pgprovider.NewPostgresProvider(dbc, nil) + ctx := context.Background() + + releases, err := provider.QueryReleases(ctx) + if err != nil { + return fmt.Errorf("querying releases: %w", err) + } + + viewsData, err := os.ReadFile(syntheticViewsFile) + if err != nil { + return fmt.Errorf("reading views file: %w", err) + } + var views apitype.SippyViews + if err := yaml.Unmarshal(viewsData, &views); err != nil { + return fmt.Errorf("parsing views file: %w", err) + } + + tracker := componentreadiness.NewRegressionTracker( + provider, dbc, + cache.RequestOptions{}, + releases, + componentreadiness.NewPostgresRegressionStore(dbc, nil), + views.ComponentReadiness, + nil, + false, + ) + tracker.Load() + if len(tracker.Errors()) > 0 { + for _, err := range tracker.Errors() { + log.WithError(err).Warn("regression tracker error") + } + } + return nil +} + +const syntheticViewsFile = "config/e2e-views.yaml" + +// writeSyntheticViewsFile generates a views file with include_variants matching the seed data. +func writeSyntheticViewsFile() error { + // Collect all unique variant values from synthetic jobs + allVariants := map[string]map[string]bool{} + for _, job := range syntheticJobs { + for k, v := range job.variants { + if allVariants[k] == nil { + allVariants[k] = map[string]bool{} + } + allVariants[k][v] = true + } + } + + includeVariants := map[string][]string{} + for k, vals := range allVariants { + sorted := make([]string, 0, len(vals)) + for v := range vals { + sorted = append(sorted, v) + } + sort.Strings(sorted) + includeVariants[k] = sorted + } + + dbGroupBy := sets.NewString("Architecture", "FeatureSet", "Installer", "Network", "Platform", + "Suite", "Topology", "Upgrade", "LayeredProduct") + columnGroupBy := sets.NewString("Network", "Platform", "Topology") + + views := apitype.SippyViews{ + ComponentReadiness: []crview.View{ + { + Name: "4.22-main", + BaseRelease: reqopts.RelativeRelease{ + Release: reqopts.Release{Name: "4.21"}, + RelativeStart: "now-60d", + RelativeEnd: "now-30d", + }, + SampleRelease: reqopts.RelativeRelease{ + Release: reqopts.Release{Name: "4.22"}, + RelativeStart: "now-3d", + RelativeEnd: "now", + }, + VariantOptions: reqopts.Variants{ + ColumnGroupBy: columnGroupBy, + DBGroupBy: dbGroupBy, + IncludeVariants: includeVariants, + }, + AdvancedOptions: reqopts.Advanced{ + Confidence: 95, + PityFactor: 5, + MinimumFailure: 3, + IncludeMultiReleaseAnalysis: true, + }, + PrimeCache: crview.PrimeCache{Enabled: true}, + RegressionTracking: crview.RegressionTracking{Enabled: true}, + }, + }, + } + + data, err := yaml.Marshal(views) + if err != nil { + return fmt.Errorf("marshaling views: %w", err) + } + + if err := os.WriteFile(syntheticViewsFile, data, 0644); err != nil { + return fmt.Errorf("writing %s: %w", syntheticViewsFile, err) + } + + log.Infof("Generated views file: %s", syntheticViewsFile) + return nil +} + +// variantMapToArray converts a variant map to a pq.StringArray. +func variantMapToArray(m map[string]string) pq.StringArray { + result := make([]string, 0, len(m)) + for k, v := range m { + result = append(result, k+":"+v) + } + return result +} + +func createTestSuite(dbc *db.DB, name string) error { + suite := models.Suite{ + Name: name, + } + + var existingSuite models.Suite + if err := dbc.DB.Where("name = ?", suite.Name).FirstOrCreate(&existingSuite, suite).Error; err != nil { + return fmt.Errorf("failed to create or find Suite %s: %v", suite.Name, err) } return nil @@ -339,13 +789,12 @@ func createLabelsAndSymptoms(dbc *db.DB) error { UpdatedBy: "seed-data", } - // Create sample labels labels := []jobrunscan.Label{ { LabelContent: jobrunscan.LabelContent{ ID: "InfraFailure", LabelTitle: "Infrastructure failure: omit job from CR", - Explanation: "Job failed due to **infrastructure issues** not related to product code. See [TRT documentation](https://docs.ci.openshift.org/docs/architecture/ci-operator/) for more details.", + Explanation: "Job failed due to **infrastructure issues** not related to product code.", }, Metadata: metadata, }, @@ -353,7 +802,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "ClusterDNSFlake", LabelTitle: "Cluster DNS resolution failure(s)", - Explanation: "Job experienced DNS resolution timeouts in the cluster:\n\n- Check for network issues\n- Review DNS server logs\n- Examine cluster network configuration", + Explanation: "Job experienced DNS resolution timeouts in the cluster.", }, Metadata: metadata, }, @@ -361,7 +810,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "ClusterInstallTimeout", LabelTitle: "Cluster install timeout", - Explanation: "Cluster installation exceeded timeout threshold. This may indicate:\n\n1. Slow infrastructure provisioning\n2. Network connectivity problems\n3. Image pull failures", + Explanation: "Cluster installation exceeded timeout threshold.", }, Metadata: metadata, }, @@ -369,7 +818,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "IntervalFile", LabelTitle: "Has interval file(s)", - Explanation: "Job produced interval monitoring files. Use the `intervals` tool to analyze timing data.", + Explanation: "Job produced interval monitoring files.", }, HideDisplayContexts: []string{jobrunscan.MetricsContext, jobrunscan.JAQOptsContext}, Metadata: metadata, @@ -378,7 +827,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "APIServerTimeout", LabelTitle: "API server timeout", - Explanation: "Requests to the API server timed out. Common causes:\n\n- High API server load\n- Network latency issues\n- Slow etcd responses", + Explanation: "Requests to the API server timed out.", }, Metadata: metadata, }, @@ -389,14 +838,8 @@ func createLabelsAndSymptoms(dbc *db.DB) error { if err := dbc.DB.Where("id = ?", label.ID).FirstOrCreate(&existing, label).Error; err != nil { return fmt.Errorf("failed to create or find label %s: %v", label.ID, err) } - if existing.CreatedAt.IsZero() || existing.CreatedAt.Equal(existing.UpdatedAt) { - log.Debugf("Created new Label: %s", label.ID) - } else { - log.Debugf("Label already exists: %s", label.ID) - } } - // Create sample symptoms symptoms := []jobrunscan.Symptom{ { SymptomContent: jobrunscan.SymptomContent{ @@ -449,71 +892,8 @@ func createLabelsAndSymptoms(dbc *db.DB) error { if err := dbc.DB.Where("id = ?", symptom.ID).FirstOrCreate(&existing, symptom).Error; err != nil { return fmt.Errorf("failed to create or find symptom %s: %v", symptom.ID, err) } - if existing.CreatedAt.IsZero() || existing.CreatedAt.Equal(existing.UpdatedAt) { - log.Debugf("Created new Symptom: %s", symptom.ID) - } else { - log.Debugf("Symptom already exists: %s", symptom.ID) - } } return nil } -func applyLabelsToJobRuns(dbc *db.DB) error { - // Fetch all job runs - var jobRuns []models.ProwJobRun - if err := dbc.DB.Find(&jobRuns).Error; err != nil { - return fmt.Errorf("failed to fetch job runs: %v", err) - } - - // Fetch all labels - var labels []jobrunscan.Label - if err := dbc.DB.Find(&labels).Error; err != nil { - return fmt.Errorf("failed to fetch labels: %v", err) - } - - if len(labels) == 0 { - log.Warn("No labels found, skipping label application") - return nil - } - - labelIDs := make([]string, len(labels)) - for i, label := range labels { - labelIDs[i] = label.ID - } - - // Apply labels to approximately 25% of job runs - labeledCount := 0 - for i := range jobRuns { - // nolint: gosec // we do not care that the randomness is weak - if rand.Float64() > 0.25 { - continue - } - // Randomly select 1-3 labels - // nolint: gosec - numLabels := rand.Intn(3) + 1 - selectedLabels := make([]string, 0, numLabels) - - // Randomly pick unique labels - usedIndices := make(map[int]bool) - for len(selectedLabels) < numLabels && len(selectedLabels) < len(labelIDs) { - // nolint: gosec - idx := rand.Intn(len(labelIDs)) - if !usedIndices[idx] { - selectedLabels = append(selectedLabels, labelIDs[idx]) - usedIndices[idx] = true - } - } - - jobRuns[i].Labels = selectedLabels - if err := dbc.DB.Save(&jobRuns[i]).Error; err != nil { - return fmt.Errorf("failed to update job run %d with labels: %v", jobRuns[i].ID, err) - } - labeledCount++ - } - - log.Infof("Applied labels to %d of %d job runs (%.1f%%)", - labeledCount, len(jobRuns), float64(labeledCount)/float64(len(jobRuns))*100) - - return nil -} diff --git a/cmd/sippy/serve.go b/cmd/sippy/serve.go index baccd65dfc..adf102c79b 100644 --- a/cmd/sippy/serve.go +++ b/cmd/sippy/serve.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "io/fs" "net/http" "os" @@ -9,7 +10,6 @@ import ( "time" "cloud.google.com/go/storage" - "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" @@ -17,14 +17,19 @@ import ( "github.com/spf13/pflag" resources "github.com/openshift/sippy" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" + bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" + pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" "github.com/openshift/sippy/pkg/apis/cache" "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/flags" "github.com/openshift/sippy/pkg/flags/configflags" "github.com/openshift/sippy/pkg/sippyserver" "github.com/openshift/sippy/pkg/sippyserver/metrics" + "github.com/openshift/sippy/pkg/testidentification" "github.com/openshift/sippy/pkg/util" ) @@ -38,6 +43,7 @@ type ServerFlags struct { ConfigFlags *configflags.ConfigFlags APIFlags *flags.APIFlags JiraFlags *flags.JiraFlags + DataProvider string } func NewServerFlags() *ServerFlags { @@ -64,9 +70,13 @@ func (f *ServerFlags) BindFlags(flagSet *pflag.FlagSet) { f.ConfigFlags.BindFlags(flagSet) f.APIFlags.BindFlags(flagSet) f.JiraFlags.BindFlags(flagSet) + flagSet.StringVar(&f.DataProvider, "data-provider", "bigquery", "Data provider for component readiness: bigquery, postgres") } func (f *ServerFlags) Validate() error { + if f.DataProvider == "postgres" { + return nil + } return f.GoogleCloudFlags.Validate() } @@ -98,38 +108,51 @@ func NewServeCommand() *cobra.Command { var bigQueryClient *bigquery.Client var gcsClient *storage.Client - if f.GoogleCloudFlags.ServiceAccountCredentialFile != "" { - opCtx := bqlabel.OperationalContext{ - App: bqlabel.AppSippy, - Command: "serve", - // outside prod, defaults to CLI as env and USER env var as operator - Environment: bqlabel.EnvCli, - Operator: os.Getenv("USER"), - } - env := bqlabel.EnvValue(os.Getenv("SIPPY_WEB_ENV")) // set in prod - if slices.Contains([]bqlabel.EnvValue{bqlabel.EnvWeb, bqlabel.EnvWebAuth, bqlabel.EnvWebQE}, env) { - opCtx.Environment = env - opCtx.Operator = string(env) - } - bigQueryClient, err = f.BigQueryFlags.GetBigQueryClient(context.Background(), opCtx, cacheClient, f.GoogleCloudFlags.ServiceAccountCredentialFile) - if err != nil { - return errors.WithMessage(err, "couldn't get bigquery client") - } + var crDataProvider dataprovider.DataProvider + switch f.DataProvider { + case "bigquery": + if f.GoogleCloudFlags.ServiceAccountCredentialFile != "" { + opCtx := bqlabel.OperationalContext{ + App: bqlabel.AppSippy, + Command: "serve", + // outside prod, defaults to CLI as env and USER env var as operator + Environment: bqlabel.EnvCli, + Operator: os.Getenv("USER"), + } + env := bqlabel.EnvValue(os.Getenv("SIPPY_WEB_ENV")) // set in prod + if slices.Contains([]bqlabel.EnvValue{bqlabel.EnvWeb, bqlabel.EnvWebAuth, bqlabel.EnvWebQE}, env) { + opCtx.Environment = env + opCtx.Operator = string(env) + } + bigQueryClient, err = f.BigQueryFlags.GetBigQueryClient(context.Background(), opCtx, cacheClient, f.GoogleCloudFlags.ServiceAccountCredentialFile) + if err != nil { + return errors.WithMessage(err, "couldn't get bigquery client") + } - if bigQueryClient != nil && f.CacheFlags.EnablePersistentCaching { - bigQueryClient = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bigQueryClient) - } + if bigQueryClient != nil && f.CacheFlags.EnablePersistentCaching { + bigQueryClient = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bigQueryClient) + } - gcsClient, err = gcs.NewGCSClient(context.TODO(), - f.GoogleCloudFlags.ServiceAccountCredentialFile, - f.GoogleCloudFlags.OAuthClientCredentialFile, - ) - if err != nil { - log.WithError(err).Warn("unable to create GCS client, some APIs may not work") + crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient) + + gcsClient, err = gcs.NewGCSClient(context.TODO(), + f.GoogleCloudFlags.ServiceAccountCredentialFile, + f.GoogleCloudFlags.OAuthClientCredentialFile, + ) + if err != nil { + log.WithError(err).Warn("unable to create GCS client, some APIs may not work") + } } + + case "postgres": + crDataProvider = pgprovider.NewPostgresProvider(dbc, cacheClient) + log.Info("Using Postgres data provider for component readiness") + + default: + return fmt.Errorf("unknown --data-provider %q, must be bigquery or postgres", f.DataProvider) } - // Make sure the db is intialized, otherwise let the user know: + // Make sure the db is initialized, otherwise let the user know: prowJobs := []models.ProwJob{} res := dbc.DB.Find(&prowJobs).Limit(1) if res.Error != nil { @@ -143,11 +166,13 @@ func NewServeCommand() *cobra.Command { pinnedDateTime := f.DBFlags.GetPinnedTime() - variantManager := f.ModeFlags.GetVariantManager(context.Background(), bigQueryClient) + var variantManager testidentification.VariantManager + if bigQueryClient != nil { + variantManager = f.ModeFlags.GetVariantManager(context.Background(), bigQueryClient) + } views, err := f.ComponentReadinessFlags.ParseViewsFile() if err != nil { log.WithError(err).Fatal("unable to load views") - } jiraClient, err := f.JiraFlags.GetJiraClient() @@ -167,6 +192,7 @@ func NewServeCommand() *cobra.Command { gcsClient, f.GoogleCloudFlags.StorageBucket, bigQueryClient, + crDataProvider, pinnedDateTime, cacheClient, f.ComponentReadinessFlags.CRTimeRoundingFactor, @@ -183,6 +209,7 @@ func NewServeCommand() *cobra.Command { context.Background(), dbc, bigQueryClient, + crDataProvider, util.GetReportEnd(pinnedDateTime), cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), views.ComponentReadiness, @@ -203,6 +230,7 @@ func NewServeCommand() *cobra.Command { context.Background(), dbc, bigQueryClient, + crDataProvider, util.GetReportEnd(pinnedDateTime), cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), views.ComponentReadiness, diff --git a/config/e2e-openshift.yaml b/config/e2e-openshift.yaml index 39814b6b2a..996e0aa39c 100644 --- a/config/e2e-openshift.yaml +++ b/config/e2e-openshift.yaml @@ -1,8 +1,8 @@ prow: url: https://prow.ci.openshift.org/prowjobs.js releases: - "4.20": + "4.22": jobs: - periodic-ci-openshift-release-master-nightly-4.20-e2e-aws-ovn-serial-2of2: true - periodic-ci-openshift-release-main-ci-4.20-e2e-azure-ovn-upgrade: true - periodic-ci-openshift-release-master-nightly-4.20-e2e-gcp-ovn-csi: true + periodic-ci-openshift-release-master-nightly-4.22-e2e-aws-ovn-serial-2of2: true + periodic-ci-openshift-release-main-ci-4.22-e2e-azure-ovn-upgrade: true + periodic-ci-openshift-release-master-nightly-4.22-e2e-gcp-ovn-csi: true diff --git a/config/e2e-views.yaml b/config/e2e-views.yaml index aab16d861c..00843238e9 100644 --- a/config/e2e-views.yaml +++ b/config/e2e-views.yaml @@ -1,81 +1,71 @@ ---- component_readiness: -- name: 4.20-main - base_release: - release: "4.19" - relative_start: ga-30d - relative_end: ga - sample_release: - release: "4.20" - relative_start: now-7d - relative_end: now - variant_options: - column_group_by: - Architecture: {} - Network: {} - Platform: {} - Topology: {} - db_group_by: - Architecture: {} - FeatureSet: {} - Installer: {} - Network: {} - Platform: {} - Suite: {} - Topology: {} - Upgrade: {} - include_variants: - Architecture: - - amd64 - FeatureSet: - - default - - techpreview - Installer: - - ipi - - upi - - hypershift - JobTier: - - blocking - - informing - - standard - LayeredProduct: - - none - - virt - Network: - - ovn - Owner: - - eng - - service-delivery - Platform: - - aws - - azure - - gcp - - metal - - rosa - - vsphere - Topology: - - ha - - microshift - - external - CGroupMode: - - v2 - ContainerRuntime: - - runc - - crun - advanced_options: - minimum_failure: 3 - confidence: 95 - pity_factor: 5 - ignore_missing: false - ignore_disruption: true - flake_as_failure: false - pass_rate_required_new_tests: 95 - include_multi_release_analysis: true - metrics: - enabled: true - regression_tracking: - enabled: true - prime_cache: - enabled: true - automate_jira: - enabled: true + - name: 4.22-main + base_release: + release: "4.21" + relative_start: now-60d + relative_end: now-30d + sample_release: + release: "4.22" + relative_start: now-3d + relative_end: now + test_id_options: {} + test_filters: {} + variant_options: + column_group_by: + Network: {} + Platform: {} + Topology: {} + db_group_by: + Architecture: {} + FeatureSet: {} + Installer: {} + LayeredProduct: {} + Network: {} + Platform: {} + Suite: {} + Topology: {} + Upgrade: {} + include_variants: + Architecture: + - amd64 + - arm64 + FeatureSet: + - default + - techpreview + Installer: + - ipi + LayeredProduct: + - none + Network: + - ovn + Platform: + - aws + - gcp + Suite: + - parallel + - serial + - unknown + Topology: + - ha + Upgrade: + - micro + - minor + - none + advanced_options: + minimum_failure: 3 + confidence: 95 + pity_factor: 5 + pass_rate_required_new_tests: 0 + pass_rate_required_all_tests: 0 + ignore_missing: false + ignore_disruption: false + flake_as_failure: false + include_multi_release_analysis: true + metrics: + enabled: false + regression_tracking: + enabled: true + automate_jira: + enabled: false + prime_cache: + enabled: true diff --git a/pkg/api/componentreadiness/component_report.go b/pkg/api/componentreadiness/component_report.go index 9128fc02fc..7c6919eaeb 100644 --- a/pkg/api/componentreadiness/component_report.go +++ b/pkg/api/componentreadiness/component_report.go @@ -9,27 +9,23 @@ import ( "reflect" "slices" "sort" - "strconv" - "strings" "sync" "time" - "cloud.google.com/go/bigquery" "github.com/apache/thrift/lib/go/thrift" fischer "github.com/glycerine/golang-fisher-exact" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware/linkinjector" regressionallowances2 "github.com/openshift/sippy/pkg/api/componentreadiness/middleware/regressionallowances" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware/regressiontracker" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" - "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "google.golang.org/api/iterator" "github.com/openshift/sippy/pkg/api" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware/releasefallback" "github.com/openshift/sippy/pkg/api/componentreadiness/query" @@ -38,7 +34,6 @@ import ( "github.com/openshift/sippy/pkg/apis/cache" configv1 "github.com/openshift/sippy/pkg/apis/config/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - bqcachedclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" @@ -59,73 +54,49 @@ var ( DefaultDBGroupBy = "Platform,Architecture,Network,Topology,FeatureSet,Upgrade,Suite,Installer,LayeredProduct" ) -func getSingleColumnResultToSlice(ctx context.Context, q *bigquery.Query) ([]string, error) { - names := []string{} - it, err := q.Read(ctx) - if err != nil { - log.WithError(err).Error("error querying test status from bigquery") - return names, err - } - - for { - row := struct{ Name string }{} - err := it.Next(&row) - if err == iterator.Done { - break - } - if err != nil { - log.WithError(err).Error("error parsing component from bigquery") - return names, err - } - names = append(names, row.Name) - } - return names, nil -} - // TODO: in several of the below functions we instantiate an entire ComponentReportGenerator // to fetch some small piece of data. These look like they should be broken out. The partial // instantiation of a complex object is risky in terms of bugs and maintenance. -func GetComponentTestVariantsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) (CacheVariants, []error) { +func GetComponentTestVariants(ctx context.Context, provider dataprovider.DataProvider) (CacheVariants, []error) { generator := ComponentReportGenerator{ - client: client, + dataProvider: provider, } - return api.GetDataFromCacheOrGenerate[CacheVariants](ctx, client.Cache, cache.RequestOptions{}, + return api.GetDataFromCacheOrGenerate[CacheVariants](ctx, provider.Cache(), cache.RequestOptions{}, api.NewCacheSpec(generator, "TestVariants~", nil), generator.GenerateCacheVariants, CacheVariants{}) } -func GetJobVariantsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) (crtest.JobVariants, +func GetJobVariants(ctx context.Context, provider dataprovider.DataProvider) (crtest.JobVariants, []error) { generator := ComponentReportGenerator{ - client: client, + dataProvider: provider, } - return api.GetDataFromCacheOrGenerate[crtest.JobVariants](ctx, client.Cache, cache.RequestOptions{}, + return api.GetDataFromCacheOrGenerate[crtest.JobVariants](ctx, provider.Cache(), cache.RequestOptions{}, api.NewCacheSpec(generator, "TestAllVariants~", nil), generator.GenerateJobVariants, crtest.JobVariants{}) } -func GetComponentReportFromBigQuery( +func GetComponentReport( ctx context.Context, - client *bqcachedclient.Client, + provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, variantJunitTableOverrides []configv1.VariantJunitTableOverride, baseURL string, ) (report crtype.ComponentReport, errs []error) { - releaseConfigs, err := api.GetReleases(ctx, client, false) + releaseConfigs, err := provider.QueryReleases(ctx) if err != nil { return report, []error{err} } - generator := NewComponentReportGenerator(client, reqOptions, dbc, variantJunitTableOverrides, releaseConfigs, baseURL) + generator := NewComponentReportGenerator(provider, reqOptions, dbc, variantJunitTableOverrides, releaseConfigs, baseURL) if os.Getenv("DEV_MODE") == "1" { report, errs = generator.GenerateReport(ctx) if errs != nil { return report, errs } - // Run the PostAnalysis, specifically, to make sure the test_details report links are injected err = generator.PostAnalysis(&report) if err != nil { return report, []error{err} @@ -135,7 +106,7 @@ func GetComponentReportFromBigQuery( report, errs = api.GetDataFromCacheOrGenerate[crtype.ComponentReport]( ctx, - generator.client.Cache, generator.ReqOptions.CacheOption, + generator.getCache(), generator.ReqOptions.CacheOption, api.NewCacheSpec(generator.GetCacheKey(ctx), ComponentReportCacheKeyPrefix, nil), generator.GenerateReport, crtype.ComponentReport{}) @@ -159,12 +130,11 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport) // Give middleware their chance to adjust the result for ri, row := range report.Rows { for ci, col := range row.Columns { + // Track the worst (most negative) status across all regressed tests after PostAnalysis. + // We only hit this loop if there are regressed tests, which is good because we know the + // cell status can't be improved or missing basis/sample. + worstStatus := report.Rows[ri].Columns[ci].Status for rti := range col.RegressedTests { - // Carefully update the column status. We only hit this loop if there are regressed tests, which is - // good because we know the cell status can't be improved or missing basis/sample. - // All we need to do now is track the lowest (i.e. worst) status we see after PostAnalysis, - // and make that our new cell status. - var initialStatus crtest.Status testKey := crtest.Identification{ RowIdentification: col.RegressedTests[rti].RowIdentification, ColumnIdentification: col.RegressedTests[rti].ColumnIdentification, @@ -172,21 +142,21 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport) if err := c.middlewares.PostAnalysis(testKey, &report.Rows[ri].Columns[ci].RegressedTests[rti].TestComparison); err != nil { return err } - if newStatus := report.Rows[ri].Columns[ci].RegressedTests[rti].TestComparison.ReportStatus; newStatus < initialStatus { - // After PostAnalysis this is our new worst status observed, so update the cell's status in the grid - report.Rows[ri].Columns[ci].Status = newStatus + if newStatus := report.Rows[ri].Columns[ci].RegressedTests[rti].TestComparison.ReportStatus; newStatus < worstStatus { + worstStatus = newStatus } } + report.Rows[ri].Columns[ci].Status = worstStatus } } return nil } -func NewComponentReportGenerator(client *bqcachedclient.Client, reqOptions reqopts.RequestOptions, dbc *db.DB, variantJunitTableOverrides []configv1.VariantJunitTableOverride, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator { +func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, variantJunitTableOverrides []configv1.VariantJunitTableOverride, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator { slices.Sort(reqOptions.Capabilities) // normalize ordering so cache keys match generator := ComponentReportGenerator{ - client: client, + dataProvider: provider, ReqOptions: reqOptions, dbc: dbc, variantJunitTableOverrides: variantJunitTableOverrides, @@ -204,7 +174,7 @@ func NewComponentReportGenerator(client *bqcachedclient.Client, reqOptions reqop // is marshalled for the cache key and should be changed when the object being // cached changes in a way that will no longer be compatible with any prior cached version. type ComponentReportGenerator struct { - client *bqcachedclient.Client + dataProvider dataprovider.DataProvider dbc *db.DB ReqOptions reqopts.RequestOptions variantJunitTableOverrides []configv1.VariantJunitTableOverride @@ -301,63 +271,18 @@ func (c *ComponentReportGenerator) GenerateCacheVariants(ctx context.Context) (C } func (c *ComponentReportGenerator) GenerateJobVariants(ctx context.Context) (crtest.JobVariants, []error) { - errs := []error{} - variants := crtest.JobVariants{Variants: map[string][]string{}} - queryString := fmt.Sprintf(`SELECT variant_name, ARRAY_AGG(DISTINCT variant_value ORDER BY variant_value) AS variant_values - FROM - %s.job_variants - WHERE - variant_value!="" - GROUP BY - variant_name`, c.client.Dataset) - q := c.client.Query(ctx, bqlabel.CRJobVariants, queryString) - it, err := q.Read(ctx) - if err != nil { - log.WithError(err).Errorf("error querying variants from bigquery for %s", queryString) - return variants, []error{err} - } - - floatVariants := sets.NewString("FromRelease", "FromReleaseMajor", "FromReleaseMinor", "Release", "ReleaseMajor", "ReleaseMinor") - for { - row := bq.JobVariant{} - err := it.Next(&row) - if err == iterator.Done { - break - } - if err != nil { - wrappedErr := errors.Wrapf(err, "error fetching variant row") - log.WithError(err).Error("error fetching variants from bigquery") - errs = append(errs, wrappedErr) - return variants, errs - } + return c.dataProvider.QueryJobVariants(ctx) +} - // Sort all releases in proper orders - if floatVariants.Has(row.VariantName) { - sort.Slice(row.VariantValues, func(i, j int) bool { - iStrings := strings.Split(row.VariantValues[i], ".") - jStrings := strings.Split(row.VariantValues[j], ".") - for idx, iString := range iStrings { - if iValue, err := strconv.ParseInt(iString, 10, 32); err == nil { - if jValue, err := strconv.ParseInt(jStrings[idx], 10, 32); err == nil { - if iValue != jValue { - return iValue < jValue - } - } - } - } - return false - }) - } - variants.Variants[row.VariantName] = row.VariantValues - } - return variants, nil +func (c *ComponentReportGenerator) getCache() cache.Cache { + return c.dataProvider.Cache() } func (c *ComponentReportGenerator) initializeMiddleware() { c.middlewares = middleware.List{} // Initialize all our middleware applicable to this request. if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis { - c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.client, c.ReqOptions, c.releaseConfigs)) + c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs)) } if c.dbc != nil { c.middlewares = append(c.middlewares, regressiontracker.NewRegressionTrackerMiddleware(c.dbc, c.ReqOptions)) @@ -375,8 +300,8 @@ func (c *ComponentReportGenerator) initializeMiddleware() { func (c *ComponentReportGenerator) GenerateReport(ctx context.Context) (crtype.ComponentReport, []error) { before := time.Now() - // Load all test pass/fail counts from bigquery, both sample and basis - componentReportTestStatus, errs := c.getTestStatusFromBigQuery(ctx) + // Load all test pass/fail counts, both sample and basis + componentReportTestStatus, errs := c.getTestStatus(ctx) if len(errs) > 0 { return crtype.ComponentReport{}, errs } @@ -402,71 +327,31 @@ func (c *ComponentReportGenerator) GenerateReport(ctx context.Context) (crtype.C return report, nil } -// getBaseQueryStatus builds the basis query, executes it, and returns the basis test status. -func (c *ComponentReportGenerator) getBaseQueryStatus(ctx context.Context, - allJobVariants crtest.JobVariants) (map[string]bq.TestStatus, []error) { - - generator := query.NewBaseQueryGenerator(c.client, c.ReqOptions, allJobVariants) - - componentReportTestStatus, errs := api.GetDataFromCacheOrGenerate[bq.ReportTestStatus](ctx, - c.client.Cache, c.ReqOptions.CacheOption, - api.NewCacheSpec(generator, "BaseTestStatus~", &c.ReqOptions.BaseRelease.End), - generator.QueryTestStatus, bq.ReportTestStatus{}) - - if len(errs) > 0 { - return nil, errs - } - - return componentReportTestStatus.BaseStatus, nil -} - -// getSampleQueryStatus builds the sample query, executes it, and returns the sample test status. -func (c *ComponentReportGenerator) getSampleQueryStatus( - ctx context.Context, - allJobVariants crtest.JobVariants, - includeVariants map[string][]string, - start, end time.Time, - junitTable string) (map[string]bq.TestStatus, []error) { - - generator := query.NewSampleQueryGenerator(c.client, c.ReqOptions, allJobVariants, includeVariants, start, end, junitTable) - - componentReportTestStatus, errs := api.GetDataFromCacheOrGenerate[bq.ReportTestStatus](ctx, - c.client.Cache, c.ReqOptions.CacheOption, - api.NewCacheSpec(generator, "SampleTestStatus~", &c.ReqOptions.SampleRelease.End), - generator.QueryTestStatus, bq.ReportTestStatus{}) - - if len(errs) > 0 { - return nil, errs - } - - return componentReportTestStatus.SampleStatus, nil -} - -// getTestStatusFromBigQuery orchestrates the actual fetching of junit test run data for both basis and sample. +// getTestStatus orchestrates the actual fetching of junit test run data for both basis and sample. // goroutines are used to concurrently request the data for basis, sample, and various other edge cases. -func (c *ComponentReportGenerator) getTestStatusFromBigQuery(ctx context.Context) (bq.ReportTestStatus, []error) { +func (c *ComponentReportGenerator) getTestStatus(ctx context.Context) (crstatus.ReportTestStatus, []error) { before := time.Now() - fLog := log.WithField("func", "getTestStatusFromBigQuery") - allJobVariants, errs := GetJobVariantsFromBigQuery(ctx, c.client) + fLog := log.WithField("func", "getTestStatus") + allJobVariants, errs := GetJobVariants(ctx, c.dataProvider) if len(errs) > 0 { - fLog.Errorf("failed to get variants from bigquery") - return bq.ReportTestStatus{}, errs + fLog.Errorf("failed to get job variants") + return crstatus.ReportTestStatus{}, errs } - var baseStatus, sampleStatus map[string]bq.TestStatus - baseStatusCh := make(chan map[string]bq.TestStatus) // TODO: not hooked up yet, just in place for the interface for now + var baseStatus, sampleStatus map[string]crstatus.TestStatus + baseStatusCh := make(chan map[string]crstatus.TestStatus) // TODO: not hooked up yet, just in place for the interface for now var baseErrs, sampleErrs []error wg := &sync.WaitGroup{} // channels for status as we may collect status from multiple queries run in separate goroutines - sampleStatusCh := make(chan map[string]bq.TestStatus) + sampleStatusCh := make(chan map[string]crstatus.TestStatus) errCh := make(chan error) statusDoneCh := make(chan struct{}) // To signal when all processing is done statusErrsDoneCh := make(chan struct{}) // To signal when all processing is done // generate inputs to the channels c.middlewares.Query(ctx, wg, allJobVariants, baseStatusCh, sampleStatusCh, errCh) - goInterruptible(ctx, wg, func() { baseStatus, baseErrs = c.getBaseQueryStatus(ctx, allJobVariants) }) + goInterruptible(ctx, wg, func() { baseStatus, baseErrs = c.dataProvider.QueryBaseTestStatus(ctx, c.ReqOptions, allJobVariants) }) goInterruptible(ctx, wg, func() { includeVariants, skipQuery := copyIncludeVariantsAndRemoveOverrides(c.variantJunitTableOverrides, -1, c.ReqOptions.VariantOption.IncludeVariants) if skipQuery { @@ -474,7 +359,7 @@ func (c *ComponentReportGenerator) getTestStatusFromBigQuery(ctx context.Context return } fLog.Infof("running default sample query with includeVariants: %+v", includeVariants) - status, errs := c.getSampleQueryStatus(ctx, allJobVariants, includeVariants, c.ReqOptions.SampleRelease.Start, c.ReqOptions.SampleRelease.End, query.DefaultJunitTable) + status, errs := c.dataProvider.QuerySampleTestStatus(ctx, c.ReqOptions, allJobVariants, includeVariants, c.ReqOptions.SampleRelease.Start, c.ReqOptions.SampleRelease.End, query.DefaultJunitTable) fLog.Infof("received %d test statuses and %d errors from default query", len(status), len(errs)) sampleStatusCh <- status for _, err := range errs { @@ -499,7 +384,7 @@ func (c *ComponentReportGenerator) getTestStatusFromBigQuery(ctx context.Context for k, v := range status { if sampleStatus == nil { fLog.Warnf("initializing sampleStatus map") - sampleStatus = make(map[string]bq.TestStatus) + sampleStatus = make(map[string]crstatus.TestStatus) } if v2, ok := sampleStatus[k]; ok { fLog.Warnf("sampleStatus already had key: %+v", k) @@ -527,17 +412,17 @@ func (c *ComponentReportGenerator) getTestStatusFromBigQuery(ctx context.Context errs = append(errs, baseErrs...) errs = append(errs, sampleErrs...) } - log.Infof("getTestStatusFromBigQuery completed in %s with %d sample results and %d base results from db", + log.Infof("getTestStatus completed in %s with %d sample results and %d base results", time.Since(before), len(sampleStatus), len(baseStatus)) now := time.Now() - return bq.ReportTestStatus{BaseStatus: baseStatus, SampleStatus: sampleStatus, GeneratedAt: &now}, errs + return crstatus.ReportTestStatus{BaseStatus: baseStatus, SampleStatus: sampleStatus, GeneratedAt: &now}, errs } // fork additional sample queries for the overrides func (c *ComponentReportGenerator) goRunOverrideSampleQueries( ctx context.Context, wg *sync.WaitGroup, fLog *log.Entry, allJobVariants crtest.JobVariants, - sampleStatusCh chan map[string]bq.TestStatus, + sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error, ) { for i, or := range c.variantJunitTableOverrides { @@ -562,7 +447,7 @@ func (c *ComponentReportGenerator) goRunOverrideSampleQueries( errCh <- err return } - status, errs := c.getSampleQueryStatus(ctx, allJobVariants, includeVariants, start, end, override.TableName) + status, errs := c.dataProvider.QuerySampleTestStatus(ctx, c.ReqOptions, allJobVariants, includeVariants, start, end, override.TableName) fLog.Infof("received %d test statuses and %d errors from override query", len(status), len(errs)) sampleStatusCh <- status for _, err := range errs { @@ -641,15 +526,15 @@ func shouldSkipVariant(overrides []configv1.VariantJunitTableOverride, currOverr return false } -var componentAndCapabilityGetter func(test crtest.KeyWithVariants, stats bq.TestStatus) (string, []string) +var componentAndCapabilityGetter func(test crtest.KeyWithVariants, stats crstatus.TestStatus) (string, []string) -func testToComponentAndCapability(_ crtest.KeyWithVariants, stats bq.TestStatus) (string, []string) { +func testToComponentAndCapability(_ crtest.KeyWithVariants, stats crstatus.TestStatus) (string, []string) { return stats.Component, stats.Capabilities } // getRowColumnIdentifications defines the rows and columns since they are variable. For rows, different pages have different row titles (component, capability etc) // Columns titles depends on the columnGroupBy parameter user requests. A particular test can belong to multiple rows of different capabilities. -func (c *ComponentReportGenerator) getRowColumnIdentifications(testIDStr string, stats bq.TestStatus) ([]crtest.RowIdentification, []crtest.ColumnID, error) { +func (c *ComponentReportGenerator) getRowColumnIdentifications(testIDStr string, stats crstatus.TestStatus) ([]crtest.RowIdentification, []crtest.ColumnID, error) { var test crtest.KeyWithVariants columnGroupByVariants := c.ReqOptions.VariantOption.ColumnGroupBy // We show column groups by DBGroupBy only for the last page before test details @@ -801,8 +686,8 @@ func updateCellStatus( func initTestAnalysisStruct( testStats *testdetails.TestComparison, reqOptions reqopts.RequestOptions, - sampleStatus bq.TestStatus, - baseStatus *bq.TestStatus) { + sampleStatus crstatus.TestStatus, + baseStatus *crstatus.TestStatus) { // Default to required confidence from request, middleware may adjust later. testStats.RequiredConfidence = reqOptions.AdvancedOption.Confidence @@ -823,7 +708,7 @@ func initTestAnalysisStruct( } } -func (c *ComponentReportGenerator) generateComponentTestReport(basisStatusMap, sampleStatusMap map[string]bq.TestStatus) (crtype.ComponentReport, error) { +func (c *ComponentReportGenerator) generateComponentTestReport(basisStatusMap, sampleStatusMap map[string]crstatus.TestStatus) (crtype.ComponentReport, error) { // aggregatedStatus is the aggregated status based on the requested rows and columns aggregatedStatus := map[crtest.RowIdentification]map[crtest.ColumnID]cellStatus{} // allRows and allColumns are used to make sure rows are ordered and all rows have the same columns in the same order @@ -1140,24 +1025,7 @@ func (c *ComponentReportGenerator) fischerExactTest(confidenceRequired, sampleFa func (c *ComponentReportGenerator) getUniqueJUnitColumnValuesLast60Days(ctx context.Context, field string, nested bool) ([]string, error) { - unnest := "" - if nested { - unnest = fmt.Sprintf(", UNNEST(%s) nested", field) - field = "nested" - } - - queryString := fmt.Sprintf(`SELECT - DISTINCT %s as name - FROM - %s.junit %s - WHERE - modified_time > DATETIME_SUB(CURRENT_DATETIME(), INTERVAL 60 DAY) - ORDER BY - name`, field, c.client.Dataset, unnest) - - q := c.client.Query(ctx, bqlabel.CRJunitColumnCount, queryString) - - return getSingleColumnResultToSlice(ctx, q) + return c.dataProvider.QueryUniqueVariantValues(ctx, field, nested) } func init() { diff --git a/pkg/api/componentreadiness/component_report_test.go b/pkg/api/componentreadiness/component_report_test.go index 3fc74081f2..ba0f00cc8b 100644 --- a/pkg/api/componentreadiness/component_report_test.go +++ b/pkg/api/componentreadiness/component_report_test.go @@ -10,7 +10,7 @@ import ( "time" "github.com/apache/thrift/lib/go/thrift" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" @@ -24,7 +24,7 @@ import ( "github.com/openshift/sippy/pkg/util/sets" ) -func fakeComponentAndCapabilityGetter(test crtest.KeyWithVariants, stats bq.TestStatus) (string, []string) { +func fakeComponentAndCapabilityGetter(test crtest.KeyWithVariants, stats crstatus.TestStatus) (string, []string) { name := stats.TestName known := map[string]struct { component string @@ -251,7 +251,7 @@ func TestGenerateComponentReport(t *testing.T) { if err != nil { assert.NoError(t, err, "error marshalling awsAMD64OVNInstallerIPITest") } - awsAMD64OVNBaseTestStats90Percent := bq.TestStatus{ + awsAMD64OVNBaseTestStats90Percent := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard"}, Count: crtest.Count{ @@ -260,7 +260,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 900, }, } - awsAMD64OVNBaseTestStats50Percent := bq.TestStatus{ + awsAMD64OVNBaseTestStats50Percent := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard"}, Count: crtest.Count{ @@ -269,7 +269,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 500, }, } - awsAMD64OVNBaseTestStatsVariants90Percent := bq.TestStatus{ + awsAMD64OVNBaseTestStatsVariants90Percent := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard", "fips"}, Count: crtest.Count{ @@ -278,7 +278,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 900, }, } - awsAMD64OVNSampleTestStats90Percent := bq.TestStatus{ + awsAMD64OVNSampleTestStats90Percent := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard"}, Count: crtest.Count{ @@ -287,7 +287,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 90, }, } - awsAMD64OVNSampleTestStats85Percent := bq.TestStatus{ + awsAMD64OVNSampleTestStats85Percent := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard"}, Count: crtest.Count{ @@ -296,7 +296,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 85, }, } - awsAMD64OVNSampleTestStats50Percent := bq.TestStatus{ + awsAMD64OVNSampleTestStats50Percent := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard"}, Count: crtest.Count{ @@ -305,7 +305,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 50, }, } - awsAMD64OVNSampleTestStatsTiny := bq.TestStatus{ + awsAMD64OVNSampleTestStatsTiny := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard"}, Count: crtest.Count{ @@ -314,7 +314,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 1, }, } - awsAMD64OVNSampleTestStatsVariants90Percent := bq.TestStatus{ + awsAMD64OVNSampleTestStatsVariants90Percent := crstatus.TestStatus{ TestName: "test 1", Variants: []string{"standard", "fips"}, Count: crtest.Count{ @@ -323,7 +323,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 90, }, } - awsAMD64SDNBaseTestStats90Percent := bq.TestStatus{ + awsAMD64SDNBaseTestStats90Percent := crstatus.TestStatus{ TestName: "test 2", Variants: []string{"standard"}, Count: crtest.Count{ @@ -332,7 +332,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 900, }, } - awsAMD64SDNBaseTestStats50Percent := bq.TestStatus{ + awsAMD64SDNBaseTestStats50Percent := crstatus.TestStatus{ TestName: "test 2", Variants: []string{"standard"}, Count: crtest.Count{ @@ -341,7 +341,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 500, }, } - awsAMD64SDNSampleTestStats90Percent := bq.TestStatus{ + awsAMD64SDNSampleTestStats90Percent := crstatus.TestStatus{ TestName: "test 2", Variants: []string{"standard"}, Count: crtest.Count{ @@ -350,7 +350,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 90, }, } - awsAMD64OVN2BaseTestStats90Percent := bq.TestStatus{ + awsAMD64OVN2BaseTestStats90Percent := crstatus.TestStatus{ TestName: "test 3", Variants: []string{"standard"}, Count: crtest.Count{ @@ -359,7 +359,7 @@ func TestGenerateComponentReport(t *testing.T) { SuccessCount: 900, }, } - awsAMD64OVN2SampleTestStats80Percent := bq.TestStatus{ + awsAMD64OVN2SampleTestStats80Percent := crstatus.TestStatus{ TestName: "test 3", Variants: []string{"standard"}, Count: crtest.Count{ @@ -446,18 +446,18 @@ func TestGenerateComponentReport(t *testing.T) { tests := []struct { name string generator ComponentReportGenerator - baseStatus map[string]bq.TestStatus - sampleStatus map[string]bq.TestStatus + baseStatus map[string]crstatus.TestStatus + sampleStatus map[string]crstatus.TestStatus expectedReport crtype.ComponentReport }{ { name: "top page test no significant and missing data", generator: defaultComponentReportGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats85Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -499,12 +499,12 @@ func TestGenerateComponentReport(t *testing.T) { { name: "top page test with both improvement and regression", generator: defaultComponentReportGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64OVN2TestBytes): awsAMD64OVN2BaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats50Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats50Percent, string(awsAMD64OVN2TestBytes): awsAMD64OVN2SampleTestStats80Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, @@ -629,11 +629,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "component page test no significant and missing data", generator: componentPageGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -671,11 +671,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "component page test with both improvement and regression", generator: componentPageGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats50Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats50Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, @@ -713,11 +713,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "capability page test no significant and missing data", generator: capabilityPageGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -742,11 +742,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "capability page test with both improvement and regression", generator: capabilityPageGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats50Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats50Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -771,11 +771,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "test page test no significant and missing data", generator: testPageGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -800,11 +800,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "test page test with both improvement and regression", generator: testPageGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats50Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats50Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -840,11 +840,11 @@ func TestGenerateComponentReport(t *testing.T) { }, }, }, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats85Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -937,11 +937,11 @@ func TestGenerateComponentReport(t *testing.T) { }, }, }, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats85Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -979,11 +979,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "top page test minimum failure no regression", generator: defaultComponentReportGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStatsTiny, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -1021,11 +1021,11 @@ func TestGenerateComponentReport(t *testing.T) { { name: "top page test group by installer", generator: groupByInstallerComponentReportGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNVariantsTestBytes): awsAMD64OVNBaseTestStatsVariants90Percent, string(awsAMD64SDNInstallerUPITestBytes): awsAMD64SDNBaseTestStats90Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNVariantsTestBytes): awsAMD64OVNSampleTestStatsVariants90Percent, string(awsAMD64SDNInstallerUPITestBytes): awsAMD64SDNSampleTestStats90Percent, }, @@ -1067,12 +1067,12 @@ func TestGenerateComponentReport(t *testing.T) { { name: "top page test with both improvement and regression flake as failure", generator: flakeFailComponentReportGenerator, - baseStatus: map[string]bq.TestStatus{ + baseStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNBaseTestStats90Percent, string(awsAMD64OVN2TestBytes): awsAMD64OVN2BaseTestStats90Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNBaseTestStats50Percent, }, - sampleStatus: map[string]bq.TestStatus{ + sampleStatus: map[string]crstatus.TestStatus{ string(awsAMD64OVNTestBytes): awsAMD64OVNSampleTestStats50Percent, string(awsAMD64OVN2TestBytes): awsAMD64OVN2SampleTestStats80Percent, string(awsAMD64SDNTestBytes): awsAMD64SDNSampleTestStats90Percent, @@ -1568,11 +1568,11 @@ func TestGenerateComponentTestDetailsReport(t *testing.T) { } componentAndCapabilityGetter = fakeComponentAndCapabilityGetter for _, tc := range tests { - baseStats := map[string][]bq.TestJobRunRows{} - sampleStats := map[string][]bq.TestJobRunRows{} + baseStats := map[string][]crstatus.TestJobRunRows{} + sampleStats := map[string][]crstatus.TestJobRunRows{} for _, testStats := range tc.baseRequiredJobStats { for i := 0; i < testStats.Success; i++ { - baseStats[testStats.job] = append(baseStats[testStats.job], bq.TestJobRunRows{ + baseStats[testStats.job] = append(baseStats[testStats.job], crstatus.TestJobRunRows{ ProwJob: testStats.job, Count: crtest.Count{ TotalCount: 1, @@ -1581,13 +1581,13 @@ func TestGenerateComponentTestDetailsReport(t *testing.T) { }) } for i := 0; i < testStats.Failure; i++ { - baseStats[testStats.job] = append(baseStats[testStats.job], bq.TestJobRunRows{ + baseStats[testStats.job] = append(baseStats[testStats.job], crstatus.TestJobRunRows{ ProwJob: testStats.job, Count: crtest.Count{TotalCount: 1}, }) } for i := 0; i < testStats.Flake; i++ { - baseStats[testStats.job] = append(baseStats[testStats.job], bq.TestJobRunRows{ + baseStats[testStats.job] = append(baseStats[testStats.job], crstatus.TestJobRunRows{ ProwJob: testStats.job, Count: crtest.Count{ TotalCount: 1, @@ -1598,7 +1598,7 @@ func TestGenerateComponentTestDetailsReport(t *testing.T) { } for _, testStats := range tc.sampleRequiredJobStats { for i := 0; i < testStats.Success; i++ { - sampleStats[testStats.job] = append(sampleStats[testStats.job], bq.TestJobRunRows{ + sampleStats[testStats.job] = append(sampleStats[testStats.job], crstatus.TestJobRunRows{ ProwJob: testStats.job, Count: crtest.Count{ TotalCount: 1, @@ -1607,13 +1607,13 @@ func TestGenerateComponentTestDetailsReport(t *testing.T) { }) } for i := 0; i < testStats.Failure; i++ { - sampleStats[testStats.job] = append(sampleStats[testStats.job], bq.TestJobRunRows{ + sampleStats[testStats.job] = append(sampleStats[testStats.job], crstatus.TestJobRunRows{ ProwJob: testStats.job, Count: crtest.Count{TotalCount: 1}, }) } for i := 0; i < testStats.Flake; i++ { - sampleStats[testStats.job] = append(sampleStats[testStats.job], bq.TestJobRunRows{ + sampleStats[testStats.job] = append(sampleStats[testStats.job], crstatus.TestJobRunRows{ ProwJob: testStats.job, Count: crtest.Count{ TotalCount: 1, diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go new file mode 100644 index 0000000000..283e2914bb --- /dev/null +++ b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go @@ -0,0 +1,412 @@ +package bigquery + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "cloud.google.com/go/bigquery" + log "github.com/sirupsen/logrus" + "google.golang.org/api/iterator" + + apiPkg "github.com/openshift/sippy/pkg/api" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" + "github.com/openshift/sippy/pkg/api/componentreadiness/query" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + apiCache "github.com/openshift/sippy/pkg/apis/cache" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/bigquery/bqlabel" + "github.com/openshift/sippy/pkg/util/param" + "github.com/openshift/sippy/pkg/util/sets" +) + +var _ dataprovider.DataProvider = &BigQueryProvider{} + +// BigQueryProvider implements dataprovider.DataProvider using Google BigQuery +// as the backing data store, wrapping the existing query generators. +type BigQueryProvider struct { + client *bqcachedclient.Client +} + +func NewBigQueryProvider(client *bqcachedclient.Client) *BigQueryProvider { + return &BigQueryProvider{client: client} +} + +// Client returns the underlying BigQuery client for callers that still need direct access +// during the migration period. +func (p *BigQueryProvider) Client() *bqcachedclient.Client { + return p.client +} + +func (p *BigQueryProvider) Cache() apiCache.Cache { + return p.client.Cache +} + +// --- TestStatusQuerier --- + +func (p *BigQueryProvider) QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string]crstatus.TestStatus, []error) { + + generator := query.NewBaseQueryGenerator(p.client, reqOptions, allJobVariants) + result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.ReportTestStatus]( + ctx, p.client.Cache, reqOptions.CacheOption, + apiPkg.NewCacheSpec(generator, "BaseTestStatus~", &reqOptions.BaseRelease.End), + generator.QueryTestStatus, crstatus.ReportTestStatus{}) + if len(errs) > 0 { + return nil, errs + } + return result.BaseStatus, nil +} + +func (p *BigQueryProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time, + dataSource string) (map[string]crstatus.TestStatus, []error) { + + generator := query.NewSampleQueryGenerator(p.client, reqOptions, allJobVariants, includeVariants, start, end, dataSource) + result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.ReportTestStatus]( + ctx, p.client.Cache, reqOptions.CacheOption, + apiPkg.NewCacheSpec(generator, "SampleTestStatus~", &reqOptions.SampleRelease.End), + generator.QueryTestStatus, crstatus.ReportTestStatus{}) + if len(errs) > 0 { + return nil, errs + } + return result.SampleStatus, nil +} + +// --- TestDetailsQuerier --- + +func (p *BigQueryProvider) QueryBaseJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string][]crstatus.TestJobRunRows, []error) { + + generator := query.NewBaseTestDetailsQueryGenerator( + log.WithField("func", "QueryBaseJobRunTestStatus"), + p.client, reqOptions, allJobVariants, + reqOptions.BaseRelease.Name, reqOptions.BaseRelease.Start, reqOptions.BaseRelease.End, + reqOptions.TestIDOptions) + + result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.TestJobRunStatuses]( + ctx, p.client.Cache, reqOptions.CacheOption, + apiPkg.NewCacheSpec(generator, "BaseJobRunTestStatus~", &reqOptions.BaseRelease.End), + generator.QueryTestStatus, crstatus.TestJobRunStatuses{}) + if len(errs) > 0 { + return nil, errs + } + return result.BaseStatus, nil +} + +func (p *BigQueryProvider) QuerySampleJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time, + dataSource string) (map[string][]crstatus.TestJobRunRows, []error) { + + generator := query.NewSampleTestDetailsQueryGenerator(p.client, reqOptions, allJobVariants, includeVariants, start, end, dataSource) + + result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.TestJobRunStatuses]( + ctx, p.client.Cache, reqOptions.CacheOption, + apiPkg.NewCacheSpec(generator, "SampleJobRunTestStatus~", &end), + generator.QueryTestStatus, crstatus.TestJobRunStatuses{}) + if len(errs) > 0 { + return nil, errs + } + return result.SampleStatus, nil +} + +// --- MetadataQuerier --- + +func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error) { + variants := crtest.JobVariants{Variants: map[string][]string{}} + queryString := fmt.Sprintf(`SELECT variant_name, ARRAY_AGG(DISTINCT variant_value ORDER BY variant_value) AS variant_values + FROM + %s.job_variants + WHERE + variant_value!="" + GROUP BY + variant_name`, p.client.Dataset) + q := p.client.Query(ctx, bqlabel.CRJobVariants, queryString) + it, err := q.Read(ctx) + if err != nil { + log.WithError(err).Errorf("error querying variants from bigquery") + return variants, []error{err} + } + + floatVariants := sets.NewString("FromRelease", "FromReleaseMajor", "FromReleaseMinor", "Release", "ReleaseMajor", "ReleaseMinor") + for { + row := crstatus.JobVariant{} + err := it.Next(&row) + if err == iterator.Done { + break + } + if err != nil { + log.WithError(err).Error("error fetching variants from bigquery") + return variants, []error{err} + } + + if floatVariants.Has(row.VariantName) { + sort.Slice(row.VariantValues, func(i, j int) bool { + iStrings := strings.Split(row.VariantValues[i], ".") + jStrings := strings.Split(row.VariantValues[j], ".") + for idx, iString := range iStrings { + if idx >= len(jStrings) { + return false + } + if iValue, err := strconv.ParseInt(iString, 10, 32); err == nil { + if jValue, err := strconv.ParseInt(jStrings[idx], 10, 32); err == nil { + if iValue != jValue { + return iValue < jValue + } + } + } + } + return len(iStrings) < len(jStrings) + }) + } + variants.Variants[row.VariantName] = row.VariantValues + } + return variants, nil +} + +func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { + return query.GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions) +} + +func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { + return apiPkg.GetReleasesFromBigQuery(ctx, p.client) +} + +func (p *BigQueryProvider) QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) { + unnest := "" + if nested { + unnest = fmt.Sprintf(", UNNEST(%s) nested", field) + field = "nested" + } + + queryString := fmt.Sprintf(`SELECT + DISTINCT %s as name + FROM + %s.junit %s + WHERE + modified_time > DATETIME_SUB(CURRENT_DATETIME(), INTERVAL 60 DAY) + ORDER BY + name`, field, p.client.Dataset, unnest) + + q := p.client.Query(ctx, bqlabel.CRJunitColumnCount, queryString) + return getSingleColumnResultToSlice(ctx, q) +} + +// --- JobQuerier --- + +func (p *BigQueryProvider) QueryJobRuns(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + release string, start, end time.Time) (map[string]dataprovider.JobRunStats, error) { + + joinVariants := "" + for _, v := range sortedKeys(allJobVariants.Variants) { + cleanV := param.Cleanse(v) + joinVariants += fmt.Sprintf( + "LEFT JOIN %s.job_variants jv_%s ON jobs.prowjob_job_name = jv_%s.job_name AND jv_%s.variant_name = '%s'\n", + p.client.Dataset, cleanV, cleanV, cleanV, v) + } + + variantFilters := "" + var params []bigquery.QueryParameter + + includeVariants := reqOptions.VariantOption.IncludeVariants + if includeVariants == nil { + includeVariants = map[string][]string{} + } + for _, group := range sortedKeys(includeVariants) { + cleanGroup := param.Cleanse(group) + paramName := fmt.Sprintf("variantGroup_%s", cleanGroup) + variantFilters += fmt.Sprintf(" AND (jv_%s.variant_value IN UNNEST(@%s))", cleanGroup, paramName) + params = append(params, bigquery.QueryParameter{ + Name: paramName, + Value: includeVariants[group], + }) + } + + queryString := fmt.Sprintf(` + SELECT + jobs.prowjob_job_name AS job_name, + COUNT(DISTINCT jobs.prowjob_build_id) AS total_runs, + COUNTIF(jobs.prowjob_state = 'success') AS successful_runs + FROM %s.jobs jobs + %s + WHERE jobs.prowjob_start >= DATETIME(@From) + AND jobs.prowjob_start < DATETIME(@To) + AND jv_Release.variant_value = @Release + AND (jobs.prowjob_job_name LIKE 'periodic-%%' OR jobs.prowjob_job_name LIKE 'release-%%' OR jobs.prowjob_job_name LIKE 'aggregator-%%') + %s + GROUP BY jobs.prowjob_job_name + ORDER BY jobs.prowjob_job_name + `, p.client.Dataset, joinVariants, variantFilters) + + params = append(params, + bigquery.QueryParameter{Name: "From", Value: start}, + bigquery.QueryParameter{Name: "To", Value: end}, + bigquery.QueryParameter{Name: "Release", Value: release}, + ) + + q := p.client.Query(ctx, bqlabel.CRViewJobs, queryString) + q.Parameters = params + + it, err := q.Read(ctx) + if err != nil { + return nil, fmt.Errorf("error executing view jobs query: %w", err) + } + + type jobRunRow struct { + JobName string `bigquery:"job_name"` + TotalRuns int `bigquery:"total_runs"` + Successful int `bigquery:"successful_runs"` + } + + results := map[string]dataprovider.JobRunStats{} + for { + var row jobRunRow + err := it.Next(&row) + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("error reading view jobs row: %w", err) + } + passRate := 0.0 + if row.TotalRuns > 0 { + passRate = float64(row.Successful) / float64(row.TotalRuns) * 100 + } + results[row.JobName] = dataprovider.JobRunStats{ + JobName: row.JobName, + TotalRuns: row.TotalRuns, + SuccessfulRuns: row.Successful, + PassRate: passRate, + } + } + return results, nil +} + +func (p *BigQueryProvider) QueryJobVariantValues(ctx context.Context, jobNames []string, + variantKeys []string) (map[string]map[string]string, error) { + if len(jobNames) == 0 { + return map[string]map[string]string{}, nil + } + + queryString := fmt.Sprintf(` + SELECT job_name, variant_name, variant_value + FROM %s.job_variants + WHERE job_name IN UNNEST(@JobNames) + AND variant_name IN UNNEST(@VariantNames) + `, p.client.Dataset) + + q := p.client.Query(ctx, bqlabel.CRViewJobs, queryString) + q.Parameters = []bigquery.QueryParameter{ + {Name: "JobNames", Value: jobNames}, + {Name: "VariantNames", Value: variantKeys}, + } + + it, err := q.Read(ctx) + if err != nil { + return nil, fmt.Errorf("error querying job variant values: %w", err) + } + + type variantRow struct { + JobName string `bigquery:"job_name"` + VariantName string `bigquery:"variant_name"` + VariantValue string `bigquery:"variant_value"` + } + + results := map[string]map[string]string{} + for { + var row variantRow + err := it.Next(&row) + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("error reading job variant row: %w", err) + } + if results[row.JobName] == nil { + results[row.JobName] = map[string]string{} + } + results[row.JobName][row.VariantName] = row.VariantValue + } + return results, nil +} + +func (p *BigQueryProvider) LookupJobVariants(ctx context.Context, jobName string) (map[string]string, error) { + queryString := fmt.Sprintf(` + SELECT variant_name, variant_value + FROM %s.job_variants + WHERE job_name = @JobName + `, p.client.Dataset) + + q := p.client.Query(ctx, bqlabel.CRViewJobs, queryString) + q.Parameters = []bigquery.QueryParameter{ + {Name: "JobName", Value: jobName}, + } + + it, err := q.Read(ctx) + if err != nil { + return nil, fmt.Errorf("error querying job variants: %w", err) + } + + type row struct { + VariantName string `bigquery:"variant_name"` + VariantValue string `bigquery:"variant_value"` + } + + variants := map[string]string{} + for { + var r row + err := it.Next(&r) + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("error reading variant row: %w", err) + } + variants[r.VariantName] = r.VariantValue + } + return variants, nil +} + +// --- Helpers --- + +func getSingleColumnResultToSlice(ctx context.Context, q *bigquery.Query) ([]string, error) { + names := []string{} + it, err := q.Read(ctx) + if err != nil { + log.WithError(err).Error("error querying from bigquery") + return names, err + } + for { + row := struct{ Name string }{} + err := it.Next(&row) + if err == iterator.Done { + break + } + if err != nil { + log.WithError(err).Error("error parsing row from bigquery") + return names, err + } + names = append(names, row.Name) + } + return names, nil +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/pkg/api/componentreadiness/dataprovider/interface.go b/pkg/api/componentreadiness/dataprovider/interface.go new file mode 100644 index 0000000000..b207c8eb32 --- /dev/null +++ b/pkg/api/componentreadiness/dataprovider/interface.go @@ -0,0 +1,90 @@ +package dataprovider + +import ( + "context" + "time" + + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" +) + +// TestStatusQuerier fetches aggregated test pass/fail counts. +type TestStatusQuerier interface { + // QueryBaseTestStatus returns test status for the basis release. + QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string]crstatus.TestStatus, []error) + + // QuerySampleTestStatus returns test status for the sample release. + // includeVariants may differ from reqOptions for junit table overrides. + QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time, + dataSource string) (map[string]crstatus.TestStatus, []error) +} + +// TestDetailsQuerier fetches per-job-run test breakdowns used for test details reports. +type TestDetailsQuerier interface { + QueryBaseJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string][]crstatus.TestJobRunRows, []error) + + QuerySampleJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time, + dataSource string) (map[string][]crstatus.TestJobRunRows, []error) +} + +// MetadataQuerier fetches reference data used to configure and parameterize reports. +type MetadataQuerier interface { + // QueryJobVariants returns all variant names and their possible values. + QueryJobVariants(ctx context.Context) (crtest.JobVariants, []error) + + // QueryReleaseDates returns the time ranges for each known release. + QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) + + // QueryReleases returns known release configurations. + QueryReleases(ctx context.Context) ([]v1.Release, error) + + // QueryUniqueVariantValues returns distinct values for a variant column + // from the past 60 days. + QueryUniqueVariantValues(ctx context.Context, field string, nested bool) ([]string, error) +} + +// JobQuerier fetches job-level data for the view-jobs and diagnose endpoints. +type JobQuerier interface { + // QueryJobRuns returns pass/fail statistics per job for a release in a time window. + QueryJobRuns(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + release string, start, end time.Time) (map[string]JobRunStats, error) + + // QueryJobVariantValues returns variant key/value pairs for the given jobs. + QueryJobVariantValues(ctx context.Context, jobNames []string, + variantKeys []string) (map[string]map[string]string, error) + + // LookupJobVariants returns all variant key/value pairs for a single job. + LookupJobVariants(ctx context.Context, jobName string) (map[string]string, error) +} + +// DataProvider combines all query capabilities needed by Component Readiness. +type DataProvider interface { + TestStatusQuerier + TestDetailsQuerier + MetadataQuerier + JobQuerier + + // Cache returns the cache implementation for storing/retrieving computed results. + Cache() cache.Cache +} + +// JobRunStats contains pass/fail statistics for a single concrete job name. +// Defined here so both the interface and implementations share the same type. +type JobRunStats struct { + JobName string `json:"job_name"` + TotalRuns int `json:"total_runs"` + SuccessfulRuns int `json:"successful_runs"` + PassRate float64 `json:"pass_rate"` +} diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go new file mode 100644 index 0000000000..a2c84d318c --- /dev/null +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -0,0 +1,790 @@ +package postgres + +import ( + "context" + "fmt" + "math/big" + "slices" + "sort" + "strings" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" + "github.com/openshift/sippy/pkg/api/componentreadiness/utils" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db" +) + +var _ dataprovider.DataProvider = &PostgresProvider{} + +// PostgresProvider implements dataprovider.DataProvider using PostgreSQL. +// Designed for local development and testing — not optimized for production scale. +type PostgresProvider struct { + dbc *db.DB + cache cache.Cache +} + +func NewPostgresProvider(dbc *db.DB, c cache.Cache) *PostgresProvider { + if c == nil { + c = &noOpCache{} + } + return &PostgresProvider{dbc: dbc, cache: c} +} + +// noOpCache never stores or returns data; no Redis needed for local dev. +type noOpCache struct{} + +func (n *noOpCache) Get(_ context.Context, _ string, _ time.Duration) ([]byte, error) { + return nil, fmt.Errorf("cache miss") +} +func (n *noOpCache) Set(_ context.Context, _ string, _ []byte, _ time.Duration) error { return nil } + +func (p *PostgresProvider) Cache() cache.Cache { + return p.cache +} + +// --- Variant helpers --- + +// parseVariants splits a pq.StringArray like ["Platform:aws", "Upgrade:none"] into a map. +func parseVariants(variants pq.StringArray) map[string]string { + result := make(map[string]string, len(variants)) + for _, v := range variants { + if k, val, ok := strings.Cut(v, ":"); ok { + result[k] = val + } + } + return result +} + +// variantMapToSlice converts a map to sorted "Key:Value" strings. +func variantMapToSlice(m map[string]string) []string { + result := make([]string, 0, len(m)) + for k, v := range m { + result = append(result, k+":"+v) + } + sort.Strings(result) + return result +} + +// filterByDBGroupBy returns a copy of the variant map keeping only keys in dbGroupBy. +func filterByDBGroupBy(variants map[string]string, dbGroupBy map[string]bool) map[string]string { + filtered := make(map[string]string, len(dbGroupBy)) + for k, v := range variants { + if dbGroupBy[k] { + filtered[k] = v + } + } + return filtered +} + +// matchesIncludeVariants checks if a variant map passes the include filter. +func matchesIncludeVariants(variants map[string]string, includeVariants map[string][]string) bool { + for key, allowed := range includeVariants { + val, exists := variants[key] + if !exists { + return false + } + if !slices.Contains(allowed, val) { + return false + } + } + return true +} + +// --- MetadataQuerier --- + +func (p *PostgresProvider) QueryJobVariants(_ context.Context) (crtest.JobVariants, []error) { + variants := crtest.JobVariants{Variants: map[string][]string{}} + + var pairs []string + err := p.dbc.DB.Raw(`SELECT DISTINCT unnest(variants) AS pair FROM prow_jobs WHERE deleted_at IS NULL`). + Pluck("pair", &pairs).Error + if err != nil { + return variants, []error{fmt.Errorf("querying job variants: %w", err)} + } + + grouped := map[string]map[string]bool{} + for _, pair := range pairs { + k, v, ok := strings.Cut(pair, ":") + if !ok { + continue + } + if grouped[k] == nil { + grouped[k] = map[string]bool{} + } + grouped[k][v] = true + } + + for k, vals := range grouped { + sorted := make([]string, 0, len(vals)) + for v := range vals { + sorted = append(sorted, v) + } + sort.Strings(sorted) + variants.Variants[k] = sorted + } + return variants, nil +} + +// releaseMetadata holds hardcoded release info for known releases. +// This avoids needing a releases table — we derive release names from prow_jobs +// and fill in metadata from this map. +var releaseMetadata = map[string]struct { + previousRelease string + gaOffsetDays int // 0 = no GA date (in development) +}{ + "4.17": {previousRelease: "4.16", gaOffsetDays: -540}, + "4.18": {previousRelease: "4.17", gaOffsetDays: -395}, + "4.19": {previousRelease: "4.18", gaOffsetDays: -289}, + "4.20": {previousRelease: "4.19", gaOffsetDays: -163}, + "4.21": {previousRelease: "4.20", gaOffsetDays: -58}, + "4.22": {previousRelease: "4.21"}, + "5.0": {previousRelease: "4.22"}, +} + +func (p *PostgresProvider) QueryReleases(_ context.Context) ([]v1.Release, error) { + var releaseNames []string + err := p.dbc.DB.Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`). + Pluck("release", &releaseNames).Error + if err != nil { + return nil, fmt.Errorf("querying releases: %w", err) + } + + caps := map[v1.ReleaseCapability]bool{ + v1.ComponentReadinessCap: true, + v1.FeatureGatesCap: true, + v1.MetricsCap: true, + v1.PayloadTagsCap: true, + v1.SippyClassicCap: true, + } + + now := time.Now().UTC() + var releases []v1.Release + for _, name := range releaseNames { + rel := v1.Release{ + Release: name, + Capabilities: caps, + } + if meta, ok := releaseMetadata[name]; ok { + rel.PreviousRelease = meta.previousRelease + if meta.gaOffsetDays != 0 { + ga := now.AddDate(0, 0, meta.gaOffsetDays) + rel.GADate = &ga + } + } + releases = append(releases, rel) + } + return releases, nil +} + +func (p *PostgresProvider) QueryReleaseDates(_ context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { + // Derive time ranges from actual data in the DB rather than hardcoded GA dates. + // This ensures fallback queries find data where it actually exists. + type releaseRange struct { + Release string + Start time.Time + End time.Time + } + var ranges []releaseRange + err := p.dbc.DB.Raw(` + SELECT pj.release, + MIN(pjr.timestamp) AS start, + MAX(pjr.timestamp) AS end + FROM prow_job_runs pjr + JOIN prow_jobs pj ON pj.id = pjr.prow_job_id + WHERE pj.deleted_at IS NULL AND pjr.deleted_at IS NULL + GROUP BY pj.release + ORDER BY pj.release DESC + `).Scan(&ranges).Error + if err != nil { + return nil, []error{fmt.Errorf("querying release dates: %w", err)} + } + + var dates []crtest.ReleaseTimeRange + for _, r := range ranges { + start := r.Start + end := r.End + dates = append(dates, crtest.ReleaseTimeRange{ + Release: r.Release, + Start: &start, + End: &end, + }) + } + return dates, nil +} + +func (p *PostgresProvider) QueryUniqueVariantValues(_ context.Context, field string, nested bool) ([]string, error) { + if nested { + // Return all variant key names + var pairs []string + err := p.dbc.DB.Raw(` + SELECT DISTINCT unnest(variants) AS pair FROM prow_jobs + WHERE deleted_at IS NULL + `).Pluck("pair", &pairs).Error + if err != nil { + return nil, err + } + keys := map[string]bool{} + for _, pair := range pairs { + if k, _, ok := strings.Cut(pair, ":"); ok { + keys[k] = true + } + } + result := make([]string, 0, len(keys)) + for k := range keys { + result = append(result, k) + } + sort.Strings(result) + return result, nil + } + + // Map BQ column names to variant key names + fieldMap := map[string]string{ + "platform": "Platform", + "network": "Network", + "arch": "Architecture", + "upgrade": "Upgrade", + } + variantKey, ok := fieldMap[field] + if !ok { + return []string{}, nil + } + + var pairs []string + err := p.dbc.DB.Raw(` + SELECT DISTINCT unnest(variants) AS pair FROM prow_jobs + WHERE deleted_at IS NULL + `).Pluck("pair", &pairs).Error + if err != nil { + return nil, err + } + + vals := map[string]bool{} + for _, pair := range pairs { + if k, v, ok := strings.Cut(pair, ":"); ok && k == variantKey { + vals[v] = true + } + } + result := make([]string, 0, len(vals)) + for v := range vals { + result = append(result, v) + } + sort.Strings(result) + return result, nil +} + +// --- TestStatusQuerier --- + +// testStatusRow is the result of the aggregation query. +type testStatusRow struct { + TestID string `gorm:"column:test_id"` + TestName string `gorm:"column:test_name"` + TestSuite string `gorm:"column:test_suite"` + Component string `gorm:"column:component"` + Capabilities pq.StringArray `gorm:"column:capabilities;type:text[]"` + ProwJobID uint `gorm:"column:prow_job_id"` + TotalCount int `gorm:"column:total_count"` + SuccessCount int `gorm:"column:success_count"` + FlakeCount int `gorm:"column:flake_count"` + LastFailure *time.Time `gorm:"column:last_failure"` +} + +const testStatusQuery = ` +WITH deduped AS ( + SELECT DISTINCT ON (pjrt.prow_job_run_id, pjrt.test_id, pjrt.suite_id) + pjrt.test_id, pjrt.suite_id, pjrt.status, + pjr.timestamp, pj.id AS prow_job_id + FROM prow_job_run_tests pjrt + JOIN prow_job_runs pjr ON pjr.id = pjrt.prow_job_run_id + JOIN prow_jobs pj ON pj.id = pjr.prow_job_id + WHERE pj.release = ? + AND pjr.timestamp >= ? AND pjr.timestamp < ? + AND pjrt.deleted_at IS NULL AND pjr.deleted_at IS NULL AND pj.deleted_at IS NULL + AND (pjr.labels IS NULL OR NOT pjr.labels @> ARRAY['InfraFailure']) + ORDER BY pjrt.prow_job_run_id, pjrt.test_id, pjrt.suite_id, + CASE WHEN pjrt.status = 13 THEN 0 WHEN pjrt.status = 1 THEN 1 ELSE 2 END +) +SELECT + tow.unique_id AS test_id, + t.name AS test_name, + COALESCE(s.name, '') AS test_suite, + tow.component, + tow.capabilities, + d.prow_job_id, + COUNT(*) AS total_count, + SUM(CASE WHEN d.status IN (1, 13) THEN 1 ELSE 0 END) AS success_count, + SUM(CASE WHEN d.status = 13 THEN 1 ELSE 0 END) AS flake_count, + MAX(CASE WHEN d.status NOT IN (1, 13) THEN d.timestamp ELSE NULL END) AS last_failure +FROM deduped d +JOIN tests t ON t.id = d.test_id +JOIN test_ownerships tow ON tow.test_id = d.test_id + AND (tow.suite_id = d.suite_id OR (tow.suite_id IS NULL AND d.suite_id IS NULL)) +LEFT JOIN suites s ON s.id = d.suite_id +WHERE tow.staff_approved_obsolete = false +GROUP BY tow.unique_id, t.name, s.name, tow.component, tow.capabilities, d.prow_job_id +` + +func (p *PostgresProvider) queryTestStatus(release string, start, end time.Time, + _ crtest.JobVariants, includeVariants map[string][]string, + dbGroupBy map[string]bool) (map[string]crstatus.TestStatus, []error) { + + var rows []testStatusRow + if err := p.dbc.DB.Raw(testStatusQuery, release, start, end).Scan(&rows).Error; err != nil { + return nil, []error{fmt.Errorf("querying test status: %w", err)} + } + + // Batch-fetch all ProwJob variants we need + jobVariantMap := p.fetchJobVariants(rows) + + result := map[string]crstatus.TestStatus{} + for _, row := range rows { + variants, ok := jobVariantMap[row.ProwJobID] + if !ok { + continue + } + + if !matchesIncludeVariants(variants, includeVariants) { + continue + } + + filtered := filterByDBGroupBy(variants, dbGroupBy) + key := crtest.KeyWithVariants{ + TestID: row.TestID, + Variants: filtered, + } + keyStr := key.KeyOrDie() + + existing, exists := result[keyStr] + if exists { + // Merge counts for same test+variant combo from different job runs + existing.Count.TotalCount += row.TotalCount + existing.Count.SuccessCount += row.SuccessCount + existing.Count.FlakeCount += row.FlakeCount + if row.LastFailure != nil && (existing.LastFailure.IsZero() || row.LastFailure.After(existing.LastFailure)) { + existing.LastFailure = *row.LastFailure + } + result[keyStr] = existing + } else { + ts := crstatus.TestStatus{ + TestName: row.TestName, + TestSuite: row.TestSuite, + Component: row.Component, + Capabilities: row.Capabilities, + Variants: variantMapToSlice(filtered), + Count: crtest.Count{ + TotalCount: row.TotalCount, + SuccessCount: row.SuccessCount, + FlakeCount: row.FlakeCount, + }, + } + if row.LastFailure != nil { + ts.LastFailure = *row.LastFailure + } + result[keyStr] = ts + } + } + + return result, nil +} + +// fetchJobVariants loads and caches ProwJob variant maps for the given rows. +func (p *PostgresProvider) fetchJobVariants(rows []testStatusRow) map[uint]map[string]string { + jobIDs := map[uint]bool{} + for _, r := range rows { + jobIDs[r.ProwJobID] = true + } + + ids := make([]uint, 0, len(jobIDs)) + for id := range jobIDs { + ids = append(ids, id) + } + + type jobRow struct { + ID uint `gorm:"column:id"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + + var jobRows []jobRow + if err := p.dbc.DB.Raw(`SELECT id, variants FROM prow_jobs WHERE id IN (?)`, ids).Scan(&jobRows).Error; err != nil { + log.WithError(err).Error("error fetching job variants") + return map[uint]map[string]string{} + } + + result := make(map[uint]map[string]string, len(jobRows)) + for _, jr := range jobRows { + result[jr.ID] = parseVariants(jr.Variants) + } + return result +} + +func (p *PostgresProvider) QueryBaseTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string]crstatus.TestStatus, []error) { + + dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) + for _, k := range reqOptions.VariantOption.DBGroupBy.List() { + dbGroupBy[k] = true + } + + includeVariants := reqOptions.VariantOption.IncludeVariants + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + return p.queryTestStatus( + reqOptions.BaseRelease.Name, + reqOptions.BaseRelease.Start, + reqOptions.BaseRelease.End, + allJobVariants, + includeVariants, + dbGroupBy, + ) +} + +func (p *PostgresProvider) QuerySampleTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time, + _ string) (map[string]crstatus.TestStatus, []error) { + + dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) + for _, k := range reqOptions.VariantOption.DBGroupBy.List() { + dbGroupBy[k] = true + } + + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + return p.queryTestStatus( + reqOptions.SampleRelease.Name, + start, end, + allJobVariants, + includeVariants, + dbGroupBy, + ) +} + +// --- TestDetailsQuerier --- + +type testDetailRow struct { + TestID string `gorm:"column:test_id"` + TestName string `gorm:"column:test_name"` + ProwJobName string `gorm:"column:prowjob_name"` + ProwJobRunID string `gorm:"column:prowjob_run_id"` + ProwJobURL string `gorm:"column:prowjob_url"` + ProwJobStart time.Time `gorm:"column:prowjob_start"` + ProwJobID uint `gorm:"column:prow_job_id"` + Status int `gorm:"column:status"` + JiraComponent string `gorm:"column:jira_component"` + JiraComponentID *uint `gorm:"column:jira_component_id"` + Capabilities pq.StringArray `gorm:"column:capabilities;type:text[]"` +} + +const testDetailQuery = ` +SELECT + tow.unique_id AS test_id, + t.name AS test_name, + pj.name AS prowjob_name, + CAST(pjr.id AS TEXT) AS prowjob_run_id, + COALESCE(pjr.url, '') AS prowjob_url, + pjr.timestamp AS prowjob_start, + pj.id AS prow_job_id, + pjrt.status, + COALESCE(tow.jira_component, '') AS jira_component, + tow.jira_component_id, + tow.capabilities +FROM prow_job_run_tests pjrt +JOIN prow_job_runs pjr ON pjr.id = pjrt.prow_job_run_id +JOIN prow_jobs pj ON pj.id = pjr.prow_job_id +JOIN tests t ON t.id = pjrt.test_id +JOIN test_ownerships tow ON tow.test_id = pjrt.test_id + AND (tow.suite_id = pjrt.suite_id OR (tow.suite_id IS NULL AND pjrt.suite_id IS NULL)) +WHERE pj.release = ? + AND pjr.timestamp >= ? AND pjr.timestamp < ? + AND pjrt.deleted_at IS NULL AND pjr.deleted_at IS NULL AND pj.deleted_at IS NULL + AND tow.staff_approved_obsolete = false + AND (pjr.labels IS NULL OR NOT pjr.labels @> ARRAY['InfraFailure']) +ORDER BY pjr.timestamp +` + +func (p *PostgresProvider) queryTestDetails(release string, start, end time.Time, + reqOptions reqopts.RequestOptions, _ crtest.JobVariants, + includeVariants map[string][]string) (map[string][]crstatus.TestJobRunRows, []error) { + + var rows []testDetailRow + if err := p.dbc.DB.Raw(testDetailQuery, release, start, end).Scan(&rows).Error; err != nil { + return nil, []error{fmt.Errorf("querying test details: %w", err)} + } + + dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) + for _, k := range reqOptions.VariantOption.DBGroupBy.List() { + dbGroupBy[k] = true + } + + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + // Batch-fetch job variants + jobIDs := map[uint]bool{} + for _, r := range rows { + jobIDs[r.ProwJobID] = true + } + ids := make([]uint, 0, len(jobIDs)) + for id := range jobIDs { + ids = append(ids, id) + } + type jobRow struct { + ID uint `gorm:"column:id"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + var jobRows []jobRow + if len(ids) > 0 { + if err := p.dbc.DB.Raw(`SELECT id, variants FROM prow_jobs WHERE id IN (?)`, ids).Scan(&jobRows).Error; err != nil { + return nil, []error{fmt.Errorf("fetching job variants: %w", err)} + } + } + jobVariantMap := make(map[uint]map[string]string, len(jobRows)) + for _, jr := range jobRows { + jobVariantMap[jr.ID] = parseVariants(jr.Variants) + } + + // Filter test IDs if specified + // Build test ID filter and per-test requested variant filters + testIDFilter := map[string]bool{} + requestedVariantsByTestID := map[string]map[string]string{} + for _, tid := range reqOptions.TestIDOptions { + testIDFilter[tid.TestID] = true + if len(tid.RequestedVariants) > 0 { + requestedVariantsByTestID[tid.TestID] = tid.RequestedVariants + } + } + + result := map[string][]crstatus.TestJobRunRows{} + for _, row := range rows { + if len(testIDFilter) > 0 && !testIDFilter[row.TestID] { + continue + } + + variants, ok := jobVariantMap[row.ProwJobID] + if !ok { + continue + } + if !matchesIncludeVariants(variants, includeVariants) { + continue + } + + // Filter by requested variants (exact match for specific test+variant combo) + if rv, ok := requestedVariantsByTestID[row.TestID]; ok { + match := true + for k, v := range rv { + if variants[k] != v { + match = false + break + } + } + if !match { + continue + } + } + + filtered := filterByDBGroupBy(variants, dbGroupBy) + key := crtest.KeyWithVariants{ + TestID: row.TestID, + Variants: filtered, + } + + successCount := 0 + flakeCount := 0 + if row.Status == 1 || row.Status == 13 { + successCount = 1 + } + if row.Status == 13 { + flakeCount = 1 + } + + var jiraComponentID *big.Rat + if row.JiraComponentID != nil { + jiraComponentID = new(big.Rat).SetUint64(uint64(*row.JiraComponentID)) + } + + entry := crstatus.TestJobRunRows{ + TestKey: key, + TestKeyStr: key.KeyOrDie(), + TestName: row.TestName, + ProwJob: utils.NormalizeProwJobName(row.ProwJobName), + ProwJobRunID: row.ProwJobRunID, + ProwJobURL: row.ProwJobURL, + StartTime: row.ProwJobStart, + Count: crtest.Count{TotalCount: 1, SuccessCount: successCount, FlakeCount: flakeCount}, + JiraComponent: row.JiraComponent, + JiraComponentID: jiraComponentID, + } + + normalizedName := utils.NormalizeProwJobName(row.ProwJobName) + result[normalizedName] = append(result[normalizedName], entry) + } + + return result, nil +} + +func (p *PostgresProvider) QueryBaseJobRunTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string][]crstatus.TestJobRunRows, []error) { + + return p.queryTestDetails( + reqOptions.BaseRelease.Name, + reqOptions.BaseRelease.Start, reqOptions.BaseRelease.End, + reqOptions, allJobVariants, reqOptions.VariantOption.IncludeVariants, + ) +} + +func (p *PostgresProvider) QuerySampleJobRunTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time, + _ string) (map[string][]crstatus.TestJobRunRows, []error) { + + return p.queryTestDetails( + reqOptions.SampleRelease.Name, + start, end, + reqOptions, allJobVariants, includeVariants, + ) +} + +// --- JobQuerier --- + +func (p *PostgresProvider) QueryJobRuns(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + release string, start, end time.Time) (map[string]dataprovider.JobRunStats, error) { + + type jobRunRow struct { + JobName string `gorm:"column:job_name"` + TotalRuns int `gorm:"column:total_runs"` + Successful int `gorm:"column:successful_runs"` + } + + var rows []jobRunRow + err := p.dbc.DB.Raw(` + SELECT + pj.name AS job_name, + COUNT(DISTINCT pjr.id) AS total_runs, + COUNT(DISTINCT CASE WHEN pjr.succeeded THEN pjr.id END) AS successful_runs + FROM prow_jobs pj + JOIN prow_job_runs pjr ON pjr.prow_job_id = pj.id + WHERE pj.release = ? + AND pjr.timestamp >= ? AND pjr.timestamp < ? + AND pj.deleted_at IS NULL AND pjr.deleted_at IS NULL + GROUP BY pj.name + ORDER BY pj.name + `, release, start, end).Scan(&rows).Error + if err != nil { + return nil, fmt.Errorf("querying job runs: %w", err) + } + + // Apply variant filtering in Go + includeVariants := reqOptions.VariantOption.IncludeVariants + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + // Fetch variants for all jobs + jobNames := make([]string, 0, len(rows)) + for _, r := range rows { + jobNames = append(jobNames, r.JobName) + } + jobVariantMap := map[string]map[string]string{} + if len(jobNames) > 0 { + type jvRow struct { + Name string `gorm:"column:name"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + var jvRows []jvRow + if err := p.dbc.DB.Raw(`SELECT name, variants FROM prow_jobs WHERE name IN (?) AND deleted_at IS NULL`, jobNames).Scan(&jvRows).Error; err != nil { + return nil, fmt.Errorf("fetching job variants: %w", err) + } + for _, jr := range jvRows { + jobVariantMap[jr.Name] = parseVariants(jr.Variants) + } + } + + results := map[string]dataprovider.JobRunStats{} + for _, row := range rows { + if variants, ok := jobVariantMap[row.JobName]; ok { + if !matchesIncludeVariants(variants, includeVariants) { + continue + } + } + passRate := 0.0 + if row.TotalRuns > 0 { + passRate = float64(row.Successful) / float64(row.TotalRuns) * 100 + } + results[row.JobName] = dataprovider.JobRunStats{ + JobName: row.JobName, + TotalRuns: row.TotalRuns, + SuccessfulRuns: row.Successful, + PassRate: passRate, + } + } + + return results, nil +} + +func (p *PostgresProvider) QueryJobVariantValues(_ context.Context, jobNames []string, + variantKeys []string) (map[string]map[string]string, error) { + + if len(jobNames) == 0 { + return map[string]map[string]string{}, nil + } + + type jvRow struct { + Name string `gorm:"column:name"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + + var rows []jvRow + if err := p.dbc.DB.Raw(`SELECT name, variants FROM prow_jobs WHERE name IN (?) AND deleted_at IS NULL`, jobNames).Scan(&rows).Error; err != nil { + return nil, fmt.Errorf("querying job variant values: %w", err) + } + + keyFilter := map[string]bool{} + for _, k := range variantKeys { + keyFilter[k] = true + } + + results := map[string]map[string]string{} + for _, row := range rows { + parsed := parseVariants(row.Variants) + if len(keyFilter) > 0 { + filtered := map[string]string{} + for k, v := range parsed { + if keyFilter[k] { + filtered[k] = v + } + } + results[row.Name] = filtered + } else { + results[row.Name] = parsed + } + } + return results, nil +} + +func (p *PostgresProvider) LookupJobVariants(_ context.Context, jobName string) (map[string]string, error) { + type jvRow struct { + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + + var row jvRow + err := p.dbc.DB.Raw(`SELECT variants FROM prow_jobs WHERE name = ? AND deleted_at IS NULL LIMIT 1`, jobName).Scan(&row).Error + if err != nil { + return nil, fmt.Errorf("looking up job variants: %w", err) + } + return parseVariants(row.Variants), nil +} diff --git a/pkg/api/componentreadiness/middleware/interface.go b/pkg/api/componentreadiness/middleware/interface.go index 32ab5840ca..db846a7c8d 100644 --- a/pkg/api/componentreadiness/middleware/interface.go +++ b/pkg/api/componentreadiness/middleware/interface.go @@ -4,7 +4,7 @@ import ( "context" "sync" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" ) @@ -17,7 +17,7 @@ type Middleware interface { // Base and sample status can be submitted using the provided channels for a map of ALL test keys // (ID plus variant info serialized) to TestStatus. Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, - baseStatusCh, sampleStatusCh chan map[string]bq.TestStatus, errCh chan error) + baseStatusCh, sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error) // QueryTestDetails phase allow middleware to load data that will later be used. QueryTestDetails(ctx context.Context, wg *sync.WaitGroup, errCh chan error, allJobVariants crtest.JobVariants) @@ -36,5 +36,5 @@ type Middleware interface { // PreTestDetailsAnalysis gives middleware the opportunity to adjust inputs to the report status // prior to analysis. - PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *bq.TestJobRunStatuses) error + PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *crstatus.TestJobRunStatuses) error } diff --git a/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go b/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go index 825294f583..84bb66d4cd 100644 --- a/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go +++ b/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go @@ -6,7 +6,7 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/middleware" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" @@ -32,7 +32,7 @@ type LinkInjector struct { baseURL string } -func (l *LinkInjector) Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, baseStatusCh, sampleStatusCh chan map[string]bq.TestStatus, errCh chan error) { +func (l *LinkInjector) Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, baseStatusCh, sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error) { // unused } @@ -92,7 +92,7 @@ func (l *LinkInjector) PostAnalysis(testKey crtest.Identification, testStats *te return nil } -func (l *LinkInjector) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *bq.TestJobRunStatuses) error { +func (l *LinkInjector) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *crstatus.TestJobRunStatuses) error { // unused return nil } diff --git a/pkg/api/componentreadiness/middleware/list.go b/pkg/api/componentreadiness/middleware/list.go index 3e52b46c64..5205677e33 100644 --- a/pkg/api/componentreadiness/middleware/list.go +++ b/pkg/api/componentreadiness/middleware/list.go @@ -4,14 +4,14 @@ import ( "context" "sync" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" ) type List []Middleware -func (l List) Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, baseStatusCh, sampleStatusCh chan map[string]bq.TestStatus, errCh chan error) { +func (l List) Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, baseStatusCh, sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error) { // Invoke the Query phase for each middleware configured: for _, mw := range l { mw.Query(ctx, wg, allJobVariants, baseStatusCh, sampleStatusCh, errCh) @@ -43,7 +43,7 @@ func (l List) PostAnalysis(testKey crtest.Identification, testStats *testdetails return nil } -func (l List) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *bq.TestJobRunStatuses) error { +func (l List) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *crstatus.TestJobRunStatuses) error { for _, mw := range l { if err := mw.PreTestDetailsAnalysis(testKey, status); err != nil { return err diff --git a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go index 5e2ff67cda..7e79cdb435 100644 --- a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go +++ b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go @@ -7,7 +7,7 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/middleware" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" @@ -41,7 +41,7 @@ type RegressionAllowances struct { } func (r *RegressionAllowances) Query(_ context.Context, _ *sync.WaitGroup, _ crtest.JobVariants, - _, _ chan map[string]bq.TestStatus, _ chan error) { + _, _ chan map[string]crstatus.TestStatus, _ chan error) { // unused } @@ -151,6 +151,6 @@ func (r *RegressionAllowances) adjustAnalysisParameters(testStats *testdetails.T func (r *RegressionAllowances) QueryTestDetails(ctx context.Context, wg *sync.WaitGroup, errCh chan error, allJobVariants crtest.JobVariants) { } -func (r *RegressionAllowances) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *bq.TestJobRunStatuses) error { +func (r *RegressionAllowances) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *crstatus.TestJobRunStatuses) error { return nil } diff --git a/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go b/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go index 8a99fb4d95..c9fc9b9e65 100644 --- a/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go +++ b/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go @@ -8,7 +8,7 @@ import ( "time" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" @@ -54,7 +54,7 @@ type RegressionTracker struct { hasLoadedRegressions bool } -func (r *RegressionTracker) Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, baseStatusCh, sampleStatusCh chan map[string]bq.TestStatus, errCh chan error) { +func (r *RegressionTracker) Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, baseStatusCh, sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error) { err := r.ensureRegressionsLoaded() if err != nil { errCh <- err @@ -220,6 +220,6 @@ func FindOpenRegression(sampleRelease, testID string, return nil } -func (r *RegressionTracker) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *bq.TestJobRunStatuses) error { +func (r *RegressionTracker) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *crstatus.TestJobRunStatuses) error { return nil } diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go index cb603aba60..51dbbe96fd 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go @@ -7,21 +7,19 @@ import ( "sync" "time" - "cloud.google.com/go/bigquery" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware" - "github.com/openshift/sippy/pkg/api/componentreadiness/query" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" + apiCache "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - "github.com/openshift/sippy/pkg/bigquery/bqlabel" "github.com/openshift/sippy/pkg/util/sets" log "github.com/sirupsen/logrus" "github.com/openshift/sippy/pkg/api" - bqcachedclient "github.com/openshift/sippy/pkg/bigquery" ) const ( @@ -32,12 +30,12 @@ const ( var _ middleware.Middleware = &ReleaseFallback{} func NewReleaseFallbackMiddleware( - client *bqcachedclient.Client, + provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, releaseConfigs []v1.Release, ) *ReleaseFallback { return &ReleaseFallback{ - client: client, + dataProvider: provider, log: log.WithField("middleware", "ReleaseFallback"), reqOptions: reqOptions, releaseConfigs: releaseConfigs, @@ -54,7 +52,7 @@ func NewReleaseFallbackMiddleware( // then replacing any basis test stats with a better releases test stats, when appropriate. // This is done when we have sufficient test coverage, and a better pass rate. type ReleaseFallback struct { - client *bqcachedclient.Client + dataProvider dataprovider.DataProvider cachedFallbackTestStatuses *FallbackReleases log log.FieldLogger reqOptions reqopts.RequestOptions @@ -62,7 +60,7 @@ type ReleaseFallback struct { // baseOverrideStatus maps test key, to job name, to the result rows for that job. // This is used in test details reports, and in the typical API case will only contain one // test ID, but when cache priming for a view, we may have multiple. - baseOverrideStatus map[string]map[string][]bq.TestJobRunRows + baseOverrideStatus map[string]map[string][]crstatus.TestJobRunRows baseOverrideMutex sync.Mutex // Mutex to protect the map releaseConfigs []v1.Release } @@ -72,7 +70,7 @@ func (r *ReleaseFallback) Analyze(testID string, variants map[string]string, rep } func (r *ReleaseFallback) Query(ctx context.Context, wg *sync.WaitGroup, allJobVariants crtest.JobVariants, - _, _ chan map[string]bq.TestStatus, errCh chan error) { + _, _ chan map[string]crstatus.TestStatus, errCh chan error) { wg.Add(1) go func() { defer wg.Done() @@ -119,7 +117,7 @@ func (r *ReleaseFallback) PreAnalysis(testKey crtest.Identification, testStats * var swappedExplanation string for err == nil { var cachedReleaseTestStatuses ReleaseTestMap - var cTestStatus bq.TestStatus + var cTestStatus crstatus.TestStatus ok := false priorRelease, err = utils.PreviousRelease(priorRelease, r.releaseConfigs) // if we fail to determine the previous release then stop @@ -174,10 +172,10 @@ func (r *ReleaseFallback) PostAnalysis(testKey crtest.Identification, testStats func (r *ReleaseFallback) getFallbackBaseQueryStatus(ctx context.Context, allJobVariants crtest.JobVariants, release string, start, end time.Time) []error { - generator := newFallbackTestQueryReleasesGenerator(r.client, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs) + generator := newFallbackTestQueryReleasesGenerator(r.dataProvider, r.reqOptions, allJobVariants, release, start, end, r.releaseConfigs) cachedFallbackTestStatuses, errs := api.GetDataFromCacheOrGenerate[*FallbackReleases]( - ctx, r.client.Cache, r.reqOptions.CacheOption, + ctx, r.dataProvider.Cache(), r.reqOptions.CacheOption, api.NewCacheSpec(generator.getCacheKey(), "FallbackReleases~", &end), generator.getTestFallbackReleases, &FallbackReleases{}) @@ -194,7 +192,7 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro r.log.Infof("Querying fallback override test statuses for %d test ID options", len(r.reqOptions.TestIDOptions)) // Lookup all release dates, we're going to need them - timeRanges, errs := query.GetReleaseDatesFromBigQuery(ctx, r.client, r.reqOptions) + timeRanges, errs := r.dataProvider.QueryReleaseDates(ctx, r.reqOptions) for _, err := range errs { errCh <- err } @@ -218,9 +216,9 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro } releaseToTestIDOptions[testIDOpts.BaseOverrideRelease] = append(releaseToTestIDOptions[testIDOpts.BaseOverrideRelease], testIDOpts) } - r.baseOverrideStatus = map[string]map[string][]bq.TestJobRunRows{} + r.baseOverrideStatus = map[string]map[string][]crstatus.TestJobRunRows{} - // Now we'll do one concurrent bigquery query for each release that has some fallback tests: + // Now we'll do one concurrent query for each release that has some fallback tests: for release, testIDOpts := range releaseToTestIDOptions { r.log.Infof("Querying %d fallback override test statuses for release %s", len(testIDOpts), release) @@ -238,25 +236,13 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro r.log.Infof("Context canceled while fetching base job run test status") return default: - var errs []error - // We cannot inject our fallback data, rather we will query it, store it internally, and apply it during TransformTestDetails - generator := query.NewBaseTestDetailsQueryGenerator( - r.log.WithField("release", release), - r.client, - r.reqOptions, - allJobVariants, - release, - *start, - *end, - testIDOpts, - ) - - jobRunTestStatus, errs := api.GetDataFromCacheOrGenerate[bq.TestJobRunStatuses]( - ctx, - r.client.Cache, r.reqOptions.CacheOption, - api.NewCacheSpec(generator, "BaseJobRunTestStatus~", end), - generator.QueryTestStatus, - bq.TestJobRunStatuses{}) + fallbackReqOpts := r.reqOptions + fallbackReqOpts.BaseRelease.Name = release + fallbackReqOpts.BaseRelease.Start = *start + fallbackReqOpts.BaseRelease.End = *end + fallbackReqOpts.TestIDOptions = testIDOpts + + baseStatus, errs := r.dataProvider.QueryBaseJobRunTestStatus(ctx, fallbackReqOpts, allJobVariants) for _, err := range errs { errCh <- err @@ -270,15 +256,15 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro // Build out a new struct where these are split up by test ID. // split the status on test ID, and pass only that tests data in for reporting: r.baseOverrideMutex.Lock() - for jobName, rows := range jobRunTestStatus.BaseStatus { + for jobName, rows := range baseStatus { for _, row := range rows { testKeyStr := row.TestKeyStr if _, ok := r.baseOverrideStatus[testKeyStr]; !ok { r.log.Infof("added test key: " + testKeyStr) - r.baseOverrideStatus[testKeyStr] = map[string][]bq.TestJobRunRows{} + r.baseOverrideStatus[testKeyStr] = map[string][]crstatus.TestJobRunRows{} } if r.baseOverrideStatus[testKeyStr][jobName] == nil { - r.baseOverrideStatus[testKeyStr][jobName] = []bq.TestJobRunRows{} + r.baseOverrideStatus[testKeyStr][jobName] = []crstatus.TestJobRunRows{} } r.baseOverrideStatus[testKeyStr][jobName] = append(r.baseOverrideStatus[testKeyStr][jobName], row) @@ -294,7 +280,7 @@ func (r *ReleaseFallback) QueryTestDetails(ctx context.Context, wg *sync.WaitGro } -func (r *ReleaseFallback) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *bq.TestJobRunStatuses) error { +func (r *ReleaseFallback) PreTestDetailsAnalysis(testKey crtest.KeyWithVariants, status *crstatus.TestJobRunStatuses) error { // Add our baseOverrideStatus to the report, unfortunate hack we have to live with for now. testKeyStr := testKey.KeyOrDie() if _, ok := r.baseOverrideStatus[testKeyStr]; ok { @@ -310,7 +296,8 @@ func (r *ReleaseFallback) TestDetailsAnalyze(report *testdetails.Report) error { // fallbackTestQueryReleasesGenerator iterates the configured number of past releases, querying base status for // each, which can then be used to return the best basis data from those past releases for comparison. type fallbackTestQueryReleasesGenerator struct { - client *bqcachedclient.Client + dataProvider dataprovider.DataProvider + cacheOption apiCache.RequestOptions allJobVariants crtest.JobVariants BaseRelease string BaseStart time.Time @@ -322,7 +309,7 @@ type fallbackTestQueryReleasesGenerator struct { } func newFallbackTestQueryReleasesGenerator( - client *bqcachedclient.Client, + provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, release string, start, end time.Time, @@ -330,7 +317,8 @@ func newFallbackTestQueryReleasesGenerator( ) fallbackTestQueryReleasesGenerator { generator := fallbackTestQueryReleasesGenerator{ - client: client, + dataProvider: provider, + cacheOption: reqOptions.CacheOption, allJobVariants: allJobVariants, BaseRelease: release, BaseStart: start, @@ -371,7 +359,7 @@ func (f *fallbackTestQueryReleasesGenerator) getCacheKey() fallbackTestQueryRele func (f *fallbackTestQueryReleasesGenerator) getTestFallbackReleases(ctx context.Context) (*FallbackReleases, []error) { wg := sync.WaitGroup{} f.CachedFallbackTestStatuses = newFallbackReleases() - timeRanges, errs := query.GetReleaseDatesFromBigQuery(ctx, f.client, f.ReqOptions) + timeRanges, errs := f.dataProvider.QueryReleaseDates(ctx, f.ReqOptions) if errs != nil { return nil, errs @@ -401,7 +389,7 @@ func (f *fallbackTestQueryReleasesGenerator) getTestFallbackReleases(ctx context log.Infof("Context canceled while fetching fallback base query status") return default: - stats, errs := f.getTestFallbackRelease(ctx, f.client, queryRelease.Release, queryStart, queryEnd) + stats, errs := f.getTestFallbackRelease(ctx, queryRelease.Release, queryStart, queryEnd) if len(errs) > 0 { log.Errorf("FallbackBaseQueryStatus for %s failed with: %v", queryRelease, errs) return @@ -445,7 +433,7 @@ func calculateFallbackReleases(startingRelease string, timeRanges []crtest.Relea return selectedTimeRanges } -func (f *fallbackTestQueryReleasesGenerator) updateTestStatuses(release crtest.ReleaseTimeRange, updateStatuses map[string]bq.TestStatus) { +func (f *fallbackTestQueryReleasesGenerator) updateTestStatuses(release crtest.ReleaseTimeRange, updateStatuses map[string]crstatus.TestStatus) { var testStatuses ReleaseTestMap var ok bool @@ -456,7 +444,7 @@ func (f *fallbackTestQueryReleasesGenerator) updateTestStatuses(release crtest.R if testStatuses, ok = f.CachedFallbackTestStatuses.Releases[release.Release]; !ok { testStatuses = ReleaseTestMap{ ReleaseTimeRange: release, - Tests: map[string]bq.TestStatus{}, + Tests: map[string]crstatus.TestStatus{}, } f.CachedFallbackTestStatuses.Releases[release.Release] = testStatuses } @@ -466,104 +454,20 @@ func (f *fallbackTestQueryReleasesGenerator) updateTestStatuses(release crtest.R } } -func (f *fallbackTestQueryReleasesGenerator) getTestFallbackRelease(ctx context.Context, client *bqcachedclient.Client, release string, start, end time.Time) (bq.ReportTestStatus, []error) { - generator := newFallbackBaseQueryGenerator(client, f.ReqOptions, f.allJobVariants, release, start, end) - cacheSpec := api.NewCacheSpec(generator.getCacheKey(), "FallbackBaseTestStatus~", &end) - testStatuses, errs := api.GetDataFromCacheOrGenerate[bq.ReportTestStatus](ctx, f.client.Cache, f.ReqOptions.CacheOption, cacheSpec, generator.getTestFallbackRelease, bq.ReportTestStatus{}) +func (f *fallbackTestQueryReleasesGenerator) getTestFallbackRelease(ctx context.Context, release string, start, end time.Time) (crstatus.ReportTestStatus, []error) { + fallbackReqOpts := f.ReqOptions + fallbackReqOpts.BaseRelease.Name = release + fallbackReqOpts.BaseRelease.Start = start + fallbackReqOpts.BaseRelease.End = end + baseStatus, errs := f.dataProvider.QueryBaseTestStatus(ctx, fallbackReqOpts, f.allJobVariants) if len(errs) > 0 { - return bq.ReportTestStatus{}, errs + return crstatus.ReportTestStatus{}, errs } - return testStatuses, nil -} - -type fallbackTestQueryGenerator struct { - client *bqcachedclient.Client - allVariants crtest.JobVariants - BaseRelease string - BaseStart time.Time - BaseEnd time.Time - ReqOptions reqopts.RequestOptions -} - -func newFallbackBaseQueryGenerator(client *bqcachedclient.Client, reqOptions reqopts.RequestOptions, allVariants crtest.JobVariants, - baseRelease string, baseStart, baseEnd time.Time) fallbackTestQueryGenerator { - generator := fallbackTestQueryGenerator{ - client: client, - allVariants: allVariants, - ReqOptions: reqOptions, - BaseRelease: baseRelease, - BaseStart: baseStart, - BaseEnd: baseEnd, - } - return generator -} - -type fallbackTestQueryGeneratorCacheKey struct { - BaseRelease string - BaseStart time.Time - BaseEnd time.Time - // IgnoreDisruption and KeyTestNames are fields within AdvancedOption that affect the query - IgnoreDisruption bool - KeyTestNames []string - IncludeVariants map[string][]string - VariantDBGroupBy sets.String - // if we ever needed fallback on cross-compare views we should include fields for that, - // but that's not likely to ever make sense. -} - -// getCacheKey creates a cache key using the generator properties that we want included for uniqueness in what -// we cache. This provides a safer option than using the generator previously which carries some public fields -// which would be serialized and thus cause unnecessary cache misses. -func (f *fallbackTestQueryGenerator) getCacheKey() fallbackTestQueryGeneratorCacheKey { - return fallbackTestQueryGeneratorCacheKey{ - BaseRelease: f.BaseRelease, - BaseStart: f.BaseStart, - BaseEnd: f.BaseEnd, - IgnoreDisruption: f.ReqOptions.AdvancedOption.IgnoreDisruption, - KeyTestNames: f.ReqOptions.AdvancedOption.KeyTestNames, - IncludeVariants: f.ReqOptions.VariantOption.IncludeVariants, - VariantDBGroupBy: f.ReqOptions.VariantOption.DBGroupBy, - } + return crstatus.ReportTestStatus{BaseStatus: baseStatus}, nil } -func (f *fallbackTestQueryGenerator) getTestFallbackRelease(ctx context.Context) (bq.ReportTestStatus, []error) { - commonQuery, groupByQuery, queryParameters := query.BuildComponentReportQuery( - f.client, f.ReqOptions, f.allVariants, f.ReqOptions.VariantOption.IncludeVariants, - query.DefaultJunitTable, false) - before := time.Now() - log.Infof("Starting Fallback (%s) QueryTestStatus", f.BaseRelease) - errs := []error{} - baseString := commonQuery + ` AND jv_Release.variant_value = @BaseRelease` - baseQuery := f.client.Query(ctx, bqlabel.CRJunitFallback, baseString+groupByQuery) - - baseQuery.Parameters = append(baseQuery.Parameters, queryParameters...) - baseQuery.Parameters = append(baseQuery.Parameters, []bigquery.QueryParameter{ - { - Name: "From", - Value: f.BaseStart, - }, - { - Name: "To", - Value: f.BaseEnd, - }, - { - Name: "BaseRelease", - Value: f.BaseRelease, - }, - }...) - - baseStatus, baseErrs := query.FetchTestStatusResults(ctx, baseQuery) - - if len(baseErrs) != 0 { - errs = append(errs, baseErrs...) - } - - log.Infof("Fallback (%s) QueryTestStatus completed in %s with %d base results from db", f.BaseRelease, time.Since(before), len(baseStatus)) - - return bq.ReportTestStatus{BaseStatus: baseStatus}, errs -} func newFallbackReleases() FallbackReleases { fb := FallbackReleases{ @@ -574,7 +478,7 @@ func newFallbackReleases() FallbackReleases { type ReleaseTestMap struct { crtest.ReleaseTimeRange - Tests map[string]bq.TestStatus + Tests map[string]crstatus.TestStatus } type FallbackReleases struct { diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go index 2c3be4fb67..cfedff70ce 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" @@ -64,7 +64,7 @@ func Test_PreAnalysis(t *testing.T) { } fallbackMap418 := ReleaseTestMap{ ReleaseTimeRange: release418, - Tests: map[string]bq.TestStatus{ + Tests: map[string]crstatus.TestStatus{ test1KeyStr: buildTestStatus("test1", test1VariantsFlattened, 100, 95, 0), }, } @@ -78,7 +78,7 @@ func Test_PreAnalysis(t *testing.T) { } fallbackMap417 := ReleaseTestMap{ ReleaseTimeRange: release417, - Tests: map[string]bq.TestStatus{ + Tests: map[string]crstatus.TestStatus{ test1KeyStr: buildTestStatus("test1", test1VariantsFlattened, 100, 98, 0), }, } @@ -224,8 +224,8 @@ func TestCalculateFallbackReleases(t *testing.T) { } //nolint:unparam -func buildTestStatus(testName string, variants []string, total, success, flake int) bq.TestStatus { - return bq.TestStatus{ +func buildTestStatus(testName string, variants []string, total, success, flake int) crstatus.TestStatus { + return crstatus.TestStatus{ TestName: testName, TestSuite: "conformance", Component: "foo", diff --git a/pkg/api/componentreadiness/query/querygenerators.go b/pkg/api/componentreadiness/query/querygenerators.go index f20d69b275..aa679d3ca1 100644 --- a/pkg/api/componentreadiness/query/querygenerators.go +++ b/pkg/api/componentreadiness/query/querygenerators.go @@ -11,7 +11,7 @@ import ( "cloud.google.com/go/bigquery" "cloud.google.com/go/civil" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/bigquery/bqlabel" @@ -71,7 +71,7 @@ func NewBaseQueryGenerator( return generator } -func (b *baseQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTestStatus, []error) { +func (b *baseQueryGenerator) QueryTestStatus(ctx context.Context) (crstatus.ReportTestStatus, []error) { commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(b.client, b.ReqOptions, b.allVariants, b.ReqOptions.VariantOption.IncludeVariants, DefaultJunitTable, false) @@ -101,7 +101,7 @@ func (b *baseQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTest errs = append(errs, baseErrs...) } - return bq.ReportTestStatus{BaseStatus: baseStatus}, errs + return crstatus.ReportTestStatus{BaseStatus: baseStatus}, errs } type sampleQueryGenerator struct { @@ -141,7 +141,7 @@ func NewSampleQueryGenerator( return generator } -func (s *sampleQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTestStatus, []error) { +func (s *sampleQueryGenerator) QueryTestStatus(ctx context.Context) (crstatus.ReportTestStatus, []error) { commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(s.client, s.ReqOptions, s.allVariants, s.IncludeVariants, s.JunitTable, true) errs := []error{} @@ -207,7 +207,7 @@ func (s *sampleQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTe errs = append(errs, sampleErrs...) } - return bq.ReportTestStatus{SampleStatus: sampleStatus}, errs + return crstatus.ReportTestStatus{SampleStatus: sampleStatus}, errs } // buildPriorityCaseStatement generates a SQL CASE statement that assigns priority based on test position in the list. @@ -693,9 +693,9 @@ func filterByCrossCompareVariants(crossCompare []string, variantGroups map[strin return } -func FetchTestStatusResults(ctx context.Context, query *bigquery.Query) (map[string]bq.TestStatus, []error) { +func FetchTestStatusResults(ctx context.Context, query *bigquery.Query) (map[string]crstatus.TestStatus, []error) { errs := []error{} - status := map[string]bq.TestStatus{} + status := map[string]crstatus.TestStatus{} bqcachedclient.LogQueryWithParamsReplaced(log.WithField("type", "ComponentReport"), query) it, err := query.Read(ctx) @@ -735,10 +735,10 @@ func FetchTestStatusResults(ctx context.Context, query *bigquery.Query) (map[str // deserializeRowToTestStatus deserializes a single row into a testID string and matching status. // This is where we handle the dynamic variant_ columns, parsing these into a map on the test identification. // Other fixed columns we expect are serialized directly to their appropriate columns. -func deserializeRowToTestStatus(row []bigquery.Value, schema bigquery.Schema) (string, bq.TestStatus, error) { +func deserializeRowToTestStatus(row []bigquery.Value, schema bigquery.Schema) (string, crstatus.TestStatus, error) { if len(row) != len(schema) { log.Infof("row is %+v, schema is %+v", row, schema) - return "", bq.TestStatus{}, fmt.Errorf("number of values in row doesn't match schema length") + return "", crstatus.TestStatus{}, fmt.Errorf("number of values in row doesn't match schema length") } // Expect: @@ -763,7 +763,7 @@ func deserializeRowToTestStatus(row []bigquery.Value, schema bigquery.Schema) (s tid := crtest.KeyWithVariants{ Variants: map[string]string{}, } - cts := bq.TestStatus{} + cts := crstatus.TestStatus{} for i, fieldSchema := range schema { col := fieldSchema.Name // Some rows we know what to expect, others are dynamic (variants) and go into the map. @@ -852,7 +852,7 @@ func NewBaseTestDetailsQueryGenerator(logger log.FieldLogger, client *bqcachedcl } } -func (b *baseTestDetailsQueryGenerator) QueryTestStatus(ctx context.Context) (bq.TestJobRunStatuses, []error) { +func (b *baseTestDetailsQueryGenerator) QueryTestStatus(ctx context.Context) (crstatus.TestJobRunStatuses, []error) { commonQuery, groupByQuery, queryParameters := buildTestDetailsQuery( b.client, b.TestIDOpts, @@ -879,7 +879,7 @@ func (b *baseTestDetailsQueryGenerator) QueryTestStatus(ctx context.Context) (bq }...) baseStatus, errs := fetchJobRunTestStatusResults(ctx, b.logger, baseQuery) - return bq.TestJobRunStatuses{BaseStatus: baseStatus}, errs + return crstatus.TestJobRunStatuses{BaseStatus: baseStatus}, errs } // sampleTestDetailsQueryGenerator generates the query we use for the sample on the test details page. @@ -919,7 +919,7 @@ func NewSampleTestDetailsQueryGenerator( } } -func (s *sampleTestDetailsQueryGenerator) QueryTestStatus(ctx context.Context) (bq.TestJobRunStatuses, []error) { +func (s *sampleTestDetailsQueryGenerator) QueryTestStatus(ctx context.Context) (crstatus.TestJobRunStatuses, []error) { commonQuery, groupByQuery, queryParameters := buildTestDetailsQuery( s.client, @@ -982,12 +982,12 @@ func (s *sampleTestDetailsQueryGenerator) QueryTestStatus(ctx context.Context) ( sampleStatus, errs := fetchJobRunTestStatusResults(ctx, log.WithField("generator", "SampleQuery"), sampleQuery) - return bq.TestJobRunStatuses{SampleStatus: sampleStatus}, errs + return crstatus.TestJobRunStatuses{SampleStatus: sampleStatus}, errs } -func fetchJobRunTestStatusResults(ctx context.Context, logger log.FieldLogger, query *bigquery.Query) (map[string][]bq.TestJobRunRows, []error) { +func fetchJobRunTestStatusResults(ctx context.Context, logger log.FieldLogger, query *bigquery.Query) (map[string][]crstatus.TestJobRunRows, []error) { errs := []error{} - status := map[string][]bq.TestJobRunRows{} + status := map[string][]crstatus.TestJobRunRows{} bqcachedclient.LogQueryWithParamsReplaced(logger.WithField("type", "TestDetails"), query) @@ -1027,13 +1027,13 @@ func fetchJobRunTestStatusResults(ctx context.Context, logger log.FieldLogger, q // deserializeRowToJobRunTestReportStatus deserializes a single row into a testID string and matching status. // This is where we handle the dynamic variant_ columns, parsing these into a map on the test identification. // Other fixed columns we expect are serialized directly to their appropriate columns. -func deserializeRowToJobRunTestReportStatus(row []bigquery.Value, schema bigquery.Schema) (bq.TestJobRunRows, error) { +func deserializeRowToJobRunTestReportStatus(row []bigquery.Value, schema bigquery.Schema) (crstatus.TestJobRunRows, error) { if len(row) != len(schema) { log.Infof("row is %+v, schema is %+v", row, schema) - return bq.TestJobRunRows{}, fmt.Errorf("number of values in row doesn't match schema length") + return crstatus.TestJobRunRows{}, fmt.Errorf("number of values in row doesn't match schema length") } - cts := bq.TestJobRunRows{ + cts := crstatus.TestJobRunRows{ TestKey: crtest.KeyWithVariants{Variants: map[string]string{}}, } for i, fieldSchema := range schema { @@ -1055,7 +1055,11 @@ func deserializeRowToJobRunTestReportStatus(row []bigquery.Value, schema bigquer cts.ProwJobURL = row[i].(string) } case col == "prowjob_start": - cts.StartTime = row[i].(civil.DateTime) + if dt, ok := row[i].(civil.DateTime); ok { + cts.StartTime = time.Date(dt.Date.Year, dt.Date.Month, dt.Date.Day, dt.Time.Hour, dt.Time.Minute, dt.Time.Second, dt.Time.Nanosecond, time.UTC) + } else if t, ok := row[i].(time.Time); ok { + cts.StartTime = t + } case col == "test_id": cts.TestKey.TestID = row[i].(string) case col == "test_name": diff --git a/pkg/api/componentreadiness/regressiontracker.go b/pkg/api/componentreadiness/regressiontracker.go index 4533bbb36b..287f5eb227 100644 --- a/pkg/api/componentreadiness/regressiontracker.go +++ b/pkg/api/componentreadiness/regressiontracker.go @@ -8,6 +8,7 @@ import ( "time" "github.com/andygrunwald/go-jira" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware/regressiontracker" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" @@ -16,7 +17,6 @@ import ( "github.com/openshift/sippy/pkg/apis/cache" configv1 "github.com/openshift/sippy/pkg/apis/config/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - sippybigquery "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" "github.com/pkg/errors" @@ -160,7 +160,7 @@ func (prs *PostgresRegressionStore) ResolveTriages() error { } func NewRegressionTracker( - bigqueryClient *sippybigquery.Client, + dataProvider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, releases []v1.Release, @@ -170,7 +170,7 @@ func NewRegressionTracker( dryRun bool) *RegressionTracker { return &RegressionTracker{ - bigqueryClient: bigqueryClient, + dataProvider: dataProvider, dbc: dbc, cacheOpts: cacheOptions, releases: releases, @@ -185,7 +185,7 @@ func NewRegressionTracker( // RegressionTracker is the primary object for managing regression tracking logic. type RegressionTracker struct { backend RegressionStore - bigqueryClient *sippybigquery.Client + dataProvider dataprovider.DataProvider dbc *db.DB cacheOpts cache.RequestOptions releases []v1.Release @@ -254,8 +254,8 @@ func (rt *RegressionTracker) SyncRegressionsForRelease(ctx context.Context, rele CacheOption: rt.cacheOpts, } - report, reportErrs := GetComponentReportFromBigQuery( - ctx, rt.bigqueryClient, rt.dbc, reportOpts, rt.variantJunitTableOverrides, "") + report, reportErrs := GetComponentReport( + ctx, rt.dataProvider, rt.dbc, reportOpts, rt.variantJunitTableOverrides, "") if len(reportErrs) > 0 { var strErrors []string for _, err := range reportErrs { diff --git a/pkg/api/componentreadiness/test_details.go b/pkg/api/componentreadiness/test_details.go index 7ad49c9928..ee6b8d24ae 100644 --- a/pkg/api/componentreadiness/test_details.go +++ b/pkg/api/componentreadiness/test_details.go @@ -11,7 +11,7 @@ import ( "time" fet "github.com/glycerine/golang-fisher-exact" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" @@ -20,23 +20,23 @@ import ( "github.com/sirupsen/logrus" "github.com/openshift/sippy/pkg/api" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/query" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" configv1 "github.com/openshift/sippy/pkg/apis/config/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/util" ) -func GetTestDetails(ctx context.Context, client *bigquery.Client, dbc *db.DB, reqOptions reqopts.RequestOptions, releases []v1.Release, baseURL string) (testdetails.Report, []error) { - generator := NewComponentReportGenerator(client, reqOptions, dbc, nil, releases, baseURL) +func GetTestDetails(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, releases []v1.Release, baseURL string) (testdetails.Report, []error) { + generator := NewComponentReportGenerator(provider, reqOptions, dbc, nil, releases, baseURL) if os.Getenv("DEV_MODE") == "1" { return generator.GenerateTestDetailsReport(ctx) } report, errs := api.GetDataFromCacheOrGenerate[testdetails.Report]( ctx, - generator.client.Cache, + generator.getCache(), generator.ReqOptions.CacheOption, api.NewCacheSpec(generator.GetCacheKey(ctx), "TestDetailsReport~", nil), generator.GenerateTestDetailsReport, @@ -77,7 +77,7 @@ func (c *ComponentReportGenerator) GenerateTestDetailsReport(ctx context.Context // This function is called from the API, and we assume only one TestIDOptions entry in that case. testIDOptions := c.ReqOptions.TestIDOptions[0] // load all pass/fails for specific jobs, both sample, basis, and override basis if requested - componentJobRunTestReportStatus, errs := c.getJobRunTestStatusFromBigQuery(ctx) + componentJobRunTestReportStatus, errs := c.getJobRunTestStatus(ctx) if len(errs) > 0 { return testdetails.Report{}, errs } @@ -89,31 +89,31 @@ func (c *ComponentReportGenerator) GenerateTestDetailsReport(ctx context.Context func (c *ComponentReportGenerator) GenerateTestDetailsReportMultiTest(ctx context.Context) ([]testdetails.Report, []error) { // load all pass/fails for specific jobs, both sample, basis, and override basis if requested before := time.Now() - allTestsJobRunStatuses, errs := c.getJobRunTestStatusFromBigQuery(ctx) + allTestsJobRunStatuses, errs := c.getJobRunTestStatus(ctx) if len(errs) > 0 { return []testdetails.Report{}, errs } - logrus.Infof("getJobRunTestStatusFromBigQuery completed in %s with %d sample results and %d base results from db", + logrus.Infof("getJobRunTestStatus completed in %s with %d sample results and %d base results", time.Since(before), len(allTestsJobRunStatuses.SampleStatus), len(allTestsJobRunStatuses.BaseStatus)) // We have a struct where the statuses are mapped by prowjob to all rows results for that prowjob, // with multiple tests intermingled in that layer. // Build out a new struct where these are split up by test ID. // split the status on test ID, and pass only that tests data in for reporting: - testKeyTestJobRunStatuses := map[string]bq.TestJobRunStatuses{} + testKeyTestJobRunStatuses := map[string]crstatus.TestJobRunStatuses{} for jobName, rows := range allTestsJobRunStatuses.BaseStatus { for _, row := range rows { testKeyStr := row.TestKeyStr if _, ok := testKeyTestJobRunStatuses[testKeyStr]; !ok { - testKeyTestJobRunStatuses[testKeyStr] = bq.TestJobRunStatuses{ - BaseStatus: map[string][]bq.TestJobRunRows{}, - BaseOverrideStatus: map[string][]bq.TestJobRunRows{}, - SampleStatus: map[string][]bq.TestJobRunRows{}, + testKeyTestJobRunStatuses[testKeyStr] = crstatus.TestJobRunStatuses{ + BaseStatus: map[string][]crstatus.TestJobRunRows{}, + BaseOverrideStatus: map[string][]crstatus.TestJobRunRows{}, + SampleStatus: map[string][]crstatus.TestJobRunRows{}, GeneratedAt: allTestsJobRunStatuses.GeneratedAt, } } if testKeyTestJobRunStatuses[testKeyStr].BaseStatus[jobName] == nil { - testKeyTestJobRunStatuses[testKeyStr].BaseStatus[jobName] = []bq.TestJobRunRows{} + testKeyTestJobRunStatuses[testKeyStr].BaseStatus[jobName] = []crstatus.TestJobRunRows{} } testKeyTestJobRunStatuses[testKeyStr].BaseStatus[jobName] = append(testKeyTestJobRunStatuses[testKeyStr].BaseStatus[jobName], row) @@ -123,15 +123,15 @@ func (c *ComponentReportGenerator) GenerateTestDetailsReportMultiTest(ctx contex for _, row := range rows { testKeyStr := row.TestKeyStr if _, ok := testKeyTestJobRunStatuses[testKeyStr]; !ok { - testKeyTestJobRunStatuses[testKeyStr] = bq.TestJobRunStatuses{ - BaseStatus: map[string][]bq.TestJobRunRows{}, - BaseOverrideStatus: map[string][]bq.TestJobRunRows{}, - SampleStatus: map[string][]bq.TestJobRunRows{}, + testKeyTestJobRunStatuses[testKeyStr] = crstatus.TestJobRunStatuses{ + BaseStatus: map[string][]crstatus.TestJobRunRows{}, + BaseOverrideStatus: map[string][]crstatus.TestJobRunRows{}, + SampleStatus: map[string][]crstatus.TestJobRunRows{}, GeneratedAt: allTestsJobRunStatuses.GeneratedAt, } } if testKeyTestJobRunStatuses[testKeyStr].BaseOverrideStatus[jobName] == nil { - testKeyTestJobRunStatuses[testKeyStr].BaseOverrideStatus[jobName] = []bq.TestJobRunRows{} + testKeyTestJobRunStatuses[testKeyStr].BaseOverrideStatus[jobName] = []crstatus.TestJobRunRows{} } testKeyTestJobRunStatuses[testKeyStr].BaseOverrideStatus[jobName] = append(testKeyTestJobRunStatuses[testKeyStr].BaseOverrideStatus[jobName], row) @@ -141,15 +141,15 @@ func (c *ComponentReportGenerator) GenerateTestDetailsReportMultiTest(ctx contex for _, row := range rows { testKeyStr := row.TestKeyStr if _, ok := testKeyTestJobRunStatuses[testKeyStr]; !ok { - testKeyTestJobRunStatuses[testKeyStr] = bq.TestJobRunStatuses{ - BaseStatus: map[string][]bq.TestJobRunRows{}, - BaseOverrideStatus: map[string][]bq.TestJobRunRows{}, - SampleStatus: map[string][]bq.TestJobRunRows{}, + testKeyTestJobRunStatuses[testKeyStr] = crstatus.TestJobRunStatuses{ + BaseStatus: map[string][]crstatus.TestJobRunRows{}, + BaseOverrideStatus: map[string][]crstatus.TestJobRunRows{}, + SampleStatus: map[string][]crstatus.TestJobRunRows{}, GeneratedAt: allTestsJobRunStatuses.GeneratedAt, } } if testKeyTestJobRunStatuses[testKeyStr].SampleStatus[jobName] == nil { - testKeyTestJobRunStatuses[testKeyStr].SampleStatus[jobName] = []bq.TestJobRunRows{} + testKeyTestJobRunStatuses[testKeyStr].SampleStatus[jobName] = []crstatus.TestJobRunRows{} } testKeyTestJobRunStatuses[testKeyStr].SampleStatus[jobName] = append(testKeyTestJobRunStatuses[testKeyStr].SampleStatus[jobName], row) @@ -183,7 +183,7 @@ func (c *ComponentReportGenerator) GenerateTestDetailsReportMultiTest(ctx contex func (c *ComponentReportGenerator) GenerateDetailsReportForTest( ctx context.Context, testIDOption reqopts.TestIdentification, - componentJobRunTestReportStatus bq.TestJobRunStatuses, + componentJobRunTestReportStatus crstatus.TestJobRunStatuses, allowUnregressedReports bool, ) (testdetails.Report, []error) { @@ -199,7 +199,7 @@ func (c *ComponentReportGenerator) GenerateDetailsReportForTest( } } - timeRanges, errs := query.GetReleaseDatesFromBigQuery(ctx, c.client, c.ReqOptions) + timeRanges, errs := c.dataProvider.QueryReleaseDates(ctx, c.ReqOptions) if errs != nil { return testdetails.Report{}, errs } @@ -339,30 +339,13 @@ func (c *ComponentReportGenerator) getBaseJobRunTestStatus( allJobVariants crtest.JobVariants, baseRelease string, baseStart time.Time, - baseEnd time.Time) (map[string][]bq.TestJobRunRows, []error) { - - generator := query.NewBaseTestDetailsQueryGenerator( - logrus.WithField("func", "getBaseJobRunTestStatus"), - c.client, - c.ReqOptions, - allJobVariants, - baseRelease, - baseStart, - baseEnd, - c.ReqOptions.TestIDOptions, - ) - - jobRunTestStatus, errs := api.GetDataFromCacheOrGenerate[bq.TestJobRunStatuses](ctx, - c.client.Cache, c.ReqOptions.CacheOption, - api.NewCacheSpec(generator, "BaseJobRunTestStatus~", &baseEnd), - generator.QueryTestStatus, - bq.TestJobRunStatuses{}) + baseEnd time.Time) (map[string][]crstatus.TestJobRunRows, []error) { - if len(errs) > 0 { - return nil, errs - } - - return jobRunTestStatus.BaseStatus, nil + reqOpts := c.ReqOptions + reqOpts.BaseRelease.Name = baseRelease + reqOpts.BaseRelease.Start = baseStart + reqOpts.BaseRelease.End = baseEnd + return c.dataProvider.QueryBaseJobRunTestStatus(ctx, reqOpts, allJobVariants) } func (c *ComponentReportGenerator) getSampleJobRunTestStatus( @@ -370,38 +353,24 @@ func (c *ComponentReportGenerator) getSampleJobRunTestStatus( allJobVariants crtest.JobVariants, includeVariants map[string][]string, start, end time.Time, - junitTable string) (map[string][]bq.TestJobRunRows, []error) { - - generator := query.NewSampleTestDetailsQueryGenerator( - c.client, c.ReqOptions, - allJobVariants, includeVariants, start, end, junitTable) - - jobRunTestStatus, errs := api.GetDataFromCacheOrGenerate[bq.TestJobRunStatuses](ctx, - c.client.Cache, c.ReqOptions.CacheOption, - api.NewCacheSpec(generator, "SampleJobRunTestStatus~", &end), - generator.QueryTestStatus, - bq.TestJobRunStatuses{}) - - if len(errs) > 0 { - return nil, errs - } + junitTable string) (map[string][]crstatus.TestJobRunRows, []error) { - return jobRunTestStatus.SampleStatus, nil + return c.dataProvider.QuerySampleJobRunTestStatus(ctx, c.ReqOptions, allJobVariants, includeVariants, start, end, junitTable) } -func (c *ComponentReportGenerator) getJobRunTestStatusFromBigQuery(ctx context.Context) (bq.TestJobRunStatuses, []error) { - fLog := logrus.WithField("func", "getJobRunTestStatusFromBigQuery") - allJobVariants, errs := GetJobVariantsFromBigQuery(ctx, c.client) +func (c *ComponentReportGenerator) getJobRunTestStatus(ctx context.Context) (crstatus.TestJobRunStatuses, []error) { + fLog := logrus.WithField("func", "getJobRunTestStatus") + allJobVariants, errs := GetJobVariants(ctx, c.dataProvider) if len(errs) > 0 { - logrus.Errorf("failed to get variants from bigquery") - return bq.TestJobRunStatuses{}, errs + logrus.Errorf("failed to get job variants") + return crstatus.TestJobRunStatuses{}, errs } - var baseStatus, sampleStatus map[string][]bq.TestJobRunRows + var baseStatus, sampleStatus map[string][]crstatus.TestJobRunRows var baseErrs, baseOverrideErrs, sampleErrs []error wg := sync.WaitGroup{} // channels for status as we may collect status from multiple queries run in separate goroutines - statusCh := make(chan map[string][]bq.TestJobRunRows) + statusCh := make(chan map[string][]crstatus.TestJobRunRows) errCh := make(chan error) statusDoneCh := make(chan struct{}) // To signal when all processing is done statusErrsDoneCh := make(chan struct{}) // To signal when all processing is done @@ -498,7 +467,7 @@ func (c *ComponentReportGenerator) getJobRunTestStatusFromBigQuery(ctx context.C for k, v := range status { if sampleStatus == nil { fLog.Warnf("initializing sampleStatus map") - sampleStatus = make(map[string][]bq.TestJobRunRows) + sampleStatus = make(map[string][]crstatus.TestJobRunRows) } if v2, ok := sampleStatus[k]; ok { fLog.Warnf("sampleStatus already had key: %+v", k) @@ -526,7 +495,7 @@ func (c *ComponentReportGenerator) getJobRunTestStatusFromBigQuery(ctx context.C errs = append(errs, baseOverrideErrs...) } - return bq.TestJobRunStatuses{BaseStatus: baseStatus, SampleStatus: sampleStatus}, errs + return crstatus.TestJobRunStatuses{BaseStatus: baseStatus, SampleStatus: sampleStatus}, errs } // internalGenerateTestDetailsReport handles the report generation for the lowest level test report including @@ -534,7 +503,7 @@ func (c *ComponentReportGenerator) getJobRunTestStatusFromBigQuery(ctx context.C func (c *ComponentReportGenerator) internalGenerateTestDetailsReport( baseRelease string, baseStart, baseEnd *time.Time, - baseStatus, sampleStatus map[string][]bq.TestJobRunRows, + baseStatus, sampleStatus map[string][]crstatus.TestJobRunRows, testIDOption reqopts.TestIdentification, ) testdetails.Report { testKey := crtest.Identification{ @@ -584,7 +553,7 @@ func (c *ComponentReportGenerator) internalGenerateTestDetailsReport( // go through all the job runs that had a test and summarize the results func (c *ComponentReportGenerator) summarizeRecordedTestStats( - baseStatus, sampleStatus map[string][]bq.TestJobRunRows, testKey crtest.Identification, + baseStatus, sampleStatus map[string][]crstatus.TestJobRunRows, testKey crtest.Identification, ) ( totalBase, totalSample crtest.Stats, report testdetails.Analysis, @@ -629,7 +598,7 @@ func (c *ComponentReportGenerator) summarizeRecordedTestStats( // assessTestStats calculates the test stats for a given list of job rows // and updates by-reference parameters with information found in the job rows. func (c *ComponentReportGenerator) assessTestStats( - jobRowsList []bq.TestJobRunRows, + jobRowsList []crstatus.TestJobRunRows, testStats *crtest.Stats, jobRunStatsList *[]testdetails.JobRunStats, jobName *string, lastFailure *time.Time, @@ -659,7 +628,7 @@ func (c *ComponentReportGenerator) assessTestStats( } } -func (c *ComponentReportGenerator) getJobRunStats(stats bq.TestJobRunRows) testdetails.JobRunStats { +func (c *ComponentReportGenerator) getJobRunStats(stats crstatus.TestJobRunRows) testdetails.JobRunStats { jobRunStats := testdetails.JobRunStats{ TestStats: crtest.NewTestStats( stats.SuccessCount, diff --git a/pkg/api/componentreadiness/utils/utils.go b/pkg/api/componentreadiness/utils/utils.go index 5b42e6cf92..751f69f7b4 100644 --- a/pkg/api/componentreadiness/utils/utils.go +++ b/pkg/api/componentreadiness/utils/utils.go @@ -12,7 +12,7 @@ import ( "github.com/sirupsen/logrus" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" @@ -57,7 +57,7 @@ func NormalizeProwJobName(prowName string) string { // DeserializeTestKey helps us workaround the limitations of a struct as a map key, where // we instead serialize a very small struct to json for a unit test key that includes test // ID and a specific set of variants. This function deserializes back to a struct. -func DeserializeTestKey(stats bq.TestStatus, testKeyStr string) (crtest.Identification, error) { +func DeserializeTestKey(stats crstatus.TestStatus, testKeyStr string) (crtest.Identification, error) { var testKey crtest.KeyWithVariants err := json.Unmarshal([]byte(testKeyStr), &testKey) if err != nil { diff --git a/pkg/apis/api/componentreport/crstatus/types.go b/pkg/apis/api/componentreport/crstatus/types.go new file mode 100644 index 0000000000..71d6ff3010 --- /dev/null +++ b/pkg/apis/api/componentreport/crstatus/types.go @@ -0,0 +1,78 @@ +package crstatus + +import ( + "math/big" + "time" + + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" +) + +// Package crstatus contains data-transfer types used between the data layer and +// the Component Readiness analysis pipeline. These types were originally in the +// bq package but are backend-agnostic — any data provider (BigQuery, PostgreSQL, +// mock) populates them identically. + +// ReportTestStatus contains the mapping of all test keys (serialized with KeyWithVariants, variants + testID) +// It is an internal type used to pass data from the data provider to report generation, +// and does not get serialized as an API response. +type ReportTestStatus struct { + // BaseStatus represents the stable basis for the comparison. Maps KeyWithVariants serialized as a string, to test status. + BaseStatus map[string]TestStatus `json:"base_status"` + + // SampleStatus represents the sample for the comparison. Maps KeyWithVariants serialized as a string, to test status. + SampleStatus map[string]TestStatus `json:"sample_status"` + GeneratedAt *time.Time `json:"generated_at"` +} + +// TestStatus is an internal type used to pass data from the data provider to the +// actual report generation. It is not serialized over the API. +type TestStatus struct { + TestName string `json:"test_name"` + TestSuite string `json:"test_suite"` + Component string `json:"component"` + Capabilities []string `json:"capabilities"` + Variants []string `json:"variants"` + crtest.Count + LastFailure time.Time `json:"last_failure"` +} + +// TestJobRunStatuses contains the rows returned from a test details query organized by base and sample, +// essentially the actual job runs and their status that was used to calculate this +// report. +// Status fields map prowjob name to each row result we received for that job. +type TestJobRunStatuses struct { + BaseStatus map[string][]TestJobRunRows `json:"base_status"` + // TODO: This could be a little cleaner if we did status.BaseStatuses plural and tied them to a release, + // allowing the release fallback mechanism to stay a little cleaner. That would more clearly + // keep middleware details out of the main codebase. + BaseOverrideStatus map[string][]TestJobRunRows `json:"base_override_status"` + SampleStatus map[string][]TestJobRunRows `json:"sample_status"` + GeneratedAt *time.Time `json:"generated_at"` +} + +// TestJobRunRows are the per job run rows from a test details report +// indicating if the test passed or failed. +type TestJobRunRows struct { + TestKey crtest.KeyWithVariants `json:"test_key"` + TestKeyStr string `json:"-"` // transient field so we dont have to keep recalculating + TestName string `bigquery:"test_name"` + ProwJob string `bigquery:"prowjob_name"` + ProwJobRunID string `bigquery:"prowjob_run_id"` + ProwJobURL string `bigquery:"prowjob_url"` + StartTime time.Time `bigquery:"prowjob_start"` + crtest.Count + JiraComponent string `bigquery:"jira_component"` + JiraComponentID *big.Rat `bigquery:"jira_component_id"` +} + +// JobVariant defines a variant and the possible values. +type JobVariant struct { + VariantName string `bigquery:"variant_name"` + VariantValues []string `bigquery:"variant_values"` +} + +// Variant is a single key/value variant pair. +type Variant struct { + Key string `bigquery:"key" json:"key"` + Value string `bigquery:"value" json:"value"` +} diff --git a/pkg/apis/api/componentreport/testdetails/types.go b/pkg/apis/api/componentreport/testdetails/types.go index 33e790b8c0..11166eb45f 100644 --- a/pkg/apis/api/componentreport/testdetails/types.go +++ b/pkg/apis/api/componentreport/testdetails/types.go @@ -4,7 +4,6 @@ import ( "math/big" "time" - "cloud.google.com/go/civil" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/db/models" ) @@ -106,7 +105,7 @@ type JobStats struct { type JobRunStats struct { JobURL string `json:"job_url"` JobRunID string `json:"job_run_id"` - StartTime civil.DateTime `json:"start_time"` + StartTime time.Time `json:"start_time"` // TestStats is the test stats from one particular job run. // For the majority of the tests, there is only one junit. But // there are cases multiple junits are generated for the same test. diff --git a/pkg/bigquery/bqlabel/labels.go b/pkg/bigquery/bqlabel/labels.go index ceb888839f..373e55d5b5 100644 --- a/pkg/bigquery/bqlabel/labels.go +++ b/pkg/bigquery/bqlabel/labels.go @@ -72,6 +72,7 @@ const ( CRJunitBase QueryValue = "component-readiness-junit-base" CRJunitSample QueryValue = "component-readiness-junit-sample" CRJunitFallback QueryValue = "component-readiness-junit-fallback" + CRViewJobs QueryValue = "component-readiness-view-jobs" TDJunitBase QueryValue = "test-details-junit-base" TDJunitSample QueryValue = "test-details-junit-sample" DisruptionDelta QueryValue = "disruption-delta" diff --git a/pkg/componentreadiness/jiraautomator/jiraautomator.go b/pkg/componentreadiness/jiraautomator/jiraautomator.go index bd6e7ada23..5f2f07221e 100644 --- a/pkg/componentreadiness/jiraautomator/jiraautomator.go +++ b/pkg/componentreadiness/jiraautomator/jiraautomator.go @@ -12,6 +12,7 @@ import ( "github.com/andygrunwald/go-jira" "github.com/openshift/sippy/pkg/api/componentreadiness" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" @@ -52,6 +53,7 @@ type JiraComponent struct { type JiraAutomator struct { jiraClient *jira.Client bqClient *bqclient.Client + dataProvider dataprovider.DataProvider dbc *db.DB cacheOptions cache.RequestOptions views []crview.View @@ -71,6 +73,7 @@ type JiraAutomator struct { func NewJiraAutomator( jiraClient *jira.Client, bqClient *bqclient.Client, + provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, views []crview.View, @@ -86,6 +89,7 @@ func NewJiraAutomator( j := JiraAutomator{ jiraClient: jiraClient, bqClient: bqClient, + dataProvider: provider, dbc: dbc, cacheOptions: cacheOptions, releases: releases, @@ -148,7 +152,7 @@ func (j JiraAutomator) getComponentReportForView(view crview.View) (crtype.Compo } // Passing empty gcs bucket and prow URL, they are not needed outside test details reports - report, errs := componentreadiness.GetComponentReportFromBigQuery(context.Background(), j.bqClient, j.dbc, reportOpts, j.variantJunitTableOverrides, "") + report, errs := componentreadiness.GetComponentReport(context.Background(), j.dataProvider, j.dbc, reportOpts, j.variantJunitTableOverrides, "") if len(errs) > 0 { var strErrors []string for _, err := range errs { diff --git a/pkg/componentreadiness/jobrunannotator/jobrunannotator.go b/pkg/componentreadiness/jobrunannotator/jobrunannotator.go index 6be2e02876..6e80e4b69a 100644 --- a/pkg/componentreadiness/jobrunannotator/jobrunannotator.go +++ b/pkg/componentreadiness/jobrunannotator/jobrunannotator.go @@ -13,7 +13,7 @@ import ( "cloud.google.com/go/civil" "cloud.google.com/go/storage" "github.com/openshift/sippy/pkg/api/jobartifacts" - "github.com/openshift/sippy/pkg/apis/api/componentreport/bq" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/cache" bqclient "github.com/openshift/sippy/pkg/bigquery" @@ -65,7 +65,7 @@ type JobRunAnnotator struct { execute bool allVariants crtest.JobVariants Release string `json:"release"` - IncludedVariants []bq.Variant `json:"included_variants"` + IncludedVariants []crstatus.Variant `json:"included_variants"` Label string `json:"label"` BuildClusters []string `json:"build_clusters"` StartTime time.Time `json:"start_time"` @@ -89,7 +89,7 @@ func NewJobRunAnnotator( execute bool, release string, allVariants crtest.JobVariants, - variants []bq.Variant, + variants []crstatus.Variant, label string, buildClusters []string, startTime time.Time, diff --git a/pkg/dataloader/crcacheloader/crcacheloader.go b/pkg/dataloader/crcacheloader/crcacheloader.go index df027d3203..90725038b1 100644 --- a/pkg/dataloader/crcacheloader/crcacheloader.go +++ b/pkg/dataloader/crcacheloader/crcacheloader.go @@ -8,6 +8,8 @@ import ( "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" + bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" sippytypes "github.com/openshift/sippy/pkg/apis/api" crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" @@ -34,6 +36,7 @@ type ComponentReadinessCacheLoader struct { releases []apiv1.Release cacheClient cache.Cache bqClient *bigquery.Client + dataProvider dataprovider.DataProvider config *v1.SippyConfig crTimeRoundingFactor time.Duration } @@ -54,6 +57,7 @@ func New( views: views, releases: releases, bqClient: bqClient, + dataProvider: bqprovider.NewBigQueryProvider(bqClient), config: config, crTimeRoundingFactor: crTimeRoundingFactor, } @@ -261,6 +265,6 @@ func (l *ComponentReadinessCacheLoader) buildGenerator( // Making a generator directly as we are going to bypass the caching to ensure we get fresh report, // explicitly set our reports in the cache, thus resetting the timer for all expiry and keeping the cache // primed. - generator := componentreadiness.NewComponentReportGenerator(l.bqClient, reqOpts, l.dbc, l.config.ComponentReadinessConfig.VariantJunitTableOverrides, l.releases, "") + generator := componentreadiness.NewComponentReportGenerator(l.dataProvider, reqOpts, l.dbc, l.config.ComponentReadinessConfig.VariantJunitTableOverrides, l.releases, "") return &generator, nil } diff --git a/pkg/sippyserver/metrics/metrics.go b/pkg/sippyserver/metrics/metrics.go index 592fe71295..f3fc7f92f2 100644 --- a/pkg/sippyserver/metrics/metrics.go +++ b/pkg/sippyserver/metrics/metrics.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" @@ -116,10 +117,11 @@ func getReleaseStatus(releases []v1.Release, release string) string { // presume in a historical context there won't be scraping of these metrics // pinning the time just to be consistent -func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, reportEnd time.Time, cacheOptions cache.RequestOptions, views []crview.View, variantJunitTableOverrides []configv1.VariantJunitTableOverride) error { +func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crProvider dataprovider.DataProvider, reportEnd time.Time, cacheOptions cache.RequestOptions, views []crview.View, variantJunitTableOverrides []configv1.VariantJunitTableOverride) error { start := time.Now() log.Info("beginning refresh metrics") - releases, err := api.GetReleases(context.Background(), bqc, false) + + releases, err := crProvider.QueryReleases(ctx) if err != nil { return err } @@ -165,10 +167,12 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, rep } - // BigQuery metrics - if bqc != nil { - refreshComponentReadinessMetrics(ctx, bqc, dbc, cacheOptions, views, releases, variantJunitTableOverrides) + if crProvider != nil { + refreshComponentReadinessMetrics(ctx, crProvider, dbc, cacheOptions, views, releases, variantJunitTableOverrides) + } + // BigQuery-only metrics + if bqc != nil { if err := refreshDisruptionMetrics(bqc, releases); err != nil { log.WithError(err).Error("error refreshing disruption metrics") } @@ -179,21 +183,11 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, rep return nil } -func refreshComponentReadinessMetrics(ctx context.Context, client *bqclient.Client, dbc *db.DB, +func refreshComponentReadinessMetrics(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, views []crview.View, releases []v1.Release, variantJunitTableOverrides []configv1.VariantJunitTableOverride) { - if client == nil || client.BQ == nil { - log.Warningf("not generating component readiness metrics as we don't have a bigquery client") - return - } - - if client.Cache == nil { - log.Warningf("not generating component readiness metrics as we don't have a cache configured") - return - } - for _, view := range views { if view.Metrics.Enabled { - err := updateComponentReadinessMetricsForView(ctx, client, dbc, cacheOptions, view, releases, variantJunitTableOverrides) + err := updateComponentReadinessMetricsForView(ctx, provider, dbc, cacheOptions, view, releases, variantJunitTableOverrides) if err != nil { log.WithError(err).Error("error") log.WithError(err).WithField("view", view.Name).Error("error refreshing metrics/regressions for view") @@ -204,7 +198,7 @@ func refreshComponentReadinessMetrics(ctx context.Context, client *bqclient.Clie } // updateComponentReadinessTrackingForView queries the report for the given view, and then updates metrics. -func updateComponentReadinessMetricsForView(ctx context.Context, client *bqclient.Client, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []v1.Release, overrides []configv1.VariantJunitTableOverride) error { +func updateComponentReadinessMetricsForView(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []v1.Release, overrides []configv1.VariantJunitTableOverride) error { logger := log.WithField("view", view.Name) logger.Info("generating report for view") @@ -233,7 +227,7 @@ func updateComponentReadinessMetricsForView(ctx context.Context, client *bqclien CacheOption: cacheOptions, } - report, errs := componentreadiness.GetComponentReportFromBigQuery(ctx, client, dbc, reportOpts, overrides, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, dbc, reportOpts, overrides, "") if len(errs) > 0 { var strErrors []string for _, err := range errs { diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index fdcb0776e7..c952f29158 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -21,6 +21,7 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" "github.com/openshift/sippy/pkg/api/jobartifacts" "github.com/openshift/sippy/pkg/apis/api/componentreport" @@ -48,6 +49,7 @@ import ( "github.com/openshift/sippy/pkg/api/jobrunintervals" apitype "github.com/openshift/sippy/pkg/apis/api" "github.com/openshift/sippy/pkg/apis/cache" + sippyv1 "github.com/openshift/sippy/pkg/apis/sippy/v1" sippybq "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" @@ -79,6 +81,7 @@ func NewServer( gcsClient *storage.Client, gcsBucket string, bigQueryClient *sippybq.Client, + crDataProvider dataprovider.DataProvider, pinnedDateTime *time.Time, cacheClient cache.Cache, crTimeRoundingFactor time.Duration, @@ -100,6 +103,7 @@ func NewServer( static: static, db: dbClient, bigQueryClient: bigQueryClient, + crDataProvider: crDataProvider, pinnedDateTime: pinnedDateTime, gcsClient: gcsClient, gcsBucket: gcsBucket, @@ -112,8 +116,13 @@ func NewServer( jiraClient: jiraClient, } - if bigQueryClient != nil { - go componentreadiness.GetComponentTestVariantsFromBigQuery(context.Background(), bigQueryClient) + if crDataProvider != nil { + go func() { + _, errs := componentreadiness.GetComponentTestVariants(context.Background(), server.crDataProvider) + if len(errs) > 0 { + log.WithField("errors", errs).Warn("errors during component test variants prefetch") + } + }() } return server @@ -153,6 +162,7 @@ type Server struct { httpServer *http.Server db *db.DB bigQueryClient *sippybq.Client + crDataProvider dataprovider.DataProvider pinnedDateTime *time.Time gcsClient *storage.Client gcsBucket string @@ -167,6 +177,19 @@ type Server struct { rateLimiters map[string]*rateLimiter } +// getReleases returns release data, preferring the BigQuery client with caching +// when available, falling back to the data provider for mock mode. +func (s *Server) getReleases(ctx context.Context, forceRefresh ...bool) ([]sippyv1.Release, error) { + if s.bigQueryClient != nil { + refresh := len(forceRefresh) > 0 && forceRefresh[0] + return api.GetReleases(ctx, s.bigQueryClient, refresh) + } + if s.crDataProvider != nil { + return s.crDataProvider.QueryReleases(ctx) + } + return nil, fmt.Errorf("no data source available for releases") +} + type rateLimiter struct { mu sync.Mutex requestTimes []time.Time @@ -386,7 +409,7 @@ func (s *Server) determineCapabilities() { capabilities = append(capabilities, OpenshiftCapability) } - if s.bigQueryClient != nil { + if s.bigQueryClient != nil || s.crDataProvider != nil { capabilities = append(capabilities, ComponentReadinessCapability) } if s.db != nil { @@ -881,11 +904,11 @@ func (s *Server) jsonTestRunsAndOutputsFromBigQuery(w http.ResponseWriter, req * } func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, req *http.Request) { - if s.bigQueryClient == nil { - failureResponse(w, http.StatusBadRequest, "component report API is only available when google-service-account-credential-file is configured") + if s.crDataProvider == nil { + failureResponse(w, http.StatusBadRequest, "component report API is only available when a data provider is configured") return } - outputs, errs := componentreadiness.GetComponentTestVariantsFromBigQuery(req.Context(), s.bigQueryClient) + outputs, errs := componentreadiness.GetComponentTestVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { log.Warningf("%d errors were encountered while querying test variants from big query:", len(errs)) for _, err := range errs { @@ -898,11 +921,11 @@ func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, re } func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Request) { - if s.bigQueryClient == nil { - failureResponse(w, http.StatusBadRequest, "job variants API is only available when google-service-account-credential-file is configured") + if s.crDataProvider == nil { + failureResponse(w, http.StatusBadRequest, "job variants API is only available when a data provider is configured") return } - outputs, errs := componentreadiness.GetJobVariantsFromBigQuery(req.Context(), s.bigQueryClient) + outputs, errs := componentreadiness.GetJobVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { log.Warningf("%d errors were encountered while querying job variants from big query:", len(errs)) for _, err := range errs { @@ -915,7 +938,7 @@ func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Re } func (s *Server) jsonComponentReadinessViews(w http.ResponseWriter, req *http.Request) { - allReleases, err := api.GetReleases(req.Context(), s.bigQueryClient, false) + allReleases, err := s.getReleases(req.Context()) if err != nil { failureResponse(w, http.StatusBadRequest, err.Error()) return @@ -978,16 +1001,16 @@ func (s *Server) getRegressedTestsForRegressions(req *http.Request, regressions // getComponentReportFromRequest creates a component report based on the HTTP request parameters func (s *Server) getComponentReportFromRequest(req *http.Request) (componentreport.ComponentReport, error) { - if s.bigQueryClient == nil { - return componentreport.ComponentReport{}, fmt.Errorf("component report API is only available when google-service-account-credential-file is configured") + if s.crDataProvider == nil { + return componentreport.ComponentReport{}, fmt.Errorf("component report API is only available when a data provider is configured") } - allJobVariants, errs := componentreadiness.GetJobVariantsFromBigQuery(req.Context(), s.bigQueryClient) + allJobVariants, errs := componentreadiness.GetJobVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - return componentreport.ComponentReport{}, fmt.Errorf("failed to get variants from bigquery") + return componentreport.ComponentReport{}, fmt.Errorf("failed to get job variants") } - allReleases, err := api.GetReleases(req.Context(), s.bigQueryClient, false) + allReleases, err := s.getReleases(req.Context()) if err != nil { return componentreport.ComponentReport{}, err } @@ -1002,9 +1025,9 @@ func (s *Server) getComponentReportFromRequest(req *http.Request) (componentrepo // This baseURL is used to generate links to test_details reports, which are frontend links baseURL := api.GetBaseFrontendURL(req) - outputs, errs := componentreadiness.GetComponentReportFromBigQuery( + outputs, errs := componentreadiness.GetComponentReport( req.Context(), - s.bigQueryClient, + s.crDataProvider, s.db, options, s.config.ComponentReadinessConfig.VariantJunitTableOverrides, @@ -1031,18 +1054,18 @@ func (s *Server) jsonComponentReportFromBigQuery(w http.ResponseWriter, req *htt } func (s *Server) jsonComponentReportTestDetailsFromBigQuery(w http.ResponseWriter, req *http.Request) { - if s.bigQueryClient == nil { - err := fmt.Errorf("component report API is only available when google-service-account-credential-file is configured") + if s.crDataProvider == nil { + err := fmt.Errorf("component report API is only available when a data provider is configured") failureResponse(w, http.StatusBadRequest, err.Error()) return } - allJobVariants, errs := componentreadiness.GetJobVariantsFromBigQuery(req.Context(), s.bigQueryClient) + allJobVariants, errs := componentreadiness.GetJobVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - err := fmt.Errorf("failed to get variants from bigquery") + err := fmt.Errorf("failed to get job variants") failureResponse(w, http.StatusBadRequest, err.Error()) return } - allReleases, err := api.GetReleases(req.Context(), s.bigQueryClient, false) + allReleases, err := s.getReleases(req.Context()) if err != nil { failureResponse(w, http.StatusBadRequest, err.Error()) return @@ -1056,7 +1079,7 @@ func (s *Server) jsonComponentReportTestDetailsFromBigQuery(w http.ResponseWrite return } baseURL := api.GetBaseURL(req) - outputs, errs := componentreadiness.GetTestDetails(req.Context(), s.bigQueryClient, s.db, reqOptions, allReleases, baseURL) + outputs, errs := componentreadiness.GetTestDetails(req.Context(), s.crDataProvider, s.db, reqOptions, allReleases, baseURL) if len(errs) > 0 { log.Warningf("%d errors were encountered while querying component test details from big query:", len(errs)) for _, err := range errs { @@ -1131,8 +1154,8 @@ func (s *Server) jsonTestDetailsReportFromDB(w http.ResponseWriter, req *http.Re } func (s *Server) jsonReleasesReportFromDB(w http.ResponseWriter, req *http.Request) { - forceRefresh := req.URL.Query().Get("forceRefresh") != "" // use to refresh cached releases from BQ - releases, err := api.GetReleases(req.Context(), s.bigQueryClient, forceRefresh) + forceRefresh := req.URL.Query().Get("forceRefresh") != "" + releases, err := s.getReleases(req.Context(), forceRefresh) if err != nil { log.WithError(err).Error("error querying releases") failureResponse(w, http.StatusInternalServerError, "error querying releases") @@ -1812,7 +1835,7 @@ func (s *Server) jsonTriagePotentialMatchingRegressions(w http.ResponseWriter, r failureResponse(w, http.StatusBadRequest, "no baseRelease provided") return } - allReleases, err := api.GetReleases(req.Context(), s.bigQueryClient, false) + allReleases, err := s.getReleases(req.Context()) if err != nil { failureResponse(w, http.StatusInternalServerError, err.Error()) return @@ -1858,7 +1881,7 @@ func (s *Server) jsonGetTriageAuditDetails(w http.ResponseWriter, req *http.Requ // jsonGetRegressions handles GET requests for listing component readiness regression records. func (s *Server) jsonGetRegressions(w http.ResponseWriter, req *http.Request) { // Get releases for view processing - allReleases, err := api.GetReleases(req.Context(), s.bigQueryClient, false) + allReleases, err := s.getReleases(req.Context()) if err != nil { failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error getting releases: %v", err)) return @@ -1910,7 +1933,7 @@ func (s *Server) jsonGetRegressionByID(w http.ResponseWriter, req *http.Request) } // Get releases for view processing - allReleases, err := api.GetReleases(req.Context(), s.bigQueryClient, false) + allReleases, err := s.getReleases(req.Context()) if err != nil { failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error getting releases: %v", err)) return @@ -1948,7 +1971,7 @@ func (s *Server) jsonRegressionPotentialMatchingTriages(w http.ResponseWriter, r return } // Get releases for view processing - allReleases, err := api.GetReleases(req.Context(), s.bigQueryClient, false) + allReleases, err := s.getReleases(req.Context()) if err != nil { failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error getting releases: %v", err)) return diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 958e504c7e..f5444fddef 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -12,8 +12,7 @@ REDIS_CONTAINER="sippy-e2e-test-redis" REDIS_PORT="23479" if [[ -z "$GCS_SA_JSON_PATH" ]]; then - echo "Must provide path to GCS credential in GCS_SA_JSON_PATH env var" 1>&2 - exit 1 + echo "WARNING: GCS_SA_JSON_PATH not set, data sync test will be skipped" 1>&2 fi @@ -52,28 +51,29 @@ sleep 5 export SIPPY_E2E_DSN="postgresql://postgres:password@localhost:$PSQL_PORT/postgres" export REDIS_URL="redis://localhost:$REDIS_PORT" +export SIPPY_E2E_REPO_ROOT="$(pwd)" echo "Loading database..." -# use an old release here as they have very few job runs and thus import quickly, ~5 minutes go build -mod vendor ./cmd/sippy ./sippy seed-data \ --init-database \ - --database-dsn="$SIPPY_E2E_DSN" \ - --release="4.20" + --database-dsn="$SIPPY_E2E_DSN" # Spawn sippy server off into a separate process: export SIPPY_API_PORT="18080" export SIPPY_ENDPOINT="127.0.0.1" -( -./sippy serve \ - --listen ":$SIPPY_API_PORT" \ - --listen-metrics ":12112" \ - --database-dsn="$SIPPY_E2E_DSN" \ + +SERVE_ARGS="--listen :$SIPPY_API_PORT \ + --listen-metrics :12112 \ + --database-dsn=$SIPPY_E2E_DSN \ --enable-write-endpoints \ --log-level debug \ - --views config/e2e-views.yaml \ - --google-service-account-credential-file $GCS_SA_JSON_PATH \ - --redis-url="$REDIS_URL" > e2e.log 2>&1 + --redis-url=$REDIS_URL \ + --data-provider postgres \ + --views config/e2e-views.yaml" + +( +./sippy serve $SERVE_ARGS > e2e.log 2>&1 )& # store the child process for cleanup CHILD_PID=$! @@ -96,6 +96,14 @@ if [ $ELAPSED -ge $TIMEOUT ]; then exit 1 fi +# Prime the component readiness cache so triage tests can find cached reports +echo "Priming component readiness cache..." +VIEWS=$(curl -s "http://localhost:$SIPPY_API_PORT/api/component_readiness/views") +for VIEW in $(echo "$VIEWS" | jq -r '.[].name'); do + echo " Priming cache for view: $VIEW" + curl -s "http://localhost:$SIPPY_API_PORT/api/component_readiness?view=$VIEW" > /dev/null +done +echo "Cache priming complete" # Run our tests that request against the API, args ensure serially and fresh test code compile: gotestsum ./test/e2e/... -count 1 -p 1 diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/componentreadiness_test.go index 5deb044923..a70bb39aab 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/componentreadiness_test.go @@ -2,10 +2,14 @@ package componentreadiness import ( "fmt" + "net/url" "testing" "github.com/openshift/sippy/pkg/apis/api/componentreport" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" + "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" + "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/test/e2e/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,6 +26,235 @@ func TestComponentReadinessViews(t *testing.T) { var report componentreport.ComponentReport err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", views[0].Name), &report) require.NoError(t, err, "error making http request") - // We expect over 50 components at time of writing, asserting 25 should be safe - assert.Greater(t, len(report.Rows), 25, "component report does not have rows we would expect") + assert.Greater(t, len(report.Rows), 10, "component report does not have rows we would expect") +} + +func TestTestDetails(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // First get a report to find a regressed test + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &report) + require.NoError(t, err, "error getting component report") + require.NotEmpty(t, report.Rows, "report should have rows") + + // Find a regressed test to query details for + var testID, component, capability string + variants := map[string]string{} + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + testID = rt.TestID + component = rt.Component + capability = rt.Capability + variants = rt.ColumnIdentification.Variants + found = true + break + } + if found { + break + } + } + if found { + break + } + } + require.True(t, found, "should find at least one regressed test in the report") + + // Build query params for test_details + params := url.Values{} + params.Set("view", viewName) + params.Set("testId", testID) + params.Set("component", component) + params.Set("capability", capability) + for k, v := range variants { + params.Set(k, v) + } + + var details testdetails.Report + err = util.SippyGet(fmt.Sprintf("/api/component_readiness/test_details?%s", params.Encode()), &details) + require.NoError(t, err, "error getting test details") + + assert.Equal(t, testID, details.TestID, "response should match requested test ID") + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + for _, analysis := range details.Analyses { + total := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, total, 0, "sample stats should have run data") + } +} + +func TestVariants(t *testing.T) { + // /api/component_readiness/variants returns CacheVariants (legacy BQ column names) + var variants map[string][]string + err := util.SippyGet("/api/component_readiness/variants", &variants) + require.NoError(t, err, "error getting variants") + assert.Contains(t, variants, "platform", "should have platform variant") + assert.Contains(t, variants, "network", "should have network variant") + assert.Contains(t, variants, "arch", "should have arch variant") + + // /api/job_variants returns JobVariants with all variant groups + var jobVariants crtest.JobVariants + err = util.SippyGet("/api/job_variants", &jobVariants) + require.NoError(t, err, "error getting job variants") + require.NotEmpty(t, jobVariants.Variants, "should have variants") + + expectedVariants := []string{"Platform", "Architecture", "Network", "Topology", "FeatureSet"} + for _, v := range expectedVariants { + assert.Contains(t, jobVariants.Variants, v, "should have %s variant", v) + assert.NotEmpty(t, jobVariants.Variants[v], "%s variant should have values", v) + } +} + +func TestRegressionByID(t *testing.T) { + // List regressions to find one we can query by ID + var regressions []models.TestRegression + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions?release=%s", util.Release), ®ressions) + require.NoError(t, err, "error listing regressions") + require.NotEmpty(t, regressions, "should have regressions from seed data") + + // Fetch one by ID + regression := regressions[0] + var fetched models.TestRegression + err = util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions/%d", regression.ID), &fetched) + require.NoError(t, err, "error getting regression by ID") + + assert.Equal(t, regression.ID, fetched.ID) + assert.Equal(t, regression.TestID, fetched.TestID) + assert.Equal(t, regression.TestName, fetched.TestName) + assert.Equal(t, regression.Release, fetched.Release) + + // HATEOAS links + assert.NotNil(t, fetched.Links, "regression should have HATEOAS links") + assert.Contains(t, fetched.Links, "test_details", "should have test_details link") +} + +func TestRegressionByIDNotFound(t *testing.T) { + var fetched models.TestRegression + err := util.SippyGet("/api/component_readiness/regressions/999999", &fetched) + require.Error(t, err, "should return error for non-existent regression") +} + +func TestColumnGroupByAndDBGroupBy(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + t.Run("default grouping from view", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + // Default view has columnGroupBy: Network, Platform, Topology + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Contains(t, col.ColumnIdentification.Variants, "Topology") + } + } + }) + + t.Run("override columnGroupBy to Platform only", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&columnGroupBy=Platform", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform", + "columns should still have Platform") + assert.Equal(t, 1, len(col.ColumnIdentification.Variants), + "columns should only have Platform when columnGroupBy is overridden to Platform") + } + } + }) + + t.Run("override columnGroupBy to Platform,Network", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&columnGroupBy=Platform,Network", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Equal(t, 2, len(col.ColumnIdentification.Variants), + "columns should have exactly Platform and Network") + } + } + }) + + t.Run("override dbGroupBy reduces aggregation", func(t *testing.T) { + // With fewer dbGroupBy dimensions, we should get fewer rows because tests + // are aggregated at a coarser level + var defaultReport componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &defaultReport) + require.NoError(t, err) + + // Use a minimal dbGroupBy — just Platform, Network, Topology (must include columnGroupBy) + var reducedReport componentreport.ComponentReport + err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&dbGroupBy=Platform,Network,Topology", viewName), &reducedReport) + require.NoError(t, err) + require.NotEmpty(t, reducedReport.Rows) + + // The reduced dbGroupBy should produce a valid report. We can't strictly assert + // fewer rows since it depends on the data, but it should still work without error. + t.Logf("default report: %d rows, reduced dbGroupBy: %d rows", len(defaultReport.Rows), len(reducedReport.Rows)) + }) + + t.Run("override both columnGroupBy and dbGroupBy together", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&columnGroupBy=Platform&dbGroupBy=Platform,Architecture,Network,Topology", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Equal(t, 1, len(col.ColumnIdentification.Variants), + "columns should only have Platform when columnGroupBy is Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + } + } + }) +} + +func TestVariantCrossCompare(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + t.Run("cross-compare Platform with specific value", func(t *testing.T) { + // The default view includes Platform:aws,gcp. Cross-comparing Platform + // with a specific value means the sample uses only that Platform value. + params := url.Values{} + params.Set("view", viewName) + params.Add("variantCrossCompare", "Platform") + params.Add("compareVariant", "Platform:gcp") + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?%s", params.Encode()), &report) + require.NoError(t, err, "cross-compare request should succeed") + t.Logf("cross-compare Platform:gcp report: %d rows", len(report.Rows)) + + // Columns should still respect the columnGroupBy from the view + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + } + } + }) + + t.Run("cross-compare without compareVariant values", func(t *testing.T) { + // Cross-compare a group without specifying compareVariant values means + // no restriction on that variant in the sample + params := url.Values{} + params.Set("view", viewName) + params.Add("variantCrossCompare", "Platform") + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?%s", params.Encode()), &report) + require.NoError(t, err, "cross-compare without compareVariant should succeed") + t.Logf("cross-compare (no compareVariant) report: %d rows", len(report.Rows)) + }) } diff --git a/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go b/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go index 809e226227..640018e876 100644 --- a/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go +++ b/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go @@ -21,11 +21,19 @@ import ( "github.com/stretchr/testify/require" ) -func cleanupAllRegressions(dbc *db.DB) { - // Delete all test regressions in the e2e postgres db. - res := dbc.DB.Where("1 = 1").Delete(&models.TestRegression{}) - if res.Error != nil { - log.Errorf("error deleting test regressions: %v", res.Error) +// cleanupRegressions deletes only the specified test regressions and their +// associated triage links from the e2e postgres db. Tests should clean up +// only what they create to avoid destroying seed data regressions. +func cleanupRegressions(dbc *db.DB, regressions ...*models.TestRegression) { + for _, r := range regressions { + if r == nil { + continue + } + dbc.DB.Exec("DELETE FROM triage_regressions WHERE test_regression_id = ?", r.ID) + res := dbc.DB.Delete(r) + if res.Error != nil { + log.Errorf("error deleting test regression %d: %v", r.ID, res.Error) + } } } @@ -36,7 +44,7 @@ func Test_RegressionTracker(t *testing.T) { newRegression := componentreport.ReportTestSummary{ TestComparison: testdetails.TestComparison{ BaseStats: &testdetails.ReleaseStats{ - Release: "4.18", + Release: "4.20", }, }, Identification: crtest.Identification{ @@ -56,27 +64,27 @@ func Test_RegressionTracker(t *testing.T) { }, } view := crview.View{ - Name: "4.19-main", + Name: "4.22-main", SampleRelease: reqopts.RelativeRelease{ Release: reqopts.Release{ - Name: "4.19", + Name: "4.22", }, }, } t.Run("open a new regression", func(t *testing.T) { - defer cleanupAllRegressions(dbc) tr, err := tracker.OpenRegression(view, newRegression) + defer cleanupRegressions(dbc, tr) require.NoError(t, err) - assert.Equal(t, "4.19", tr.Release) - assert.Equal(t, "4.18", tr.BaseRelease, "BaseRelease should be set from BaseStats.Release") + assert.Equal(t, "4.22", tr.Release) + assert.Equal(t, "4.20", tr.BaseRelease, "BaseRelease should be set from BaseStats.Release") assert.ElementsMatch(t, pq.StringArray([]string{"a:b", "c:d"}), tr.Variants) assert.True(t, tr.ID > 0) }) t.Run("close and reopen a regression", func(t *testing.T) { - defer cleanupAllRegressions(dbc) tr, err := tracker.OpenRegression(view, newRegression) + defer cleanupRegressions(dbc, tr) require.NoError(t, err) // look it up just to be sure: @@ -105,63 +113,75 @@ func Test_RegressionTracker(t *testing.T) { }) t.Run("list current regressions for release", func(t *testing.T) { - defer cleanupAllRegressions(dbc) var err error - open419, err := rawCreateRegression(dbc, "4.19", + open422, err := rawCreateRegression(dbc, "4.22", "test1ID", "test 1", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Time{}) require.NoError(t, err) - recentlyClosed419, err := rawCreateRegression(dbc, "4.19", + recentlyClosed422, err := rawCreateRegression(dbc, "4.22", "test2ID", "test 2", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Now().Add(-2*24*time.Hour)) require.NoError(t, err) - _, err = rawCreateRegression(dbc, "4.19", + oldClosed422, err := rawCreateRegression(dbc, "4.22", "test3ID", "test 3", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Now().Add(-70*24*time.Hour)) require.NoError(t, err) - _, err = rawCreateRegression(dbc, "4.18", + other421, err := rawCreateRegression(dbc, "4.21", "test1ID", "test 1", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Time{}) require.NoError(t, err) + defer cleanupRegressions(dbc, open422, recentlyClosed422, oldClosed422, other421) - // List all regressions for 4.19, should exclude 4.18, include recently closed regressions, + // List all regressions for 4.22, should exclude 4.21, include recently closed regressions, // and exclude older closed regressions. - relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.19") + relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.22") require.NoError(t, err) - assert.Equal(t, 2, len(relRegressions)) + + // Verify our expected regressions are present (open + recently closed) + foundIDs := make(map[uint]bool) for _, rel := range relRegressions { - assert.True(t, rel.ID == open419.ID || rel.ID == recentlyClosed419.ID, - "unexpected regression was returned: %+v", *rel) + foundIDs[rel.ID] = true } + assert.True(t, foundIDs[open422.ID], "open regression should be in list") + assert.True(t, foundIDs[recentlyClosed422.ID], "recently closed regression should be in list") + assert.False(t, foundIDs[oldClosed422.ID], "old closed regression should not be in list") + assert.False(t, foundIDs[other421.ID], "4.21 regression should not be in list") }) t.Run("list returns regressions with BaseRelease set", func(t *testing.T) { - defer cleanupAllRegressions(dbc) - tr, err := rawCreateRegressionWithBase(dbc, "4.19", "4.18", "baseTestID", "base test", + tr, err := rawCreateRegressionWithBase(dbc, "4.22", "4.21", "baseTestID", "base test", []string{"a:b"}, time.Now().Add(-1*24*time.Hour), time.Time{}) require.NoError(t, err) - relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.19") + defer cleanupRegressions(dbc, tr) + relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.22") require.NoError(t, err) - require.Len(t, relRegressions, 1) - assert.Equal(t, tr.ID, relRegressions[0].ID) - assert.Equal(t, "4.18", relRegressions[0].BaseRelease, "ListCurrentRegressionsForRelease should return BaseRelease") + + // Find our regression in the list and verify BaseRelease is set + var found *models.TestRegression + for _, rel := range relRegressions { + if rel.ID == tr.ID { + found = rel + break + } + } + require.NotNil(t, found, "created regression should be in list") + assert.Equal(t, "4.21", found.BaseRelease, "ListCurrentRegressionsForRelease should return BaseRelease") }) t.Run("closing a regression should resolve associated triages that have no other active regressions", func(t *testing.T) { - defer cleanupAllRegressions(dbc) - regressionToClose, err := tracker.OpenRegression(view, newRegression) require.NoError(t, err) + defer cleanupRegressions(dbc, regressionToClose) // Create a second regression that will remain open secondRegression := componentreport.ReportTestSummary{ TestComparison: testdetails.TestComparison{ BaseStats: &testdetails.ReleaseStats{ - Release: "4.18", + Release: "4.20", }, }, Identification: crtest.Identification{ @@ -182,6 +202,7 @@ func Test_RegressionTracker(t *testing.T) { } regressionToRemainOpened, err := tracker.OpenRegression(view, secondRegression) require.NoError(t, err) + defer cleanupRegressions(dbc, regressionToRemainOpened) // Create first triage associated only with the first regression triage := models.Triage{ @@ -195,6 +216,10 @@ func Test_RegressionTracker(t *testing.T) { dbWithContext := dbc.DB.WithContext(context.WithValue(context.Background(), models.CurrentUserKey, "e2e-test")) res := dbWithContext.Create(&triage) require.NoError(t, res.Error) + defer func() { + dbc.DB.Exec("DELETE FROM triage_regressions WHERE triage_id = ?", triage.ID) + dbc.DB.Delete(&triage) + }() // Create second triage associated with both regressions triage2 := models.Triage{ @@ -208,6 +233,10 @@ func Test_RegressionTracker(t *testing.T) { } res = dbWithContext.Create(&triage2) require.NoError(t, res.Error) + defer func() { + dbc.DB.Exec("DELETE FROM triage_regressions WHERE triage_id = ?", triage2.ID) + dbc.DB.Delete(&triage2) + }() // Close the regression with a time of NOW, this should not result in resolved triage regressionToClose.Closed = sql.NullTime{Valid: true, Time: time.Now()} diff --git a/test/e2e/componentreadiness/report/report_test.go b/test/e2e/componentreadiness/report/report_test.go new file mode 100644 index 0000000000..bcd457d02a --- /dev/null +++ b/test/e2e/componentreadiness/report/report_test.go @@ -0,0 +1,295 @@ +package report + +import ( + "context" + "strings" + "testing" + "time" + + componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" + pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" + "github.com/openshift/sippy/pkg/util/sets" + "github.com/openshift/sippy/test/e2e/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupProvider(t *testing.T) (*pgprovider.PostgresProvider, reqopts.RequestOptions) { + dbc := util.CreateE2EPostgresConnection(t) + provider := pgprovider.NewPostgresProvider(dbc, nil) + + now := time.Now().UTC().Truncate(time.Hour) + + reqOptions := reqopts.RequestOptions{ + BaseRelease: reqopts.Release{ + Name: "4.21", + Start: now.Add(-60 * 24 * time.Hour), + End: now.Add(-30 * 24 * time.Hour), + }, + SampleRelease: reqopts.Release{ + Name: "4.22", + Start: now.Add(-3 * 24 * time.Hour), + End: now, + }, + VariantOption: reqopts.Variants{ + ColumnGroupBy: sets.NewString("Network", "Platform", "Topology"), + DBGroupBy: sets.NewString("Architecture", "FeatureSet", "Installer", "Network", "Platform", "Suite", "Topology", "Upgrade", "LayeredProduct"), + }, + AdvancedOption: reqopts.Advanced{ + Confidence: 95, + PityFactor: 5, + MinimumFailure: 3, + IncludeMultiReleaseAnalysis: true, + }, + CacheOption: cache.RequestOptions{}, + } + + return provider, reqOptions +} + +func TestReportStatuses(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + require.Empty(t, errs, "GetComponentReport returned errors: %v", errs) + require.NotEmpty(t, report.Rows, "report should have rows") + + // Collect regressed test statuses keyed by testID+column platform. + type cellKey struct { + testID string + platform string + } + regressedStatuses := map[cellKey]crtest.Status{} + + for _, row := range report.Rows { + for _, col := range row.Columns { + if col.Status <= crtest.SignificantRegression { + for _, rt := range col.RegressedTests { + key := cellKey{testID: rt.TestID, platform: col.ColumnIdentification.Variants["Platform"]} + if existing, ok := regressedStatuses[key]; !ok || rt.ReportStatus < existing { + regressedStatuses[key] = rt.ReportStatus + } + } + } + } + } + + // Verify structural integrity + for _, row := range report.Rows { + assert.NotEmpty(t, row.RowIdentification.Component, "every row should have a component") + for _, col := range row.Columns { + assert.NotEmpty(t, col.ColumnIdentification.Variants, "column should have variants") + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Contains(t, col.ColumnIdentification.Variants, "Topology") + } + } + + // Collect all cell statuses across the grid + statusCounts := map[crtest.Status]int{} + for _, row := range report.Rows { + for _, col := range row.Columns { + statusCounts[col.Status]++ + } + } + + t.Logf("Status distribution: %v", statusCounts) + + assert.Contains(t, statusCounts, crtest.NotSignificant, "should have NotSignificant cells") + assert.Contains(t, statusCounts, crtest.MissingSample, "should have MissingSample cells") + + hasRegression := statusCounts[crtest.SignificantRegression] > 0 || statusCounts[crtest.ExtremeRegression] > 0 + assert.True(t, hasRegression, "should have at least one regression cell") + + // Verify specific regressed test statuses per platform + assert.Equal(t, crtest.ExtremeRegression, regressedStatuses[cellKey{"test-extreme-regression", "aws"}], + "extreme regression on aws should have ExtremeRegression status") + assert.Equal(t, crtest.SignificantRegression, regressedStatuses[cellKey{"test-significant-regression", "aws"}], + "significant regression on aws should have SignificantRegression status") +} + +func TestTestDetails(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + releases, err := provider.QueryReleases(ctx) + require.NoError(t, err) + + // Generate report to find a regressed test + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + require.Empty(t, errs) + + var testID reqopts.TestIdentification + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + testID = reqopts.TestIdentification{ + Component: rt.RowIdentification.Component, + Capability: rt.RowIdentification.Capability, + TestID: rt.RowIdentification.TestID, + RequestedVariants: rt.ColumnIdentification.Variants, + } + found = true + break + } + if found { + break + } + } + if found { + break + } + } + require.True(t, found, "should find at least one regressed test") + + detailReqOpts := reqOptions + detailReqOpts.TestIDOptions = []reqopts.TestIdentification{testID} + + details, detailErrs := componentreadiness.GetTestDetails(ctx, provider, nil, detailReqOpts, releases, "") + require.Empty(t, detailErrs, "GetTestDetails returned errors: %v", detailErrs) + + assert.Equal(t, testID.TestID, details.Identification.RowIdentification.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // With postgres provider, test details should have actual run data + for _, analysis := range details.Analyses { + total := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, total, 0, "sample stats should have run data") + } +} + +func TestFallback(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + require.Empty(t, errs, "GetComponentReport returned errors: %v", errs) + + type regressedInfo struct { + status crtest.Status + explanations []string + } + regressedByID := map[string]regressedInfo{} + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + regressedByID[rt.TestID] = regressedInfo{ + status: rt.ReportStatus, + explanations: rt.Explanations, + } + } + } + } + + if info, ok := regressedByID["test-fallback-improves"]; ok { + t.Logf("test-fallback-improves: status=%d, explanations=%v", info.status, info.explanations) + hasOverride := false + for _, exp := range info.explanations { + if strings.Contains(exp, "Overrode base stats") && strings.Contains(exp, "4.20") { + hasOverride = true + } + } + assert.True(t, hasOverride, "fallback-improves should mention override to 4.20 in explanations") + } else { + t.Error("test-fallback-improves should be in regressed tests") + } + + if info, ok := regressedByID["test-fallback-double"]; ok { + t.Logf("test-fallback-double: status=%d, explanations=%v", info.status, info.explanations) + hasOverride := false + for _, exp := range info.explanations { + if strings.Contains(exp, "Overrode base stats") && strings.Contains(exp, "4.19") { + hasOverride = true + } + } + assert.True(t, hasOverride, "fallback-double should mention override to 4.19 in explanations") + } else { + t.Error("test-fallback-double should be in regressed tests") + } +} + +func TestFallbackInsufficientRuns(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + require.Empty(t, errs) + + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-fallback-insufficient-runs" { + found = true + for _, exp := range rt.Explanations { + assert.NotContains(t, exp, "Overrode base stats", + "insufficient-runs test should NOT have fallback override explanation") + } + } + } + } + } + assert.True(t, found, "test-fallback-insufficient-runs should be in the report as a regression") +} + +func TestMissingBasis(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + require.Empty(t, errs) + + hasMissingBasis := false + hasMissingSample := false + for _, row := range report.Rows { + for _, col := range row.Columns { + if col.Status == crtest.MissingBasis { + hasMissingBasis = true + } + if col.Status == crtest.MissingSample { + hasMissingSample = true + } + } + } + + assert.True(t, hasMissingBasis, "report should have at least one MissingBasis cell") + assert.True(t, hasMissingSample, "report should have at least one MissingSample cell") +} + +func TestSignificantImprovement(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + require.Empty(t, errs) + + hasImprovement := false + for _, row := range report.Rows { + for _, col := range row.Columns { + if col.Status == crtest.SignificantImprovement { + hasImprovement = true + } + } + } + assert.True(t, hasImprovement, "report should have at least one SignificantImprovement cell") +} + +func TestJobVariants(t *testing.T) { + provider, _ := setupProvider(t) + ctx := context.Background() + + variants, errs := componentreadiness.GetJobVariants(ctx, provider) + require.Empty(t, errs) + require.NotEmpty(t, variants.Variants) + + assert.Contains(t, variants.Variants, "Platform") + assert.Contains(t, variants.Variants, "Architecture") + assert.Contains(t, variants.Variants, "Network") + assert.Contains(t, variants.Variants, "Topology") + assert.Contains(t, variants.Variants, "FeatureSet") +} diff --git a/test/e2e/componentreadiness/triage/triageapi_test.go b/test/e2e/componentreadiness/triage/triageapi_test.go index f854055178..bbdf010e96 100644 --- a/test/e2e/componentreadiness/triage/triageapi_test.go +++ b/test/e2e/componentreadiness/triage/triageapi_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "testing" "time" @@ -134,22 +135,20 @@ func Test_TriageAPI(t *testing.T) { t.Run("get with expanded regressions", func(t *testing.T) { defer cleanupAllTriages(dbc) - r := createTestRegressionWithDetails(t, tracker, view, "expanded-test-1", "component-expand", "capability-expand", "TestExpanded1", nil, crtest.ExtremeRegression) - defer dbc.DB.Delete(r.Regression) + // Use real regressions from seed data instead of injecting fake data into cache. + // We filter to seed data regressions (test IDs starting with "test-") because + // other subtests in this function create synthetic regressions that won't appear + // in the component report. + realRegressions := getSeedDataRegressions(t) + require.GreaterOrEqual(t, len(realRegressions), 2, "seed data should produce at least 2 regressions") - r2 := createTestRegressionWithDetails(t, tracker, view, "expanded-test-2", "component-expand", "capability-expand", "TestExpanded2", nil, crtest.SignificantRegression) - defer dbc.DB.Delete(r2.Regression) - - // TODO(sgoeddel): If we ever have a need for another view available within e2e tests we could verify that we could get regressed_tests - // for multiple views at once here, but it isn't worth the overhead now. - - // Create a triage with the test regressions + // Create a triage with real regressions triage := models.Triage{ URL: jiraBug.URL, Type: models.TriageTypeProduct, Regressions: []models.TestRegression{ - {ID: r.Regression.ID}, - {ID: r2.Regression.ID}, + {ID: realRegressions[0].ID}, + {ID: realRegressions[1].ID}, }, } @@ -158,18 +157,6 @@ func Test_TriageAPI(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(triageResponse.Regressions)) - // Add the test regressions to the component report cache so they can be found by the expanded endpoint - cache, err := util.NewE2ECacheManipulator(util.Release) - if err != nil { - t.Fatalf("Failed to create component report cache: %v", err) - } - defer cache.Close() - - err = cache.AddTestRegressionsToReport([]componentreport.ReportTestSummary{r, r2}) - if err != nil { - t.Fatalf("Failed to add test regressions to component report: %v", err) - } - // Validate that the expanded regressions are present var expandedTriage sippyserver.ExpandedTriage err = util.SippyGet(fmt.Sprintf("/api/component_readiness/triages/%d?view=%s-main&expand=regressions", triageResponse.ID, util.Release), &expandedTriage) @@ -183,16 +170,16 @@ func Test_TriageAPI(t *testing.T) { regressedTestsForView := expandedTriage.RegressedTests[expectedViewKey] assert.Len(t, regressedTestsForView, 2, "ExpandedTriage should contain 2 regressed tests for view %q", expectedViewKey) - // Verify status values are marked as their respective triaged values in the expanded response - statusMap := make(map[uint]crtest.Status) + // Verify status values are marked as triaged (the triage causes status transformation) for _, regressedTest := range regressedTestsForView { - if regressedTest != nil && regressedTest.Regression != nil { - statusMap[regressedTest.Regression.ID] = regressedTest.TestComparison.ReportStatus - } + require.NotNil(t, regressedTest, "regressed test should not be nil") + require.NotNil(t, regressedTest.Regression, "regressed test should have regression data") + assert.True(t, + regressedTest.TestComparison.ReportStatus == crtest.ExtremeTriagedRegression || + regressedTest.TestComparison.ReportStatus == crtest.SignificantTriagedRegression, + "regressed test %s should have a triaged status, got %d", + regressedTest.Regression.TestID, regressedTest.TestComparison.ReportStatus) } - - assert.Equal(t, crtest.ExtremeTriagedRegression, statusMap[r.Regression.ID], "First regressed test should have ExtremeTriagedRegression status") - assert.Equal(t, crtest.SignificantTriagedRegression, statusMap[r2.Regression.ID], "Second regressed test should have SignificantTriagedRegression status") }) t.Run("list", func(t *testing.T) { defer cleanupAllTriages(dbc) @@ -948,6 +935,23 @@ func deepCopyTriage(t *testing.T, original models.Triage) models.Triage { return triageCopy } +// getSeedDataRegressions fetches regressions from the API and filters to only those +// from the seed data (test IDs starting with "test-"), excluding any synthetic regressions +// created by other test functions in this package. +func getSeedDataRegressions(t *testing.T) []models.TestRegression { + var allRegressions []models.TestRegression + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions?release=%s", util.Release), &allRegressions) + require.NoError(t, err) + + var seedRegressions []models.TestRegression + for _, r := range allRegressions { + if strings.HasPrefix(r.TestID, "test-") { + seedRegressions = append(seedRegressions, r) + } + } + return seedRegressions +} + func assertTriageDataMatches(t *testing.T, expectedTriage, actualTriage models.Triage, field string) { assert.Equal(t, expectedTriage.ID, actualTriage.ID, "%s ID should match the expected triage ID", field) assert.Equal(t, expectedTriage.URL, actualTriage.URL, "%s URL should match the expected triage URL", field) @@ -1012,47 +1016,39 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { testRegressions[9] = createTestRegressionWithDetails(t, tracker, view, "match-similar-2", "component-j", "capability-q", "TestAnotheOne", &differentFailureTime, crtest.SignificantImprovement) // missing 'r' from "TestAnotherOne" defer dbc.DB.Delete(testRegressions[9].Regression) - // Add all test regressions to the component report so they can be found by GetTriagePotentialMatches - cache, err := util.NewE2ECacheManipulator(util.Release) - if err != nil { - t.Fatalf("Failed to create component report cache: %v", err) - } - defer cache.Close() - - err = cache.AddTestRegressionsToReport(testRegressions) - if err != nil { - t.Fatalf("Failed to add test regressions to component report: %v", err) - } - t.Run("find potential matching regressions", func(t *testing.T) { defer cleanupAllTriages(dbc) - // Create a triage with two linked regressions + // Use real regressions from seed data that appear in the component report. + // Synthetic regressions with fake test IDs won't appear in the report, so + // GetTriagePotentialMatches would skip them entirely. + realRegressions := getSeedDataRegressions(t) + require.GreaterOrEqual(t, len(realRegressions), 3, "seed data should produce at least 3 regressions") + + // Create a triage linked to the first real regression triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-1234", Type: models.TriageTypeProduct, Regressions: []models.TestRegression{ - {ID: testRegressions[0].Regression.ID}, // TestSomething with commonFailureTime - {ID: testRegressions[1].Regression.ID}, // TestAnother with unique failure time + {ID: realRegressions[0].ID}, }, } var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) - require.Equal(t, 2, len(triageResponse.Regressions)) + require.Equal(t, 1, len(triageResponse.Regressions)) - // Query for potential matches + // Query for potential matches — the other real regressions should appear var potentialMatches []componentreadiness.PotentialMatchingRegression - endpoint := fmt.Sprintf("/api/component_readiness/triages/%d/matches?baseRelease=%s&sampleRelease=%s", triageResponse.ID, view.BaseRelease.Release.Name, view.SampleRelease.Release.Name) err = util.SippyGet(endpoint, &potentialMatches) require.NoError(t, err) - // Verify the results - assert.True(t, len(potentialMatches) > 0, "Should find some potential matches") + // Should find some potential matches from the other real regressions + assert.True(t, len(potentialMatches) > 0, "Should find potential matches from real seed data regressions") - // Verify HATEOAS links are present in potential match responses + // Verify HATEOAS links are present on matches baseURL := fmt.Sprintf("http://%s:%s", os.Getenv("SIPPY_ENDPOINT"), os.Getenv("SIPPY_API_PORT")) for _, match := range potentialMatches { assert.Equal(t, fmt.Sprintf("%s/api/component_readiness/triages/%d/matches", baseURL, triageResponse.ID), @@ -1061,118 +1057,23 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { match.Links["triage"], "Potential match should have triage link") } - // Verify status values are correctly returned for the potential matches - statusMap := make(map[uint]crtest.Status) + // Verify that the linked regression is NOT in the potential matches + foundRegressionIDs := make(map[uint]bool) for _, match := range potentialMatches { if match.RegressedTest.Regression != nil { - statusMap[match.RegressedTest.Regression.ID] = match.RegressedTest.TestComparison.ReportStatus + foundRegressionIDs[match.RegressedTest.Regression.ID] = true } } + assert.False(t, foundRegressionIDs[realRegressions[0].ID], "Linked regression should not appear in potential matches") - // Build maps for easier verification - foundRegressionIDs := make(map[uint]bool) - matchesBySimilarName := make(map[uint][]componentreadiness.SimilarlyNamedTest) - matchesBySameFailure := make(map[uint][]models.TestRegression) - confidenceLevels := make(map[uint]int) - + // Verify each match has valid regression data and a confidence level for _, match := range potentialMatches { - if match.RegressedTest.Regression == nil { - continue // Skip if no regression data - } - regressionID := match.RegressedTest.Regression.ID - foundRegressionIDs[regressionID] = true - confidenceLevels[regressionID] = match.ConfidenceLevel - if len(match.SimilarlyNamedTests) > 0 { - matchesBySimilarName[regressionID] = match.SimilarlyNamedTests - } - if len(match.SameLastFailures) > 0 { - matchesBySameFailure[regressionID] = match.SameLastFailures - } - } - - // Verify that linked regressions are NOT in the potential matches - assert.False(t, foundRegressionIDs[testRegressions[0].Regression.ID], "Linked regression 0 should not appear in potential matches") - assert.False(t, foundRegressionIDs[testRegressions[1].Regression.ID], "Linked regression 1 should not appear in potential matches") - - // Verify expected matches are found - assert.True(t, foundRegressionIDs[testRegressions[2].Regression.ID], "Should find regression 2 (similar name to TestSomething)") - assert.True(t, foundRegressionIDs[testRegressions[3].Regression.ID], "Should find regression 3 (same failure time)") - assert.True(t, foundRegressionIDs[testRegressions[4].Regression.ID], "Should find regression 4 (both similar name and same failure)") - assert.True(t, foundRegressionIDs[testRegressions[5].Regression.ID], "Should find regression 5 (similar name)") - assert.True(t, foundRegressionIDs[testRegressions[8].Regression.ID], "Should find regression 8 (same failure time)") - assert.True(t, foundRegressionIDs[testRegressions[9].Regression.ID], "Should find regression 9 (similar name to TestAnother)") - - // Verify the status values are correctly returned - assert.Equal(t, crtest.ExtremeTriagedRegression, statusMap[testRegressions[2].Regression.ID], "Regression 2 should have ExtremeTriagedRegression status") - assert.Equal(t, crtest.SignificantTriagedRegression, statusMap[testRegressions[3].Regression.ID], "Regression 3 should have SignificantTriagedRegression status") - assert.Equal(t, crtest.FixedRegression, statusMap[testRegressions[4].Regression.ID], "Regression 4 should have FixedRegression status") - assert.Equal(t, crtest.MissingSample, statusMap[testRegressions[5].Regression.ID], "Regression 5 should have MissingSample status") - assert.Equal(t, crtest.MissingBasisAndSample, statusMap[testRegressions[8].Regression.ID], "Regression 8 should have MissingBasisAndSample status") - assert.Equal(t, crtest.SignificantImprovement, statusMap[testRegressions[9].Regression.ID], "Regression 9 should have SignificantImprovement status") - - // Verify non-matches are not found - assert.False(t, foundRegressionIDs[testRegressions[6].Regression.ID], "Should not find regression 6 (no match)") - assert.False(t, foundRegressionIDs[testRegressions[7].Regression.ID], "Should not find regression 7 (name too different)") - - // Verify match reasons are correct - - // Regression 2: Should match by similar name to "TestSomething" - if assert.Contains(t, matchesBySimilarName, testRegressions[2].Regression.ID) { - matches := matchesBySimilarName[testRegressions[2].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].Regression.ID, "Should match against TestSomething regression") - // TestSomthng vs TestSomething = edit distance 2, so score = 6-2 = 4 - assert.Equal(t, 4, confidenceLevels[testRegressions[2].Regression.ID], "Confidence should be 4 (edit distance 2: 6-2)") - } - - // Regression 3: Should match by same failure time - if assert.Contains(t, matchesBySameFailure, testRegressions[3].Regression.ID) { - matches := matchesBySameFailure[testRegressions[3].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one same failure time") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].ID, "Should match against commonFailureTime regression") - assert.Equal(t, 1, confidenceLevels[testRegressions[3].Regression.ID], "Confidence should be 1 (1 failure match * 1)") - } - - // Regression 4: Should match both similar name AND same failure time - if assert.Contains(t, matchesBySimilarName, testRegressions[4].Regression.ID) { - nameMatches := matchesBySimilarName[testRegressions[4].Regression.ID] - assert.Equal(t, 1, len(nameMatches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[1].Regression.ID, nameMatches[0].Regression.ID, "Should match against TestAnotherOne regression") - } - if assert.Contains(t, matchesBySameFailure, testRegressions[4].Regression.ID) { - failureMatches := matchesBySameFailure[testRegressions[4].Regression.ID] - assert.Equal(t, 1, len(failureMatches), "Should match exactly one same failure time") - assert.Equal(t, testRegressions[0].Regression.ID, failureMatches[0].ID, "Should match against commonFailureTime regression") - // TestAnoterOne vs TestAnotherOne = edit distance 1, so name score = 6-1 = 5, failure = 1, total = 6 - assert.Equal(t, 6, confidenceLevels[testRegressions[4].Regression.ID], "Confidence should be 6 (name edit distance 1: 6-1=5, plus 1 failure match)") - } - - // Regression 5: Should match by similar name only - if assert.Contains(t, matchesBySimilarName, testRegressions[5].Regression.ID) { - matches := matchesBySimilarName[testRegressions[5].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].Regression.ID, "Should match against TestSomething regression") - // TestSomthing vs TestSomething = edit distance 1, so score = 6-1 = 5 - assert.Equal(t, 5, confidenceLevels[testRegressions[5].Regression.ID], "Confidence should be 5 (edit distance 1: 6-1)") - } - assert.NotContains(t, matchesBySameFailure, testRegressions[5].Regression.ID, "Should not match by failure time") - - // Regression 8: Should match by same failure time only - if assert.Contains(t, matchesBySameFailure, testRegressions[8].Regression.ID) { - matches := matchesBySameFailure[testRegressions[8].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one same failure time") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].ID, "Should match against commonFailureTime regression") - assert.Equal(t, 1, confidenceLevels[testRegressions[8].Regression.ID], "Confidence should be 1 (1 failure match * 1)") - } - assert.NotContains(t, matchesBySimilarName, testRegressions[8].Regression.ID, "Should not match by similar name") - - // Regression 9: Should match by similar name to "TestAnotherOne" - if assert.Contains(t, matchesBySimilarName, testRegressions[9].Regression.ID) { - matches := matchesBySimilarName[testRegressions[9].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[1].Regression.ID, matches[0].Regression.ID, "Should match against TestAnotherOne regression") - // TestAnotheOne vs TestAnotherOne = edit distance 1, so score = 6-1 = 5 - assert.Equal(t, 5, confidenceLevels[testRegressions[9].Regression.ID], "Confidence should be 5 (edit distance 1: 6-1)") + require.NotNil(t, match.RegressedTest.Regression, "matched regression should not be nil") + assert.Greater(t, match.ConfidenceLevel, 0, "confidence level should be positive") + // Each match should have at least one reason (similar name or same failure time) + assert.True(t, + len(match.SimilarlyNamedTests) > 0 || len(match.SameLastFailures) > 0, + "match for regression %d should have at least one match reason", match.RegressedTest.Regression.ID) } }) @@ -1234,7 +1135,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { }) t.Run("error when triage not found", func(t *testing.T) { - var potentialMatches []interface{} + var potentialMatches []any endpoint := "/api/component_readiness/triages/999999/matches" err := util.SippyGet(endpoint, &potentialMatches) @@ -1244,35 +1145,42 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { t.Run("verify status values in triage responses", func(t *testing.T) { defer cleanupAllTriages(dbc) - // Create a triage with regressions that have different status values + // Use real regressions that appear in the component report so we can verify + // status transformation via the expand endpoint + realRegressions := getSeedDataRegressions(t) + require.GreaterOrEqual(t, len(realRegressions), 2, "seed data should produce at least 2 regressions") + triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-5678", Type: models.TriageTypeProduct, Regressions: []models.TestRegression{ - {ID: testRegressions[0].Regression.ID}, // ExtremeRegression - {ID: testRegressions[1].Regression.ID}, // SignificantRegression - {ID: testRegressions[4].Regression.ID}, // FixedRegression + {ID: realRegressions[0].ID}, + {ID: realRegressions[1].ID}, }, } var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) - require.Equal(t, 3, len(triageResponse.Regressions)) - - // Note: TestComparison (including status) is not available on the basic TestRegression model - // returned by the triage API. Status is only available in the potential matches endpoint - // where regressions are represented as ReportTestSummary with full component report data. - // - // However, we can verify that our test setup correctly created regressions with different IDs - regressionIDs := make(map[uint]bool) - for _, regression := range triageResponse.Regressions { - regressionIDs[regression.ID] = true - } + require.Equal(t, 2, len(triageResponse.Regressions)) + + // Use the expand endpoint to get status values from the component report + var expandedTriage sippyserver.ExpandedTriage + err = util.SippyGet(fmt.Sprintf("/api/component_readiness/triages/%d?view=%s-main&expand=regressions", triageResponse.ID, util.Release), &expandedTriage) + require.NoError(t, err) - assert.True(t, regressionIDs[testRegressions[0].Regression.ID], "Should find first regression") - assert.True(t, regressionIDs[testRegressions[1].Regression.ID], "Should find second regression") - assert.True(t, regressionIDs[testRegressions[4].Regression.ID], "Should find third regression") + regressedTests := expandedTriage.RegressedTests[view.Name] + require.NotEmpty(t, regressedTests, "expanded triage should contain regressed tests") + + // Verify each regressed test has a triaged status (the triage transforms the status) + for _, rt := range regressedTests { + require.NotNil(t, rt, "regressed test should not be nil") + require.NotNil(t, rt.Regression, "regressed test should have regression data") + status := rt.TestComparison.ReportStatus + assert.True(t, + status == crtest.ExtremeTriagedRegression || status == crtest.SignificantTriagedRegression, + "regression %s status should be triaged, got %d", rt.Regression.TestID, status) + } }) } diff --git a/test/e2e/datasync/datasync_test.go b/test/e2e/datasync/datasync_test.go new file mode 100644 index 0000000000..00f6b45ff3 --- /dev/null +++ b/test/e2e/datasync/datasync_test.go @@ -0,0 +1,54 @@ +package datasync + +import ( + "os" + "os/exec" + "testing" + + "github.com/openshift/sippy/test/e2e/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDataSync(t *testing.T) { + if os.Getenv("GCS_SA_JSON_PATH") == "" { + t.Skip("GCS_SA_JSON_PATH not set, skipping data sync test") + } + + dbc := util.CreateE2EPostgresConnection(t) + + // Count prow_job_runs before sync to compare after + var countBefore int64 + dbc.DB.Table("prow_job_runs").Count(&countBefore) + t.Logf("prow_job_runs before sync: %d", countBefore) + + // SIPPY_E2E_REPO_ROOT is set by e2e.sh to the repo root where the sippy + // binary and config files live. + repoRoot := os.Getenv("SIPPY_E2E_REPO_ROOT") + require.NotEmpty(t, repoRoot, "SIPPY_E2E_REPO_ROOT must be set") + + // Run sippy load with minimal scope: just prow loader, single release, + // last 2 hours of data only + cmd := exec.Command(repoRoot+"/sippy", "load", + "--loader", "prow", + "--release", util.Release, + "--prow-load-since", "2h", + "--config", "config/e2e-openshift.yaml", + "--google-service-account-credential-file", os.Getenv("GCS_SA_JSON_PATH"), + "--database-dsn", os.Getenv("SIPPY_E2E_DSN"), + "--skip-matview-refresh", + "--log-level", "debug", + ) + cmd.Dir = repoRoot + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + require.NoError(t, err, "sippy load command should complete without error") + + // Verify some real prow job runs were loaded (with real job names from e2e-openshift.yaml) + var countAfter int64 + dbc.DB.Table("prow_job_runs").Count(&countAfter) + t.Logf("prow_job_runs after sync: %d (loaded %d new)", countAfter, countAfter-countBefore) + assert.Greater(t, countAfter, countBefore, "sync should have loaded new prow job runs") +} diff --git a/test/e2e/util/cache_manipulator.go b/test/e2e/util/cache_manipulator.go deleted file mode 100644 index 416095fdf0..0000000000 --- a/test/e2e/util/cache_manipulator.go +++ /dev/null @@ -1,238 +0,0 @@ -package util - -import ( - "encoding/json" - "fmt" - "os" - "strings" - "time" - - "gopkg.in/redis.v5" - - "github.com/openshift/sippy/pkg/api/componentreadiness" - "github.com/openshift/sippy/pkg/apis/api/componentreport" - "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" - log "github.com/sirupsen/logrus" -) - -// E2ECacheManipulator provides utilities for manipulating values in the Redis cache -type E2ECacheManipulator struct { - release string - client *redis.Client -} - -func NewE2ECacheManipulator(release string) (*E2ECacheManipulator, error) { - client, err := connectToRedis() - if err != nil { - return nil, err - } - return &E2ECacheManipulator{ - release: release, - client: client, - }, nil -} - -func (c *E2ECacheManipulator) Close() { - if c.client != nil { - c.client.Close() - } -} - -// connectToRedis creates a Redis client connection -func connectToRedis() (*redis.Client, error) { - redisURL := os.Getenv("REDIS_URL") - if redisURL == "" { - redisURL = "redis://localhost:23479" // Default for e2e tests - } - - opts, err := redis.ParseURL(redisURL) - if err != nil { - return nil, fmt.Errorf("failed to parse Redis URL: %w", err) - } - - client := redis.NewClient(opts) - - // Test Redis connection - if err := client.Ping().Err(); err != nil { - client.Close() - return nil, fmt.Errorf("failed to connect to Redis: %w", err) - } - - return client, nil -} - -// AddTestRegressionsToReport adds test regressions to the component report in cache -// so they can be found by GetTriagePotentialMatches function -func (c *E2ECacheManipulator) AddTestRegressionsToReport(testRegressions []componentreport.ReportTestSummary) error { - // Get the current component report directly from the cache - report, cacheKey, err := c.GetReport() - if err != nil { - return fmt.Errorf("failed to get component report from cache: %w", err) - } - - // Add each test regression to the appropriate place in the report - for _, testRegression := range testRegressions { - if testRegression.Regression == nil { - continue // Skip if no regression data - } - - // Find or create the appropriate row for this component - var targetRowIndex = -1 - for i, row := range report.Rows { - if row.Component == testRegression.Component { - targetRowIndex = i - break - } - } - - // If no row exists for this component, create one - if targetRowIndex == -1 { - newRow := componentreport.ReportRow{ - RowIdentification: crtest.RowIdentification{ - Component: testRegression.Component, - Capability: testRegression.Capability, - }, - Columns: []componentreport.ReportColumn{ - { - ColumnIdentification: crtest.ColumnIdentification{ - Variants: testRegression.Variants, - }, - Status: crtest.SignificantRegression, - RegressedTests: []componentreport.ReportTestSummary{testRegression}, - }, - }, - } - report.Rows = append(report.Rows, newRow) - } else { - // Add to existing row - find or create a matching column - row := &report.Rows[targetRowIndex] - var targetColumnIndex = -1 - - // Look for a column with matching variants - for i, col := range row.Columns { - if variantsMatch(col.Variants, testRegression.Variants) { - targetColumnIndex = i - break - } - } - - if targetColumnIndex == -1 { - // Create new column - newColumn := componentreport.ReportColumn{ - ColumnIdentification: crtest.ColumnIdentification{ - Variants: testRegression.Variants, - }, - Status: crtest.SignificantRegression, - RegressedTests: []componentreport.ReportTestSummary{testRegression}, - } - row.Columns = append(row.Columns, newColumn) - } else { - // Add to existing column - row.Columns[targetColumnIndex].RegressedTests = append(row.Columns[targetColumnIndex].RegressedTests, testRegression) - } - } - } - - // Update the cached component report so GetTriagePotentialMatches can find the test regressions - err = c.updateReportWithKey(report, cacheKey) - if err != nil { - log.WithError(err).Warn("Failed to update cached component report, test may not work as expected") - } - - return nil -} - -// GetReport retrieves the component report directly from Redis cache -func (c *E2ECacheManipulator) GetReport() (componentreport.ComponentReport, string, error) { - // Find the ComponentReport cache key by scanning for keys that start with "ComponentReport~" - keyPattern := "_SIPPY_*ComponentReport~*" - keys, err := c.client.Keys(keyPattern).Result() - if err != nil { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to scan for ComponentReport keys: %w", err) - } - - var cacheKey string - for _, key := range keys { - // Strip the prefixes to get the JSON part - // Key format: "_SIPPY_cc:ComponentReport~{JSON}" or "_SIPPY_ComponentReport~{JSON}" - jsonPart := key - - // Remove "_SIPPY_" prefix if present - jsonPart = strings.TrimPrefix(jsonPart, "_SIPPY_") - - // Remove "cc:" prefix if present (compressed cache prefix) - jsonPart = strings.TrimPrefix(jsonPart, "cc:") - - // Remove "ComponentReport~" prefix - if !strings.HasPrefix(jsonPart, "ComponentReport~") { - log.Warnf("Unexpected cache key format, missing ComponentReport~ prefix: %s", key) - continue - } - jsonPart = jsonPart[len("ComponentReport~"):] - - gk := &componentreadiness.GeneratorCacheKey{} - if err := json.Unmarshal([]byte(jsonPart), gk); err != nil { - log.Warnf("Failed to unmarshal ComponentReport key JSON '%s' from key '%s': %v", jsonPart, key, err) - continue - } - if gk.SampleRelease.Name == Release { - cacheKey = key - break - } - } - if cacheKey == "" { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to find proper ComponentReport key") - } - log.Debugf("Found ComponentReport cache key: %s", cacheKey) - - // Get the cached data - cachedData, err := c.client.Get(cacheKey).Bytes() - if err != nil { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to get cached data for key %s: %w", cacheKey, err) - } - - // Unmarshal the component report - var report componentreport.ComponentReport - err = json.Unmarshal(cachedData, &report) - if err != nil { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to unmarshal component report: %w", err) - } - - log.Debugf("Retrieved cached component report with %d rows", len(report.Rows)) - return report, cacheKey, nil -} - -// updateReportWithKey updates the Redis cache with the modified component report using the exact cache key -func (c *E2ECacheManipulator) updateReportWithKey(report componentreport.ComponentReport, cacheKey string) error { - if cacheKey == "" { - return fmt.Errorf("cache key is empty, cannot update cache") - } - - // Marshal the modified report to JSON - reportJSON, err := json.Marshal(report) - if err != nil { - return fmt.Errorf("failed to marshal component report: %w", err) - } - - // Store plain JSON in Redis cache with a reasonable expiration (1 hour) - err = c.client.Set(cacheKey, reportJSON, time.Hour).Err() - if err != nil { - return fmt.Errorf("failed to store in Redis cache: %w", err) - } - - log.Debugf("Updated cached component report with key %s, %d rows", cacheKey, len(report.Rows)) - return nil -} - -// variantsMatch checks if two variant maps are equivalent -func variantsMatch(a, b map[string]string) bool { - if len(a) != len(b) { - return false - } - for k, v := range a { - if b[k] != v { - return false - } - } - return true -} diff --git a/test/e2e/util/e2erequest.go b/test/e2e/util/e2erequest.go index aca9bb91cb..98ce6638bb 100644 --- a/test/e2e/util/e2erequest.go +++ b/test/e2e/util/e2erequest.go @@ -13,8 +13,8 @@ import ( const ( // Needs to match what we import in the e2e.sh script - Release = "4.20" - BaseRelease = "4.19" + Release = "4.22" + BaseRelease = "4.21" // APIPort is the port e2e.sh launches the sippy API on. These values must be kept in sync. APIPort = 18080 From 2bce9f94dc36373d95ea15d497cc09d1fd3494f7 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 11:54:23 -0400 Subject: [PATCH 02/25] Fix lint: gofmt, gosec, gocyclo, gocritic warnings - Fix gofmt formatting in interface.go, provider.go, types.go, releasefallback.go, report_test.go - Fix gosec G204: add #nosec for exec.Command in e2e test - Fix gosec G306: use 0o600 permissions for WriteFile - Fix gocritic typeAssertChain: rewrite to type switch - Fix gocyclo: extract seedProwJobs, seedTestsAndOwnerships, seedJobRunsAndResults, seedRunsForJob from seedSyntheticData Co-Authored-By: Claude Opus 4.6 --- cmd/sippy/seed_data.go | 302 +++++++++--------- .../dataprovider/interface.go | 2 +- .../dataprovider/postgres/provider.go | 32 +- .../releasefallback/releasefallback.go | 1 - .../query/querygenerators.go | 9 +- .../api/componentreport/testdetails/types.go | 4 +- .../componentreadiness/report/report_test.go | 6 +- test/e2e/datasync/datasync_test.go | 2 +- 8 files changed, 187 insertions(+), 171 deletions(-) diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index e9533398b0..ba15b1f2b2 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -392,13 +392,46 @@ func seedSyntheticData(dbc *db.DB) error { return nil } - // 1. Create test suite if err := createTestSuite(dbc, "synthetic"); err != nil { return errors.WithMessage(err, "failed to create test suite") } log.Info("Created test suite 'synthetic'") - // 2. Create ProwJobs for all releases + if err := seedProwJobs(dbc); err != nil { + return err + } + + if err := seedTestsAndOwnerships(dbc); err != nil { + return err + } + + totalRuns, totalResults, err := seedJobRunsAndResults(dbc) + if err != nil { + return err + } + + if err := createLabelsAndSymptoms(dbc); err != nil { + return errors.WithMessage(err, "failed to create labels and symptoms") + } + + if err := writeSyntheticViewsFile(); err != nil { + return errors.WithMessage(err, "failed to write views file") + } + + log.Info("Refreshing materialized views...") + sippyserver.RefreshData(dbc, nil, false) + + log.Info("Syncing regressions...") + if err := syncRegressions(dbc); err != nil { + log.WithError(err).Warn("failed to sync regressions") + } + + log.Infof("Seeded synthetic data: %d ProwJobRuns, %d test results across %d releases", + totalRuns, totalResults, len(syntheticReleases)) + return nil +} + +func seedProwJobs(dbc *db.DB) error { for _, release := range syntheticReleases { for _, job := range syntheticJobs { name := fmt.Sprintf(job.nameTemplate, release) @@ -416,20 +449,22 @@ func seedSyntheticData(dbc *db.DB) error { } } log.Infof("Created ProwJobs for %d releases x %d jobs", len(syntheticReleases), len(syntheticJobs)) + return nil +} + +type testInfo struct { + name string + uniqueID string + component string + capabilities []string +} - // 3. Create Tests and TestOwnerships +func seedTestsAndOwnerships(dbc *db.DB) error { var suite models.Suite if err := dbc.DB.Where("name = ?", "synthetic").First(&suite).Error; err != nil { return fmt.Errorf("failed to find suite: %w", err) } - // Collect unique tests - type testInfo struct { - name string - uniqueID string - component string - capabilities []string - } seenTests := map[string]testInfo{} for _, ts := range syntheticTests { if _, ok := seenTests[ts.testName]; !ok { @@ -443,14 +478,12 @@ func seedSyntheticData(dbc *db.DB) error { } for _, info := range seenTests { - // Create test testModel := models.Test{Name: info.name} var existingTest models.Test if err := dbc.DB.Where("name = ?", info.name).FirstOrCreate(&existingTest, testModel).Error; err != nil { return fmt.Errorf("failed to create Test %s: %w", info.name, err) } - // Create test ownership ownership := models.TestOwnership{ UniqueID: info.uniqueID, Name: info.name, @@ -466,18 +499,20 @@ func seedSyntheticData(dbc *db.DB) error { } } log.Infof("Created %d tests with ownership records", len(seenTests)) + return nil +} + +type jobReleaseKey struct { + jobTemplate string + release string +} - // 4. Create deterministic ProwJobRuns and test results - // - // Strategy: for each (job template, release), determine the max run count needed - // across all tests assigned to that job+release. Create that many shared runs. - // Then assign test results per test with exact counts. - type jobReleaseKey struct { - jobTemplate string - release string +func seedJobRunsAndResults(dbc *db.DB) (int, int, error) { + var suite models.Suite + if err := dbc.DB.Where("name = ?", "synthetic").First(&suite).Error; err != nil { + return 0, 0, fmt.Errorf("failed to find suite: %w", err) } - // Compute max runs needed per job+release maxRuns := map[jobReleaseKey]int{} for _, ts := range syntheticTests { for jobTpl, releaseCounts := range ts.jobCounts { @@ -490,17 +525,15 @@ func seedSyntheticData(dbc *db.DB) error { } } - // Load all tests by name for ID lookup testIDsByName := map[string]uint{} var allTests []models.Test if err := dbc.DB.Find(&allTests).Error; err != nil { - return fmt.Errorf("failed to fetch tests: %w", err) + return 0, 0, fmt.Errorf("failed to fetch tests: %w", err) } for _, t := range allTests { testIDsByName[t.Name] = t.ID } - // Create runs and test results totalRuns := 0 totalResults := 0 for jrKey, runCount := range maxRuns { @@ -508,148 +541,132 @@ func seedSyntheticData(dbc *db.DB) error { continue } - // Look up the ProwJob jobName := fmt.Sprintf(jrKey.jobTemplate, jrKey.release) var prowJob models.ProwJob if err := dbc.DB.Where("name = ?", jobName).First(&prowJob).Error; err != nil { - return fmt.Errorf("failed to find ProwJob %s: %w", jobName, err) + return 0, 0, fmt.Errorf("failed to find ProwJob %s: %w", jobName, err) } - // Create runs spread across the release time window. - // Reserve last 2 runs as infra failures (no test results). - start, end := releaseTimeWindow(jrKey.release) - window := end.Sub(start) - interval := window / time.Duration(runCount) - - runIDs := make([]uint, runCount) - for i := 0; i < runCount; i++ { - timestamp := start.Add(time.Duration(i) * interval) - run := models.ProwJobRun{ - ProwJobID: prowJob.ID, - Cluster: "build01", - Timestamp: timestamp, - Duration: 3 * time.Hour, - } - if err := dbc.DB.Create(&run).Error; err != nil { - return fmt.Errorf("failed to create ProwJobRun: %w", err) - } - runIDs[i] = run.ID - totalRuns++ + runs, results, err := seedRunsForJob(dbc, &suite, prowJob, jrKey, runCount, testIDsByName) + if err != nil { + return 0, 0, err } + totalRuns += runs + totalResults += results - // Runs that get test results (all except the last 2) - testableRuns := runCount - if testableRuns > 2 { - testableRuns = runCount - 2 - } + log.Debugf("Created %d runs for %s", runCount, jobName) + } - // Track which runs have at least one test failure - runsWithFailure := map[uint]bool{} + return totalRuns, totalResults, nil +} - // Assign test results to testable runs only - for _, ts := range syntheticTests { - releaseCounts, hasJob := ts.jobCounts[jrKey.jobTemplate] - if !hasJob { - continue - } - counts, hasRelease := releaseCounts[jrKey.release] - if !hasRelease || counts.total == 0 { - continue - } +func seedRunsForJob(dbc *db.DB, suite *models.Suite, prowJob models.ProwJob, jrKey jobReleaseKey, runCount int, testIDsByName map[string]uint) (int, int, error) { + start, end := releaseTimeWindow(jrKey.release) + window := end.Sub(start) + interval := window / time.Duration(runCount) + + runIDs := make([]uint, runCount) + for i := range runCount { + timestamp := start.Add(time.Duration(i) * interval) + run := models.ProwJobRun{ + ProwJobID: prowJob.ID, + Cluster: "build01", + Timestamp: timestamp, + Duration: 3 * time.Hour, + } + if err := dbc.DB.Create(&run).Error; err != nil { + return 0, 0, fmt.Errorf("failed to create ProwJobRun: %w", err) + } + runIDs[i] = run.ID + } - testID, ok := testIDsByName[ts.testName] - if !ok { - return fmt.Errorf("test %q not found in DB", ts.testName) - } + // Runs that get test results (all except the last 2) + testableRuns := runCount + if testableRuns > 2 { + testableRuns = runCount - 2 + } - for i := 0; i < counts.total && i < testableRuns; i++ { - var status int - switch { - case i < counts.success-counts.flake: - status = 1 // pass - case i < counts.success: - status = 13 // flake (counts as success too) - default: - status = 12 // failure - runsWithFailure[runIDs[i]] = true - } + runsWithFailure := map[uint]bool{} + totalResults := 0 - result := models.ProwJobRunTest{ - ProwJobRunID: runIDs[i], - TestID: testID, - SuiteID: &suite.ID, - Status: status, - Duration: 5.0, - CreatedAt: start.Add(time.Duration(i) * interval), - } - if err := dbc.DB.Create(&result).Error; err != nil { - return fmt.Errorf("failed to create ProwJobRunTest: %w", err) - } - totalResults++ - } + for _, ts := range syntheticTests { + releaseCounts, hasJob := ts.jobCounts[jrKey.jobTemplate] + if !hasJob { + continue + } + counts, hasRelease := releaseCounts[jrKey.release] + if !hasRelease || counts.total == 0 { + continue } - // Set OverallResult on all runs - for i, runID := range runIDs { - var overallResult v1.JobOverallResult - var succeeded, failed bool - - if i >= testableRuns { - // Last 2 runs: infra failure, no tests ran - overallResult = v1.JobInternalInfrastructureFailure - failed = true - } else if runsWithFailure[runID] { - overallResult = v1.JobTestFailure - failed = true - } else { - overallResult = v1.JobSucceeded - succeeded = true - } + testID, ok := testIDsByName[ts.testName] + if !ok { + return 0, 0, fmt.Errorf("test %q not found in DB", ts.testName) + } - if err := dbc.DB.Model(&models.ProwJobRun{}).Where("id = ?", runID). - Updates(map[string]interface{}{ - "overall_result": overallResult, - "succeeded": succeeded, - "failed": failed, - }).Error; err != nil { - return fmt.Errorf("failed to update ProwJobRun result: %w", err) + for i := 0; i < counts.total && i < testableRuns; i++ { + var status int + switch { + case i < counts.success-counts.flake: + status = 1 // pass + case i < counts.success: + status = 13 // flake (counts as success too) + default: + status = 12 // failure + runsWithFailure[runIDs[i]] = true } - } - // Update test_failures count - if err := dbc.DB.Exec(` - UPDATE prow_job_runs SET test_failures = COALESCE(( - SELECT COUNT(*) FROM prow_job_run_tests - WHERE prow_job_run_id = prow_job_runs.id AND status = 12 - ), 0) WHERE prow_job_id = ?`, prowJob.ID).Error; err != nil { - log.WithError(err).Warn("failed to update test_failures count") + result := models.ProwJobRunTest{ + ProwJobRunID: runIDs[i], + TestID: testID, + SuiteID: &suite.ID, + Status: status, + Duration: 5.0, + CreatedAt: start.Add(time.Duration(i) * interval), + } + if err := dbc.DB.Create(&result).Error; err != nil { + return 0, 0, fmt.Errorf("failed to create ProwJobRunTest: %w", err) + } + totalResults++ } - - log.Debugf("Created %d runs for %s", runCount, jobName) } - // Create labels and symptoms - if err := createLabelsAndSymptoms(dbc); err != nil { - return errors.WithMessage(err, "failed to create labels and symptoms") - } + // Set OverallResult on all runs + for i, runID := range runIDs { + var overallResult v1.JobOverallResult + var succeeded, failed bool + + if i >= testableRuns { + overallResult = v1.JobInternalInfrastructureFailure + failed = true + } else if runsWithFailure[runID] { + overallResult = v1.JobTestFailure + failed = true + } else { + overallResult = v1.JobSucceeded + succeeded = true + } - // Generate views file with include_variants matching our seed data - if err := writeSyntheticViewsFile(); err != nil { - return errors.WithMessage(err, "failed to write views file") + if err := dbc.DB.Model(&models.ProwJobRun{}).Where("id = ?", runID). + Updates(map[string]any{ + "overall_result": overallResult, + "succeeded": succeeded, + "failed": failed, + }).Error; err != nil { + return 0, 0, fmt.Errorf("failed to update ProwJobRun result: %w", err) + } } - log.Info("Refreshing materialized views...") - sippyserver.RefreshData(dbc, nil, false) - - // Run regression tracker to populate test_regressions table - log.Info("Syncing regressions...") - if err := syncRegressions(dbc); err != nil { - log.WithError(err).Warn("failed to sync regressions") + // Update test_failures count + if err := dbc.DB.Exec(` + UPDATE prow_job_runs SET test_failures = COALESCE(( + SELECT COUNT(*) FROM prow_job_run_tests + WHERE prow_job_run_id = prow_job_runs.id AND status = 12 + ), 0) WHERE prow_job_id = ?`, prowJob.ID).Error; err != nil { + log.WithError(err).Warn("failed to update test_failures count") } - log.Infof("Seeded synthetic data: %d ProwJobRuns, %d test results across %d releases", - totalRuns, totalResults, len(syntheticReleases)) - return nil + return runCount, totalResults, nil } func syncRegressions(dbc *db.DB) error { @@ -753,7 +770,7 @@ func writeSyntheticViewsFile() error { return fmt.Errorf("marshaling views: %w", err) } - if err := os.WriteFile(syntheticViewsFile, data, 0644); err != nil { + if err := os.WriteFile(syntheticViewsFile, data, 0o600); err != nil { return fmt.Errorf("writing %s: %w", syntheticViewsFile, err) } @@ -896,4 +913,3 @@ func createLabelsAndSymptoms(dbc *db.DB) error { return nil } - diff --git a/pkg/api/componentreadiness/dataprovider/interface.go b/pkg/api/componentreadiness/dataprovider/interface.go index b207c8eb32..106309e02d 100644 --- a/pkg/api/componentreadiness/dataprovider/interface.go +++ b/pkg/api/componentreadiness/dataprovider/interface.go @@ -4,8 +4,8 @@ import ( "context" "time" - "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index a2c84d318c..73ada4e14d 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -14,8 +14,8 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" @@ -159,10 +159,10 @@ func (p *PostgresProvider) QueryReleases(_ context.Context) ([]v1.Release, error caps := map[v1.ReleaseCapability]bool{ v1.ComponentReadinessCap: true, - v1.FeatureGatesCap: true, - v1.MetricsCap: true, - v1.PayloadTagsCap: true, - v1.SippyClassicCap: true, + v1.FeatureGatesCap: true, + v1.MetricsCap: true, + v1.PayloadTagsCap: true, + v1.SippyClassicCap: true, } now := time.Now().UTC() @@ -474,17 +474,17 @@ func (p *PostgresProvider) QuerySampleTestStatus(_ context.Context, reqOptions r // --- TestDetailsQuerier --- type testDetailRow struct { - TestID string `gorm:"column:test_id"` - TestName string `gorm:"column:test_name"` - ProwJobName string `gorm:"column:prowjob_name"` - ProwJobRunID string `gorm:"column:prowjob_run_id"` - ProwJobURL string `gorm:"column:prowjob_url"` - ProwJobStart time.Time `gorm:"column:prowjob_start"` - ProwJobID uint `gorm:"column:prow_job_id"` - Status int `gorm:"column:status"` - JiraComponent string `gorm:"column:jira_component"` - JiraComponentID *uint `gorm:"column:jira_component_id"` - Capabilities pq.StringArray `gorm:"column:capabilities;type:text[]"` + TestID string `gorm:"column:test_id"` + TestName string `gorm:"column:test_name"` + ProwJobName string `gorm:"column:prowjob_name"` + ProwJobRunID string `gorm:"column:prowjob_run_id"` + ProwJobURL string `gorm:"column:prowjob_url"` + ProwJobStart time.Time `gorm:"column:prowjob_start"` + ProwJobID uint `gorm:"column:prow_job_id"` + Status int `gorm:"column:status"` + JiraComponent string `gorm:"column:jira_component"` + JiraComponentID *uint `gorm:"column:jira_component_id"` + Capabilities pq.StringArray `gorm:"column:capabilities;type:text[]"` } const testDetailQuery = ` diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go index 51dbbe96fd..4c2b11111d 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go @@ -468,7 +468,6 @@ func (f *fallbackTestQueryReleasesGenerator) getTestFallbackRelease(ctx context. return crstatus.ReportTestStatus{BaseStatus: baseStatus}, nil } - func newFallbackReleases() FallbackReleases { fb := FallbackReleases{ Releases: map[string]ReleaseTestMap{}, diff --git a/pkg/api/componentreadiness/query/querygenerators.go b/pkg/api/componentreadiness/query/querygenerators.go index aa679d3ca1..21dd6da166 100644 --- a/pkg/api/componentreadiness/query/querygenerators.go +++ b/pkg/api/componentreadiness/query/querygenerators.go @@ -1055,10 +1055,11 @@ func deserializeRowToJobRunTestReportStatus(row []bigquery.Value, schema bigquer cts.ProwJobURL = row[i].(string) } case col == "prowjob_start": - if dt, ok := row[i].(civil.DateTime); ok { - cts.StartTime = time.Date(dt.Date.Year, dt.Date.Month, dt.Date.Day, dt.Time.Hour, dt.Time.Minute, dt.Time.Second, dt.Time.Nanosecond, time.UTC) - } else if t, ok := row[i].(time.Time); ok { - cts.StartTime = t + switch v := row[i].(type) { + case civil.DateTime: + cts.StartTime = time.Date(v.Date.Year, v.Date.Month, v.Date.Day, v.Time.Hour, v.Time.Minute, v.Time.Second, v.Time.Nanosecond, time.UTC) + case time.Time: + cts.StartTime = v } case col == "test_id": cts.TestKey.TestID = row[i].(string) diff --git a/pkg/apis/api/componentreport/testdetails/types.go b/pkg/apis/api/componentreport/testdetails/types.go index 11166eb45f..0ff83d2dd3 100644 --- a/pkg/apis/api/componentreport/testdetails/types.go +++ b/pkg/apis/api/componentreport/testdetails/types.go @@ -103,8 +103,8 @@ type JobStats struct { } type JobRunStats struct { - JobURL string `json:"job_url"` - JobRunID string `json:"job_run_id"` + JobURL string `json:"job_url"` + JobRunID string `json:"job_run_id"` StartTime time.Time `json:"start_time"` // TestStats is the test stats from one particular job run. // For the majority of the tests, there is only one junit. But diff --git a/test/e2e/componentreadiness/report/report_test.go b/test/e2e/componentreadiness/report/report_test.go index bcd457d02a..43b5b06530 100644 --- a/test/e2e/componentreadiness/report/report_test.go +++ b/test/e2e/componentreadiness/report/report_test.go @@ -39,9 +39,9 @@ func setupProvider(t *testing.T) (*pgprovider.PostgresProvider, reqopts.RequestO DBGroupBy: sets.NewString("Architecture", "FeatureSet", "Installer", "Network", "Platform", "Suite", "Topology", "Upgrade", "LayeredProduct"), }, AdvancedOption: reqopts.Advanced{ - Confidence: 95, - PityFactor: 5, - MinimumFailure: 3, + Confidence: 95, + PityFactor: 5, + MinimumFailure: 3, IncludeMultiReleaseAnalysis: true, }, CacheOption: cache.RequestOptions{}, diff --git a/test/e2e/datasync/datasync_test.go b/test/e2e/datasync/datasync_test.go index 00f6b45ff3..65e4988552 100644 --- a/test/e2e/datasync/datasync_test.go +++ b/test/e2e/datasync/datasync_test.go @@ -29,7 +29,7 @@ func TestDataSync(t *testing.T) { // Run sippy load with minimal scope: just prow loader, single release, // last 2 hours of data only - cmd := exec.Command(repoRoot+"/sippy", "load", + cmd := exec.Command(repoRoot+"/sippy", "load", // #nosec G204 "--loader", "prow", "--release", util.Release, "--prow-load-since", "2h", From 33d7c36ee4b554e3fa0fe8ce919a8196b6d9612d Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 12:24:07 -0400 Subject: [PATCH 03/25] Fix lint and update CI e2e to use seed data + postgres provider CI e2e tests previously loaded real data from BigQuery/GCS via `sippy load`, which was slow and required GCS credentials. Switch to `sippy seed-data` with the postgres data provider, matching what scripts/e2e.sh already does. This makes CI e2e faster and removes the dependency on GCS credentials for the core test suite. - Update setup script to run seed-data instead of sippy load - Update test script to serve with --data-provider postgres - Remove GCS credential requirements from CI e2e scripts - Fix lint: gofmt, gosec (G204, G306), gocritic typeAssertChain, gocyclo (extract seedProwJobs, seedTestsAndOwnerships, seedJobRunsAndResults, seedRunsForJob from seedSyntheticData) Co-Authored-By: Claude Opus 4.6 --- .../sippy-e2e-sippy-e2e-setup-commands.sh | 53 ++++--------------- .../sippy-e2e-sippy-e2e-test-commands.sh | 27 +--------- 2 files changed, 13 insertions(+), 67 deletions(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh index 3d514297c8..ab60a3214d 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh @@ -6,11 +6,6 @@ # When running locally, the user has to define SIPPY_IMAGE. echo "The sippy CI image: ${SIPPY_IMAGE}" -# The GCS_CRED allows us to pull artifacts from GCS when importing prow jobs. -# Redefine GCS_CRED to use your own. -GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" -echo "The GCS cred is: ${GCS_CRED}" - # If you're using Openshift, we use oc, if you're using plain Kubernetes, # we use kubectl. # @@ -61,14 +56,6 @@ fi e2e_pause -echo "Checking for presense of GCS credentials ..." -if [ -f ${GCS_CRED} ]; then - ls -l ${GCS_CRED} -else - echo "Aborting: GCS credential file ${GCS_CRED} not found" - exit 1 -fi - echo "Starting postgres on cluster-pool cluster..." # Make the "postgres" namespace and pod. @@ -212,22 +199,15 @@ fi ${KUBECTL_CMD} -n sippy-e2e get po -o wide ${KUBECTL_CMD} -n sippy-e2e get svc,ep -# Get the gcs credentials out to the cluster-pool cluster. -# These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations). -# See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa -# - -${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred=$GCS_CRED -n sippy-e2e - # Get the registry credentials for all build farm clusters out to the cluster-pool cluster. ${KUBECTL_CMD} -n sippy-e2e create secret generic regcred --from-file=.dockerconfigjson=${DOCKERCONFIGJSON} --type=kubernetes.io/dockerconfigjson -# Make the "sippy loader" pod. +# Seed synthetic data into postgres. cat << END | ${KUBECTL_CMD} apply -f - apiVersion: batch/v1 kind: Job metadata: - name: sippy-load-job + name: sippy-seed-job namespace: sippy-e2e spec: template: @@ -243,20 +223,9 @@ spec: terminationMessagePolicy: File command: ["/bin/sh", "-c"] args: - - /bin/sippy load --init-database --log-level=debug --release 4.20 --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres --redis-url=redis://redis.sippy-e2e.svc.cluster.local:6379 --mode=ocp --config ./config/e2e-openshift.yaml --google-service-account-credential-file /tmp/secrets/gcs-cred - env: - - name: GCS_SA_JSON_PATH - value: /tmp/secrets/gcs-cred - volumeMounts: - - mountPath: /tmp/secrets - name: gcs-cred - readOnly: true + - /bin/sippy seed-data --init-database --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres imagePullSecrets: - name: regcred - volumes: - - name: gcs-cred - secret: - secretName: gcs-cred dnsPolicy: ClusterFirst restartPolicy: Never schedulerName: default-scheduler @@ -266,23 +235,23 @@ spec: END date -echo "Waiting for sippy loader job to finish ..." -${KUBECTL_CMD} -n sippy-e2e get job sippy-load-job -${KUBECTL_CMD} -n sippy-e2e describe job sippy-load-job +echo "Waiting for sippy seed job to finish ..." +${KUBECTL_CMD} -n sippy-e2e get job sippy-seed-job +${KUBECTL_CMD} -n sippy-e2e describe job sippy-seed-job # We set +e to avoid the script aborting before we can retrieve logs. set +e -echo "Waiting up to ${SIPPY_LOAD_TIMEOUT:=1200s} for the sippy-load-job to complete..." -${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-load-job --timeout ${SIPPY_LOAD_TIMEOUT} +echo "Waiting up to ${SIPPY_LOAD_TIMEOUT:=1200s} for the sippy-seed-job to complete..." +${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-seed-job --timeout ${SIPPY_LOAD_TIMEOUT} retVal=$? set -e -job_pod=$(${KUBECTL_CMD} -n sippy-e2e get pod --selector=job-name=sippy-load-job --output=jsonpath='{.items[0].metadata.name}') -${KUBECTL_CMD} -n sippy-e2e logs ${job_pod} > ${ARTIFACT_DIR}/sippy-load.log +job_pod=$(${KUBECTL_CMD} -n sippy-e2e get pod --selector=job-name=sippy-seed-job --output=jsonpath='{.items[0].metadata.name}') +${KUBECTL_CMD} -n sippy-e2e logs ${job_pod} > ${ARTIFACT_DIR}/sippy-seed.log if [ ${retVal} -ne 0 ]; then - echo "sippy loading never finished on time." + echo "sippy seeding never finished on time." exit 1 fi diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index 3deba69025..b8302420f1 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -16,23 +16,12 @@ trap cleanup EXIT # When running locally, the user has to define SIPPY_IMAGE. echo "The sippy CI image: ${SIPPY_IMAGE}" -# The GCS_CRED allows us to pull artifacts from GCS when importing prow jobs. -# Redefine GCS_CRED to use your own. -GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" -echo "The GCS cred is: ${GCS_CRED}" - # If you're using Openshift, we use oc, if you're using plain Kubernetes, # we use kubectl. # KUBECTL_CMD="${KUBECTL_CMD:=oc}" echo "The kubectl command is: ${KUBECTL_CMD}" -# Get the gcs credentials out to the cluster-pool cluster. -# These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations). -# See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa -# -${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred=$GCS_CRED -n sippy-e2e - # Launch the sippy api server pod. cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 @@ -74,8 +63,8 @@ spec: - ":12112" - --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres - --redis-url=redis://redis.sippy-e2e.svc.cluster.local:6379 - - --google-service-account-credential-file - - /tmp/secrets/gcs-cred + - --data-provider + - postgres - --log-level - debug - --enable-write-endpoints @@ -83,19 +72,8 @@ spec: - ocp - --views - ./config/e2e-views.yaml - env: - - name: GCS_SA_JSON_PATH - value: /tmp/secrets/gcs-cred - volumeMounts: - - mountPath: /tmp/secrets - name: gcs-cred - readOnly: true imagePullSecrets: - name: regcred - volumes: - - name: gcs-cred - secret: - secretName: gcs-cred dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler @@ -134,7 +112,6 @@ ${KUBECTL_CMD} -n sippy-e2e expose pod postg1 ${KUBECTL_CMD} -n sippy-e2e port-forward pod/postg1 ${SIPPY_PSQL_PORT}:5432 & # Random port for redis as well, between 19000 and 19500 -# Direct redis access is used for e2e tests to manipulate cache during testing. SIPPY_REDIS_PORT=$((RANDOM % 501 + 19000)) export SIPPY_REDIS_PORT export REDIS_URL="redis://localhost:${SIPPY_REDIS_PORT}" From daa13f1f25727b76ef00624c3c2797e11e7ecf71 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 12:37:58 -0400 Subject: [PATCH 04/25] Address code review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix POSIX compatibility: [[ → [ in e2e.sh - Add error handling to cache priming curl calls in e2e.sh - Add DB error assertions and context timeout to datasync test - Fix last_failure type assertion chain → type switch in querygenerators.go - Return error from regression tracker failures in seed_data.go - Fix misleading re-run docstring in seed_data.go - Deduplicate BQ provider creation in component_readiness.go - Fix successful_runs dedup with COUNT(DISTINCT IF(...)) in BQ provider - Include error details in GetJobVariants failure message Co-Authored-By: Claude Opus 4.6 --- cmd/sippy/annotatejobruns.go | 2 +- cmd/sippy/component_readiness.go | 6 +++--- cmd/sippy/seed_data.go | 5 +++-- .../dataprovider/bigquery/provider.go | 2 +- .../query/querygenerators.go | 12 +++++------- scripts/e2e.sh | 6 +++--- test/e2e/datasync/datasync_test.go | 19 ++++++++----------- 7 files changed, 24 insertions(+), 28 deletions(-) diff --git a/cmd/sippy/annotatejobruns.go b/cmd/sippy/annotatejobruns.go index b1248d6231..6df8c389ac 100644 --- a/cmd/sippy/annotatejobruns.go +++ b/cmd/sippy/annotatejobruns.go @@ -182,7 +182,7 @@ Example run: sippy annotate-job-runs --google-service-account-credential-file=f allVariants, errs := componentreadiness.GetJobVariants(ctx, bqprovider.NewBigQueryProvider(bigQueryClient)) if len(errs) > 0 { - return fmt.Errorf("failed to get job variants") + return fmt.Errorf("failed to get job variants: %v", errs) } if err = f.Validate(allVariants); err != nil { return errors.WithMessage(err, "error validating options") diff --git a/cmd/sippy/component_readiness.go b/cmd/sippy/component_readiness.go index 69f60ec70a..be2d121889 100644 --- a/cmd/sippy/component_readiness.go +++ b/cmd/sippy/component_readiness.go @@ -187,6 +187,8 @@ func (f *ComponentReadinessFlags) runServerMode() error { log.WithError(err).Warn("unable to initialize Jira client, bug filing will be disabled") } + crDataProvider := bqprovider.NewBigQueryProvider(bigQueryClient) + server := sippyserver.NewServer( sippyserver.ModeOpenShift, f.APIFlags.ListenAddr, @@ -199,7 +201,7 @@ func (f *ComponentReadinessFlags) runServerMode() error { gcsClient, f.GoogleCloudFlags.StorageBucket, bigQueryClient, - bqprovider.NewBigQueryProvider(bigQueryClient), + crDataProvider, nil, cacheClient, f.ComponentReadinessFlags.CRTimeRoundingFactor, @@ -210,8 +212,6 @@ func (f *ComponentReadinessFlags) runServerMode() error { jiraClient, ) - crDataProvider := bqprovider.NewBigQueryProvider(bigQueryClient) - if f.APIFlags.MetricsAddr != "" { // Do an immediate metrics update err = metrics.RefreshMetricsDB( diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index ba15b1f2b2..ce451270a0 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -59,7 +59,7 @@ Creates deterministic Component Readiness data covering all CR statuses MissingBasis, BasisOnly, SignificantImprovement, BelowMinFailure) and fallback scenarios. Use with 'sippy serve --data-provider postgres'. -The command can be re-run to refresh data. +Drop and recreate the database to re-seed (e.g. docker compose down -v). `, RunE: func(cmd *cobra.Command, args []string) error { if strings.Contains(f.DBFlags.DSN, "amazonaws.com") { @@ -423,7 +423,7 @@ func seedSyntheticData(dbc *db.DB) error { log.Info("Syncing regressions...") if err := syncRegressions(dbc); err != nil { - log.WithError(err).Warn("failed to sync regressions") + return errors.WithMessage(err, "failed to sync regressions") } log.Infof("Seeded synthetic data: %d ProwJobRuns, %d test results across %d releases", @@ -701,6 +701,7 @@ func syncRegressions(dbc *db.DB) error { for _, err := range tracker.Errors() { log.WithError(err).Warn("regression tracker error") } + return fmt.Errorf("regression tracker encountered %d errors", len(tracker.Errors())) } return nil } diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go index 283e2914bb..26cdbbd8d4 100644 --- a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go @@ -237,7 +237,7 @@ func (p *BigQueryProvider) QueryJobRuns(ctx context.Context, reqOptions reqopts. SELECT jobs.prowjob_job_name AS job_name, COUNT(DISTINCT jobs.prowjob_build_id) AS total_runs, - COUNTIF(jobs.prowjob_state = 'success') AS successful_runs + COUNT(DISTINCT IF(jobs.prowjob_state = 'success', jobs.prowjob_build_id, NULL)) AS successful_runs FROM %s.jobs jobs %s WHERE jobs.prowjob_start >= DATETIME(@From) diff --git a/pkg/api/componentreadiness/query/querygenerators.go b/pkg/api/componentreadiness/query/querygenerators.go index 21dd6da166..059b6c8e2e 100644 --- a/pkg/api/componentreadiness/query/querygenerators.go +++ b/pkg/api/componentreadiness/query/querygenerators.go @@ -781,14 +781,12 @@ func deserializeRowToTestStatus(row []bigquery.Value, schema bigquery.Schema) (s case col == "flake_count": cts.FlakeCount = int(row[i].(int64)) case col == "last_failure": - // ignore when we cant parse, its usually null - var err error if row[i] != nil { - layout := "2006-01-02T15:04:05" - lftCivilDT := row[i].(civil.DateTime) - cts.LastFailure, err = time.Parse(layout, lftCivilDT.String()) - if err != nil { - log.WithError(err).Error("error parsing last failure time from bigquery") + switch v := row[i].(type) { + case civil.DateTime: + cts.LastFailure = time.Date(v.Date.Year, v.Date.Month, v.Date.Day, v.Time.Hour, v.Time.Minute, v.Time.Second, v.Time.Nanosecond, time.UTC) + case time.Time: + cts.LastFailure = v } } case col == "component": diff --git a/scripts/e2e.sh b/scripts/e2e.sh index f5444fddef..2d583c1d70 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -11,7 +11,7 @@ PSQL_PORT="23433" REDIS_CONTAINER="sippy-e2e-test-redis" REDIS_PORT="23479" -if [[ -z "$GCS_SA_JSON_PATH" ]]; then +if [ -z "$GCS_SA_JSON_PATH" ]; then echo "WARNING: GCS_SA_JSON_PATH not set, data sync test will be skipped" 1>&2 fi @@ -98,10 +98,10 @@ fi # Prime the component readiness cache so triage tests can find cached reports echo "Priming component readiness cache..." -VIEWS=$(curl -s "http://localhost:$SIPPY_API_PORT/api/component_readiness/views") +VIEWS=$(curl -sf "http://localhost:$SIPPY_API_PORT/api/component_readiness/views") || { echo "Failed to fetch views"; exit 1; } for VIEW in $(echo "$VIEWS" | jq -r '.[].name'); do echo " Priming cache for view: $VIEW" - curl -s "http://localhost:$SIPPY_API_PORT/api/component_readiness?view=$VIEW" > /dev/null + curl -sf "http://localhost:$SIPPY_API_PORT/api/component_readiness?view=$VIEW" > /dev/null || { echo "Failed to prime cache for view: $VIEW"; exit 1; } done echo "Cache priming complete" diff --git a/test/e2e/datasync/datasync_test.go b/test/e2e/datasync/datasync_test.go index 65e4988552..dbbf2e95a4 100644 --- a/test/e2e/datasync/datasync_test.go +++ b/test/e2e/datasync/datasync_test.go @@ -1,12 +1,13 @@ package datasync import ( + "context" "os" "os/exec" "testing" + "time" "github.com/openshift/sippy/test/e2e/util" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,19 +18,17 @@ func TestDataSync(t *testing.T) { dbc := util.CreateE2EPostgresConnection(t) - // Count prow_job_runs before sync to compare after var countBefore int64 - dbc.DB.Table("prow_job_runs").Count(&countBefore) + require.NoError(t, dbc.DB.Table("prow_job_runs").Count(&countBefore).Error) t.Logf("prow_job_runs before sync: %d", countBefore) - // SIPPY_E2E_REPO_ROOT is set by e2e.sh to the repo root where the sippy - // binary and config files live. repoRoot := os.Getenv("SIPPY_E2E_REPO_ROOT") require.NotEmpty(t, repoRoot, "SIPPY_E2E_REPO_ROOT must be set") - // Run sippy load with minimal scope: just prow loader, single release, - // last 2 hours of data only - cmd := exec.Command(repoRoot+"/sippy", "load", // #nosec G204 + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + cmd := exec.CommandContext(ctx, repoRoot+"/sippy", "load", // #nosec G204 "--loader", "prow", "--release", util.Release, "--prow-load-since", "2h", @@ -46,9 +45,7 @@ func TestDataSync(t *testing.T) { err := cmd.Run() require.NoError(t, err, "sippy load command should complete without error") - // Verify some real prow job runs were loaded (with real job names from e2e-openshift.yaml) var countAfter int64 - dbc.DB.Table("prow_job_runs").Count(&countAfter) + require.NoError(t, dbc.DB.Table("prow_job_runs").Count(&countAfter).Error) t.Logf("prow_job_runs after sync: %d (loaded %d new)", countAfter, countAfter-countBefore) - assert.Greater(t, countAfter, countBefore, "sync should have loaded new prow job runs") } From c68a4fdd7cf0018a83d5d1bbe0b4c9a66d6d6626 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 12:42:03 -0400 Subject: [PATCH 05/25] Fix gofmt alignment in jobrunannotator.go Co-Authored-By: Claude Opus 4.6 --- .../jobrunannotator/jobrunannotator.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/componentreadiness/jobrunannotator/jobrunannotator.go b/pkg/componentreadiness/jobrunannotator/jobrunannotator.go index 6e80e4b69a..29f33485d6 100644 --- a/pkg/componentreadiness/jobrunannotator/jobrunannotator.go +++ b/pkg/componentreadiness/jobrunannotator/jobrunannotator.go @@ -64,18 +64,18 @@ type JobRunAnnotator struct { cache cache.Cache execute bool allVariants crtest.JobVariants - Release string `json:"release"` - IncludedVariants []crstatus.Variant `json:"included_variants"` - Label string `json:"label"` - BuildClusters []string `json:"build_clusters"` - StartTime time.Time `json:"start_time"` - Duration time.Duration `json:"duration"` - MinFailures int `json:"minimum_failure"` - FlakeAsFailure bool `json:"flake_as_failure"` - TextContains string `json:"text_contains"` - TextRegex string `json:"text_regex"` - PathGlob string `json:"path_glob"` - JobRunIDs []int64 `json:"job_run_ids"` + Release string `json:"release"` + IncludedVariants []crstatus.Variant `json:"included_variants"` + Label string `json:"label"` + BuildClusters []string `json:"build_clusters"` + StartTime time.Time `json:"start_time"` + Duration time.Duration `json:"duration"` + MinFailures int `json:"minimum_failure"` + FlakeAsFailure bool `json:"flake_as_failure"` + TextContains string `json:"text_contains"` + TextRegex string `json:"text_regex"` + PathGlob string `json:"path_glob"` + JobRunIDs []int64 `json:"job_run_ids"` comment string user string } From 0d59f8c618ee3bdc09d0db772c9dda0573d98a72 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 13:22:40 -0400 Subject: [PATCH 06/25] Add code coverage to e2e tests Build sippy with -cover instrumentation and collect coverage data from the running server during e2e tests. Coverage report is printed at the end of the run, and a coverage.out file is generated for HTML reports. Also adds graceful shutdown signal handling to the server so coverage data is properly flushed on exit. Co-Authored-By: Claude Opus 4.6 --- .gitignore | 3 +++ pkg/sippyserver/server.go | 15 +++++++++++++++ scripts/e2e.sh | 39 +++++++++++++++++++++++++-------------- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 0f715cb799..6a9ae9fa38 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ report.sh /sippy-ng/build/* .env *.log +/e2e-coverage/ +/e2e-coverage.out +/e2e-coverage.html diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index c952f29158..9df846c2f1 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -10,7 +10,9 @@ import ( "net/http" "net/http/httptest" "os" + "os/signal" "regexp" + "syscall" sorting "sort" "strconv" "strings" @@ -2907,6 +2909,19 @@ func (s *Server) Serve() { log.Infof("Serving reports on %s ", s.listenAddr) + // Handle graceful shutdown on SIGINT/SIGTERM so coverage data is flushed + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-sigCh + log.Infof("Received %s, shutting down server...", sig) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.httpServer.Shutdown(ctx); err != nil { + log.WithError(err).Error("Error during server shutdown") + } + }() + if err := s.httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { log.WithError(err).Error("Server exited") } diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 2d583c1d70..0880201ae4 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -18,14 +18,22 @@ fi clean_up () { ARG=$? - echo "Killing sippy API child process: $CHILD_PID" - kill $CHILD_PID - echo "Tearing down container $PSQL_CONTAINER" - $DOCKER stop -i $PSQL_CONTAINER - $DOCKER rm -i $PSQL_CONTAINER - echo "Tearing down container $REDIS_CONTAINER" - $DOCKER stop -i $REDIS_CONTAINER - $DOCKER rm -i $REDIS_CONTAINER + echo "Stopping sippy API child process: $CHILD_PID" + kill $CHILD_PID 2>/dev/null && wait $CHILD_PID 2>/dev/null + # Generate coverage report from the server's coverage data + if [ -d "$COVDIR" ] && find "$COVDIR" -name 'covcounters.*' -print -quit | grep -q .; then + echo "Generating coverage report..." + go tool covdata percent -i="$COVDIR" + go tool covdata textfmt -i="$COVDIR" -o=e2e-coverage.out + echo "Coverage data written to e2e-coverage.out" + echo "View HTML report: go tool cover -html=e2e-coverage.out -o=e2e-coverage.html" + fi + echo "Tearing down container $PSQL_CONTAINER" + $DOCKER stop -i $PSQL_CONTAINER + $DOCKER rm -i $PSQL_CONTAINER + echo "Tearing down container $REDIS_CONTAINER" + $DOCKER stop -i $REDIS_CONTAINER + $DOCKER rm -i $REDIS_CONTAINER exit $ARG } trap clean_up EXIT @@ -53,9 +61,15 @@ export SIPPY_E2E_DSN="postgresql://postgres:password@localhost:$PSQL_PORT/postgr export REDIS_URL="redis://localhost:$REDIS_PORT" export SIPPY_E2E_REPO_ROOT="$(pwd)" +# Build with coverage instrumentation +COVDIR="$(pwd)/e2e-coverage" +rm -rf "$COVDIR" +mkdir -p "$COVDIR" +echo "Building sippy with coverage instrumentation..." +go build -cover -coverpkg=./cmd/...,./pkg/... -mod vendor -o ./sippy ./cmd/sippy + echo "Loading database..." -go build -mod vendor ./cmd/sippy -./sippy seed-data \ +GOCOVERDIR="$COVDIR" ./sippy seed-data \ --init-database \ --database-dsn="$SIPPY_E2E_DSN" @@ -72,10 +86,7 @@ SERVE_ARGS="--listen :$SIPPY_API_PORT \ --data-provider postgres \ --views config/e2e-views.yaml" -( -./sippy serve $SERVE_ARGS > e2e.log 2>&1 -)& -# store the child process for cleanup +GOCOVERDIR="$COVDIR" ./sippy serve $SERVE_ARGS > e2e.log 2>&1 & CHILD_PID=$! # Give it time to start up, and fill the redis cache From e8216044d1e09af5e06fa75849b1145860a582a7 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 14:35:47 -0400 Subject: [PATCH 07/25] Improve CR e2e test coverage and merge query package into bigquery provider - Move query package into dataprovider/bigquery, removing the separate query/ directory - Remove dataSource from DataProvider interface, pushing variant junit table override logic into the BigQuery provider - Add e2e tests for new test pass rate regression, fallback test details, include variants filtering, explicit API params, and triage validation - Merge test binary coverage into e2e report so direct-call tests in report_test.go contribute to measured coverage - Add seed data for new test pass rate regression scenario (70% pass rate below 80% extreme threshold) Co-Authored-By: Claude Opus 4.6 --- cmd/sippy/annotatejobruns.go | 2 +- cmd/sippy/automatejira.go | 5 +- cmd/sippy/component_readiness.go | 8 +- cmd/sippy/load.go | 3 +- cmd/sippy/seed_data.go | 11 +- cmd/sippy/serve.go | 8 +- config/e2e-views.yaml | 2 +- .../componentreadiness/component_report.go | 140 ++-------- .../component_report_test.go | 99 +------ .../dataprovider/bigquery/provider.go | 249 ++++++++++++++++-- .../dataprovider/bigquery/provider_test.go | 105 ++++++++ .../bigquery}/querygenerators.go | 2 +- .../bigquery}/querygenerators_test.go | 2 +- .../bigquery}/releasedates.go | 2 +- .../dataprovider/interface.go | 7 +- .../dataprovider/postgres/provider.go | 6 +- .../componentreadiness/regressiontracker.go | 40 ++- pkg/api/componentreadiness/test_details.go | 114 ++------ .../jiraautomator/jiraautomator.go | 32 +-- pkg/dataloader/crcacheloader/crcacheloader.go | 4 +- pkg/sippyserver/metrics/metrics.go | 14 +- pkg/sippyserver/server.go | 1 - scripts/e2e.sh | 7 +- .../componentreadiness_test.go | 119 +++++++++ .../componentreadiness/report/report_test.go | 171 +++++++++++- .../triage/triageapi_test.go | 16 ++ 26 files changed, 744 insertions(+), 425 deletions(-) create mode 100644 pkg/api/componentreadiness/dataprovider/bigquery/provider_test.go rename pkg/api/componentreadiness/{query => dataprovider/bigquery}/querygenerators.go (99%) rename pkg/api/componentreadiness/{query => dataprovider/bigquery}/querygenerators_test.go (99%) rename pkg/api/componentreadiness/{query => dataprovider/bigquery}/releasedates.go (98%) diff --git a/cmd/sippy/annotatejobruns.go b/cmd/sippy/annotatejobruns.go index 6df8c389ac..86567f3eb3 100644 --- a/cmd/sippy/annotatejobruns.go +++ b/cmd/sippy/annotatejobruns.go @@ -180,7 +180,7 @@ Example run: sippy annotate-job-runs --google-service-account-credential-file=f return errors.WithMessage(err, "couldn't get DB client") } - allVariants, errs := componentreadiness.GetJobVariants(ctx, bqprovider.NewBigQueryProvider(bigQueryClient)) + allVariants, errs := componentreadiness.GetJobVariants(ctx, bqprovider.NewBigQueryProvider(bigQueryClient, nil)) if len(errs) > 0 { return fmt.Errorf("failed to get job variants: %v", errs) } diff --git a/cmd/sippy/automatejira.go b/cmd/sippy/automatejira.go index bb69d2d1ad..d0b0289c10 100644 --- a/cmd/sippy/automatejira.go +++ b/cmd/sippy/automatejira.go @@ -170,7 +170,7 @@ func NewAutomateJiraCommand() *cobra.Command { log.WithError(err).Warn("error reading config file") } - provider := bqprovider.NewBigQueryProvider(bigQueryClient) + provider := bqprovider.NewBigQueryProvider(bigQueryClient, config.ComponentReadinessConfig.VariantJunitTableOverrides) allVariants, errs := componentreadiness.GetJobVariants(ctx, provider) if len(errs) > 0 { return fmt.Errorf("failed to get job variants") @@ -191,8 +191,7 @@ func NewAutomateJiraCommand() *cobra.Command { jiraClient, bigQueryClient, provider, dbc, cacheOpts, views.ComponentReadiness, releases, f.SippyURL, f.JiraAccount, f.IncludeComponents, f.ColumnThresholds, - f.DryRun, variantToJiraComponents, - config.ComponentReadinessConfig.VariantJunitTableOverrides) + f.DryRun, variantToJiraComponents) if err != nil { panic(err) } diff --git a/cmd/sippy/component_readiness.go b/cmd/sippy/component_readiness.go index be2d121889..13838614e2 100644 --- a/cmd/sippy/component_readiness.go +++ b/cmd/sippy/component_readiness.go @@ -187,7 +187,7 @@ func (f *ComponentReadinessFlags) runServerMode() error { log.WithError(err).Warn("unable to initialize Jira client, bug filing will be disabled") } - crDataProvider := bqprovider.NewBigQueryProvider(bigQueryClient) + crDataProvider := bqprovider.NewBigQueryProvider(bigQueryClient, config.ComponentReadinessConfig.VariantJunitTableOverrides) server := sippyserver.NewServer( sippyserver.ModeOpenShift, @@ -221,8 +221,7 @@ func (f *ComponentReadinessFlags) runServerMode() error { crDataProvider, time.Time{}, cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), - views.ComponentReadiness, - config.ComponentReadinessConfig.VariantJunitTableOverrides) + views.ComponentReadiness) if err != nil { log.WithError(err).Error("error refreshing metrics") } @@ -242,8 +241,7 @@ func (f *ComponentReadinessFlags) runServerMode() error { crDataProvider, time.Time{}, cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), - views.ComponentReadiness, - config.ComponentReadinessConfig.VariantJunitTableOverrides) + views.ComponentReadiness) if err != nil { log.WithError(err).Error("error refreshing metrics") } diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 861efc07d9..84718930fe 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -319,10 +319,9 @@ func NewLoadCommand() *cobra.Command { } regressionTracker := componentreadiness.NewRegressionTracker( - bqprovider.NewBigQueryProvider(bqc), dbc, cacheOpts, releases, + bqprovider.NewBigQueryProvider(bqc, config.ComponentReadinessConfig.VariantJunitTableOverrides), dbc, cacheOpts, releases, componentreadiness.NewPostgresRegressionStore(dbc, jiraClient), views.ComponentReadiness, - config.ComponentReadinessConfig.VariantJunitTableOverrides, false) loaders = append(loaders, regressionTracker) } diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index ce451270a0..764ea188ce 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -245,6 +245,15 @@ var syntheticTests = []syntheticTestSpec{ }, }, + // --- NewTestPassRateRegression: new test only in sample, below PassRateRequiredNewTests threshold --- + { + testID: "test-new-test-pass-rate-fail", testName: "[sig-node] New flaky pod readiness test", + component: "comp-NewTestPassRate", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.22": {100, 70, 0}}, + }, + }, + // --- BasisOnly: test in base, absent from sample --- { testID: "test-basis-only", testName: "[sig-apps] Removed deployment test", @@ -693,7 +702,6 @@ func syncRegressions(dbc *db.DB) error { releases, componentreadiness.NewPostgresRegressionStore(dbc, nil), views.ComponentReadiness, - nil, false, ) tracker.Load() @@ -758,6 +766,7 @@ func writeSyntheticViewsFile() error { Confidence: 95, PityFactor: 5, MinimumFailure: 3, + PassRateRequiredNewTests: 90, IncludeMultiReleaseAnalysis: true, }, PrimeCache: crview.PrimeCache{Enabled: true}, diff --git a/cmd/sippy/serve.go b/cmd/sippy/serve.go index adf102c79b..8e02bd823a 100644 --- a/cmd/sippy/serve.go +++ b/cmd/sippy/serve.go @@ -133,7 +133,7 @@ func NewServeCommand() *cobra.Command { bigQueryClient = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bigQueryClient) } - crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient) + crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient, config.ComponentReadinessConfig.VariantJunitTableOverrides) gcsClient, err = gcs.NewGCSClient(context.TODO(), f.GoogleCloudFlags.ServiceAccountCredentialFile, @@ -212,8 +212,7 @@ func NewServeCommand() *cobra.Command { crDataProvider, util.GetReportEnd(pinnedDateTime), cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), - views.ComponentReadiness, - config.ComponentReadinessConfig.VariantJunitTableOverrides) + views.ComponentReadiness) if err != nil { log.WithError(err).Error("error refreshing metrics") } @@ -233,8 +232,7 @@ func NewServeCommand() *cobra.Command { crDataProvider, util.GetReportEnd(pinnedDateTime), cache.NewStandardCROptions(f.ComponentReadinessFlags.CRTimeRoundingFactor), - views.ComponentReadiness, - config.ComponentReadinessConfig.VariantJunitTableOverrides) + views.ComponentReadiness) if err != nil { log.WithError(err).Error("error refreshing metrics") } diff --git a/config/e2e-views.yaml b/config/e2e-views.yaml index 00843238e9..36b9e3abb6 100644 --- a/config/e2e-views.yaml +++ b/config/e2e-views.yaml @@ -55,7 +55,7 @@ component_readiness: minimum_failure: 3 confidence: 95 pity_factor: 5 - pass_rate_required_new_tests: 0 + pass_rate_required_new_tests: 90 pass_rate_required_all_tests: 0 ignore_missing: false ignore_disruption: false diff --git a/pkg/api/componentreadiness/component_report.go b/pkg/api/componentreadiness/component_report.go index 7c6919eaeb..eed74c9709 100644 --- a/pkg/api/componentreadiness/component_report.go +++ b/pkg/api/componentreadiness/component_report.go @@ -28,14 +28,11 @@ import ( "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware" "github.com/openshift/sippy/pkg/api/componentreadiness/middleware/releasefallback" - "github.com/openshift/sippy/pkg/api/componentreadiness/query" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/cache" - configv1 "github.com/openshift/sippy/pkg/apis/config/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" - "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/sets" ) @@ -82,7 +79,6 @@ func GetComponentReport( provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, - variantJunitTableOverrides []configv1.VariantJunitTableOverride, baseURL string, ) (report crtype.ComponentReport, errs []error) { releaseConfigs, err := provider.QueryReleases(ctx) @@ -90,7 +86,7 @@ func GetComponentReport( return report, []error{err} } - generator := NewComponentReportGenerator(provider, reqOptions, dbc, variantJunitTableOverrides, releaseConfigs, baseURL) + generator := NewComponentReportGenerator(provider, reqOptions, dbc, releaseConfigs, baseURL) if os.Getenv("DEV_MODE") == "1" { report, errs = generator.GenerateReport(ctx) @@ -153,15 +149,14 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport) return nil } -func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, variantJunitTableOverrides []configv1.VariantJunitTableOverride, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator { +func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions reqopts.RequestOptions, dbc *db.DB, releaseConfigs []v1.Release, baseURL string) ComponentReportGenerator { slices.Sort(reqOptions.Capabilities) // normalize ordering so cache keys match generator := ComponentReportGenerator{ - dataProvider: provider, - ReqOptions: reqOptions, - dbc: dbc, - variantJunitTableOverrides: variantJunitTableOverrides, - releaseConfigs: releaseConfigs, - baseURL: baseURL, + dataProvider: provider, + ReqOptions: reqOptions, + dbc: dbc, + releaseConfigs: releaseConfigs, + baseURL: baseURL, } generator.initializeMiddleware() return generator @@ -174,13 +169,12 @@ func NewComponentReportGenerator(provider dataprovider.DataProvider, reqOptions // is marshalled for the cache key and should be changed when the object being // cached changes in a way that will no longer be compatible with any prior cached version. type ComponentReportGenerator struct { - dataProvider dataprovider.DataProvider - dbc *db.DB - ReqOptions reqopts.RequestOptions - variantJunitTableOverrides []configv1.VariantJunitTableOverride - middlewares middleware.List - releaseConfigs []v1.Release - baseURL string + dataProvider dataprovider.DataProvider + dbc *db.DB + ReqOptions reqopts.RequestOptions + middlewares middleware.List + releaseConfigs []v1.Release + baseURL string } type GeneratorCacheKey struct { @@ -353,21 +347,14 @@ func (c *ComponentReportGenerator) getTestStatus(ctx context.Context) (crstatus. c.middlewares.Query(ctx, wg, allJobVariants, baseStatusCh, sampleStatusCh, errCh) goInterruptible(ctx, wg, func() { baseStatus, baseErrs = c.dataProvider.QueryBaseTestStatus(ctx, c.ReqOptions, allJobVariants) }) goInterruptible(ctx, wg, func() { - includeVariants, skipQuery := copyIncludeVariantsAndRemoveOverrides(c.variantJunitTableOverrides, -1, c.ReqOptions.VariantOption.IncludeVariants) - if skipQuery { - fLog.Infof("skipping default sample query as all values for a variant were overridden") - return - } - fLog.Infof("running default sample query with includeVariants: %+v", includeVariants) - status, errs := c.dataProvider.QuerySampleTestStatus(ctx, c.ReqOptions, allJobVariants, includeVariants, c.ReqOptions.SampleRelease.Start, c.ReqOptions.SampleRelease.End, query.DefaultJunitTable) - fLog.Infof("received %d test statuses and %d errors from default query", len(status), len(errs)) + fLog.Infof("running sample query with includeVariants: %+v", c.ReqOptions.VariantOption.IncludeVariants) + status, errs := c.dataProvider.QuerySampleTestStatus(ctx, c.ReqOptions, allJobVariants, c.ReqOptions.VariantOption.IncludeVariants, c.ReqOptions.SampleRelease.Start, c.ReqOptions.SampleRelease.End) + fLog.Infof("received %d test statuses and %d errors from sample query", len(status), len(errs)) sampleStatusCh <- status for _, err := range errs { errCh <- err } }) - // TODO: move to a variantjunitoverride middleware with Query implemented - c.goRunOverrideSampleQueries(ctx, wg, fLog, allJobVariants, sampleStatusCh, errCh) // clean up channels after all queries are done go func() { @@ -418,45 +405,6 @@ func (c *ComponentReportGenerator) getTestStatus(ctx context.Context) (crstatus. return crstatus.ReportTestStatus{BaseStatus: baseStatus, SampleStatus: sampleStatus, GeneratedAt: &now}, errs } -// fork additional sample queries for the overrides -func (c *ComponentReportGenerator) goRunOverrideSampleQueries( - ctx context.Context, wg *sync.WaitGroup, fLog *log.Entry, - allJobVariants crtest.JobVariants, - sampleStatusCh chan map[string]crstatus.TestStatus, - errCh chan error, -) { - for i, or := range c.variantJunitTableOverrides { - if !utils.ContainsOverriddenVariant(c.ReqOptions.VariantOption.IncludeVariants, or.VariantName, or.VariantValue) { - continue - } - - index, override := i, or // copy loop vars to avoid them changing during goroutine - goInterruptible(ctx, wg, func() { - // only do this additional query if the specified override variant is actually included in this request - includeVariants, skipQuery := copyIncludeVariantsAndRemoveOverrides(c.variantJunitTableOverrides, index, c.ReqOptions.VariantOption.IncludeVariants) - if skipQuery { - fLog.Infof("skipping override sample query as all values for a variant were overridden") - return - } - fLog.Infof("running override sample query for %+v with includeVariants: %+v", override, includeVariants) - // Calculate a start time relative to the requested end time: (i.e. for rarely run jobs) - end := c.ReqOptions.SampleRelease.End - start, err := util.ParseCRReleaseTime([]v1.Release{}, "", override.RelativeStart, - true, &c.ReqOptions.SampleRelease.End, c.ReqOptions.CacheOption.CRTimeRoundingFactor) - if err != nil { - errCh <- err - return - } - status, errs := c.dataProvider.QuerySampleTestStatus(ctx, c.ReqOptions, allJobVariants, includeVariants, start, end, override.TableName) - fLog.Infof("received %d test statuses and %d errors from override query", len(status), len(errs)) - sampleStatusCh <- status - for _, err := range errs { - errCh <- err - } - }) - } -} - func goInterruptible(ctx context.Context, wg *sync.WaitGroup, closure func()) { wg.Add(1) go func() { @@ -470,62 +418,6 @@ func goInterruptible(ctx context.Context, wg *sync.WaitGroup, closure func()) { }() } -// copyIncludeVariantsAndRemoveOverrides is used when VariantJunitTableOverrides are in play, and we'll be merging in -// some results from separate junit tables. In this case, when we do the normal default query, we want to remove those -// overridden variants just in case, to make sure no results slip in that shouldn't be there. -// -// An index into the overrides slice can be provided if we're copying the include variants for that subquery. This is -// just to be careful for any future cases where we might have multiple overrides in play, and want to make sure we -// don't accidentally pull data for one, from the others junit table. -// -// Return includes a bool which may indicate to skip the query entirely because we've overridden all values for a variant. -func copyIncludeVariantsAndRemoveOverrides( - overrides []configv1.VariantJunitTableOverride, - currOverride int, // index into the overrides if we're copying for that specific override query - includeVariants map[string][]string) (map[string][]string, bool) { - - cp := make(map[string][]string) - for key, values := range includeVariants { - newSlice := []string{} - for _, v := range values { - if !shouldSkipVariant(overrides, currOverride, key, v) { - newSlice = append(newSlice, v) - } - - } - if len(newSlice) == 0 { - // If we overrode a value for a variant, and no other values are specified for that - // variant, we want to skip this query entirely. - // i.e. if we include JobTier blocking, informing, and rare, we still want to do the default - // query for blocking and informing even though rare was overridden. - // However if we specify only JobTier rare, this leaves no JobTier's left in the default query resulting - // in a normal query without considering JobTier and thus duplicate results we don't want. In this case, - // we want to skip the default. - // - // TODO: With two overridden variants in one query, we could easily get into a problem - // where no results are returned, because we AND the include variants. If JobTier rare is in table1, and - // Foo=bar is in table2, both queries would be skipped because neither contains data for the other and we're - // doing an AND. For now, I think this is a limitation we'll have to live with - return cp, true - } - cp[key] = newSlice - } - return cp, false -} - -func shouldSkipVariant(overrides []configv1.VariantJunitTableOverride, currOverride int, key, value string) bool { - for i, override := range overrides { - // if we're building a list of include variants for an override, then don't skip that variants inclusion - if i == currOverride { - return false - } - if override.VariantName == key && override.VariantValue == value { - return true - } - } - return false -} - var componentAndCapabilityGetter func(test crtest.KeyWithVariants, stats crstatus.TestStatus) (string, []string) func testToComponentAndCapability(_ crtest.KeyWithVariants, stats crstatus.TestStatus) (string, []string) { diff --git a/pkg/api/componentreadiness/component_report_test.go b/pkg/api/componentreadiness/component_report_test.go index ba0f00cc8b..da05888b2a 100644 --- a/pkg/api/componentreadiness/component_report_test.go +++ b/pkg/api/componentreadiness/component_report_test.go @@ -4,7 +4,6 @@ package componentreadiness import ( "encoding/json" "fmt" - "reflect" "strings" "testing" "time" @@ -18,7 +17,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - v1 "github.com/openshift/sippy/pkg/apis/config/v1" crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/util/sets" @@ -1860,98 +1858,5 @@ func Test_componentReportGenerator_assessComponentStatus(t *testing.T) { } } -func TestCopyIncludeVariantsAndRemoveOverrides(t *testing.T) { - tests := []struct { - name string - overrides []v1.VariantJunitTableOverride - currOverride int - includeVariants map[string][]string - expected map[string][]string - expectedSkipQuery bool - }{ - { - name: "No overrides, no variants removed", - overrides: []v1.VariantJunitTableOverride{}, - currOverride: -1, - includeVariants: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - expected: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - }, - { - name: "Single override removes matching variant", - overrides: []v1.VariantJunitTableOverride{ - {VariantName: "key1", VariantValue: "value1"}, - }, - currOverride: -1, - includeVariants: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - expected: map[string][]string{ - "key1": {"value2"}, - "key2": {"value3"}, - }, - }, - { - name: "Override does not remove its own variant", - overrides: []v1.VariantJunitTableOverride{ - {VariantName: "key1", VariantValue: "value1"}, - }, - currOverride: 0, - includeVariants: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - expected: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - }, - { - name: "Multiple overrides remove multiple variants", - overrides: []v1.VariantJunitTableOverride{ - {VariantName: "key1", VariantValue: "value1"}, - {VariantName: "key2", VariantValue: "value3"}, - }, - currOverride: -1, - includeVariants: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3", "value4"}, - }, - expected: map[string][]string{ - "key1": {"value2"}, - "key2": {"value4"}, - }, - }, - { - name: "All variants removed", - overrides: []v1.VariantJunitTableOverride{ - {VariantName: "key1", VariantValue: "value1"}, - {VariantName: "key1", VariantValue: "value2"}, - {VariantName: "key2", VariantValue: "value3"}, - }, - currOverride: -1, - includeVariants: map[string][]string{ - "key1": {"value1", "value2"}, - "key2": {"value3"}, - }, - expected: map[string][]string{}, - expectedSkipQuery: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, skipQuery := copyIncludeVariantsAndRemoveOverrides(tt.overrides, tt.currOverride, tt.includeVariants) - assert.Equal(t, tt.expectedSkipQuery, skipQuery) - if !reflect.DeepEqual(result, tt.expected) { - t.Errorf("expected %v, got %v", tt.expected, result) - } - }) - } -} +// TestCopyIncludeVariantsAndRemoveOverrides moved to dataprovider/bigquery package +// where the function now lives. diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go index 26cdbbd8d4..e9a372097c 100644 --- a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go @@ -6,6 +6,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "cloud.google.com/go/bigquery" @@ -14,14 +15,15 @@ import ( apiPkg "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" - "github.com/openshift/sippy/pkg/api/componentreadiness/query" "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" apiCache "github.com/openshift/sippy/pkg/apis/cache" + configv1 "github.com/openshift/sippy/pkg/apis/config/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqcachedclient "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" + "github.com/openshift/sippy/pkg/util" "github.com/openshift/sippy/pkg/util/param" "github.com/openshift/sippy/pkg/util/sets" ) @@ -31,11 +33,12 @@ var _ dataprovider.DataProvider = &BigQueryProvider{} // BigQueryProvider implements dataprovider.DataProvider using Google BigQuery // as the backing data store, wrapping the existing query generators. type BigQueryProvider struct { - client *bqcachedclient.Client + client *bqcachedclient.Client + variantJunitTableOverrides []configv1.VariantJunitTableOverride } -func NewBigQueryProvider(client *bqcachedclient.Client) *BigQueryProvider { - return &BigQueryProvider{client: client} +func NewBigQueryProvider(client *bqcachedclient.Client, overrides []configv1.VariantJunitTableOverride) *BigQueryProvider { + return &BigQueryProvider{client: client, variantJunitTableOverrides: overrides} } // Client returns the underlying BigQuery client for callers that still need direct access @@ -53,7 +56,7 @@ func (p *BigQueryProvider) Cache() apiCache.Cache { func (p *BigQueryProvider) QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants) (map[string]crstatus.TestStatus, []error) { - generator := query.NewBaseQueryGenerator(p.client, reqOptions, allJobVariants) + generator := NewBaseQueryGenerator(p.client, reqOptions, allJobVariants) result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.ReportTestStatus]( ctx, p.client.Cache, reqOptions.CacheOption, apiPkg.NewCacheSpec(generator, "BaseTestStatus~", &reqOptions.BaseRelease.End), @@ -65,12 +68,96 @@ func (p *BigQueryProvider) QueryBaseTestStatus(ctx context.Context, reqOptions r } func (p *BigQueryProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time) (map[string]crstatus.TestStatus, []error) { + + fLog := log.WithField("func", "QuerySampleTestStatus") + + // Filter out overridden variants from the default query + filteredVariants, skipDefault := copyIncludeVariantsAndRemoveOverrides(p.variantJunitTableOverrides, -1, includeVariants) + + type result struct { + status map[string]crstatus.TestStatus + errs []error + } + resultCh := make(chan result) + var wg sync.WaitGroup + + // Run default query (unless all variants were overridden) + if !skipDefault { + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + return + default: + fLog.Infof("running default sample query with includeVariants: %+v", filteredVariants) + s, e := p.querySampleTestStatusForTable(ctx, reqOptions, allJobVariants, filteredVariants, start, end, DefaultJunitTable) + resultCh <- result{s, e} + } + }() + } + + // Run override queries for applicable variants + for i, or := range p.variantJunitTableOverrides { + if !containsOverriddenVariant(includeVariants, or.VariantName, or.VariantValue) { + continue + } + index, override := i, or + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + return + default: + overrideVariants, skip := copyIncludeVariantsAndRemoveOverrides(p.variantJunitTableOverrides, index, includeVariants) + if skip { + fLog.Infof("skipping override sample query as all values for a variant were overridden") + return + } + overrideEnd := end + overrideStart, err := util.ParseCRReleaseTime([]v1.Release{}, "", override.RelativeStart, + true, &overrideEnd, reqOptions.CacheOption.CRTimeRoundingFactor) + if err != nil { + resultCh <- result{nil, []error{err}} + return + } + fLog.Infof("running override sample query for %+v with includeVariants: %+v", override, overrideVariants) + s, e := p.querySampleTestStatusForTable(ctx, reqOptions, allJobVariants, overrideVariants, overrideStart, overrideEnd, override.TableName) + resultCh <- result{s, e} + } + }() + } + + go func() { + wg.Wait() + close(resultCh) + }() + + merged := make(map[string]crstatus.TestStatus) + var allErrs []error + for r := range resultCh { + allErrs = append(allErrs, r.errs...) + for k, v := range r.status { + merged[k] = v + } + } + if len(allErrs) > 0 { + return nil, allErrs + } + return merged, nil +} + +func (p *BigQueryProvider) querySampleTestStatusForTable(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, includeVariants map[string][]string, start, end time.Time, - dataSource string) (map[string]crstatus.TestStatus, []error) { + junitTable string) (map[string]crstatus.TestStatus, []error) { - generator := query.NewSampleQueryGenerator(p.client, reqOptions, allJobVariants, includeVariants, start, end, dataSource) + generator := NewSampleQueryGenerator(p.client, reqOptions, allJobVariants, includeVariants, start, end, junitTable) result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.ReportTestStatus]( ctx, p.client.Cache, reqOptions.CacheOption, apiPkg.NewCacheSpec(generator, "SampleTestStatus~", &reqOptions.SampleRelease.End), @@ -86,7 +173,7 @@ func (p *BigQueryProvider) QuerySampleTestStatus(ctx context.Context, reqOptions func (p *BigQueryProvider) QueryBaseJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants) (map[string][]crstatus.TestJobRunRows, []error) { - generator := query.NewBaseTestDetailsQueryGenerator( + generator := NewBaseTestDetailsQueryGenerator( log.WithField("func", "QueryBaseJobRunTestStatus"), p.client, reqOptions, allJobVariants, reqOptions.BaseRelease.Name, reqOptions.BaseRelease.Start, reqOptions.BaseRelease.End, @@ -105,11 +192,91 @@ func (p *BigQueryProvider) QueryBaseJobRunTestStatus(ctx context.Context, reqOpt func (p *BigQueryProvider) QuerySampleJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, includeVariants map[string][]string, - start, end time.Time, - dataSource string) (map[string][]crstatus.TestJobRunRows, []error) { + start, end time.Time) (map[string][]crstatus.TestJobRunRows, []error) { - generator := query.NewSampleTestDetailsQueryGenerator(p.client, reqOptions, allJobVariants, includeVariants, start, end, dataSource) + fLog := log.WithField("func", "QuerySampleJobRunTestStatus") + filteredVariants, skipDefault := copyIncludeVariantsAndRemoveOverrides(p.variantJunitTableOverrides, -1, includeVariants) + + type result struct { + status map[string][]crstatus.TestJobRunRows + errs []error + } + resultCh := make(chan result) + var wg sync.WaitGroup + + if !skipDefault { + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + return + default: + fLog.Infof("running default sample job run query with includeVariants: %+v", filteredVariants) + s, e := p.querySampleJobRunTestStatusForTable(ctx, reqOptions, allJobVariants, filteredVariants, start, end, DefaultJunitTable) + resultCh <- result{s, e} + } + }() + } + + for i, or := range p.variantJunitTableOverrides { + if !containsOverriddenVariant(includeVariants, or.VariantName, or.VariantValue) { + continue + } + index, override := i, or + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + return + default: + overrideVariants, skip := copyIncludeVariantsAndRemoveOverrides(p.variantJunitTableOverrides, index, includeVariants) + if skip { + fLog.Infof("skipping override sample job run query as all values for a variant were overridden") + return + } + overrideEnd := end + overrideStart, err := util.ParseCRReleaseTime([]v1.Release{}, "", override.RelativeStart, + true, &overrideEnd, reqOptions.CacheOption.CRTimeRoundingFactor) + if err != nil { + resultCh <- result{nil, []error{err}} + return + } + fLog.Infof("running override sample job run query for %+v with includeVariants: %+v", override, overrideVariants) + s, e := p.querySampleJobRunTestStatusForTable(ctx, reqOptions, allJobVariants, overrideVariants, overrideStart, overrideEnd, override.TableName) + resultCh <- result{s, e} + } + }() + } + + go func() { + wg.Wait() + close(resultCh) + }() + + merged := make(map[string][]crstatus.TestJobRunRows) + var allErrs []error + for r := range resultCh { + allErrs = append(allErrs, r.errs...) + for k, v := range r.status { + merged[k] = v + } + } + if len(allErrs) > 0 { + return nil, allErrs + } + return merged, nil +} + +func (p *BigQueryProvider) querySampleJobRunTestStatusForTable(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time, + junitTable string) (map[string][]crstatus.TestJobRunRows, []error) { + + generator := NewSampleTestDetailsQueryGenerator(p.client, reqOptions, allJobVariants, includeVariants, start, end, junitTable) result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.TestJobRunStatuses]( ctx, p.client.Cache, reqOptions.CacheOption, apiPkg.NewCacheSpec(generator, "SampleJobRunTestStatus~", &end), @@ -175,7 +342,7 @@ func (p *BigQueryProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari } func (p *BigQueryProvider) QueryReleaseDates(ctx context.Context, reqOptions reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { - return query.GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions) + return GetReleaseDatesFromBigQuery(ctx, p.client, reqOptions) } func (p *BigQueryProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { @@ -402,11 +569,57 @@ func getSingleColumnResultToSlice(ctx context.Context, q *bigquery.Query) ([]str return names, nil } -func sortedKeys[V any](m map[string]V) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) +// containsOverriddenVariant checks whether the given variant key/value pair +// is present in the includeVariants map (i.e. the request actually includes +// data for this overridden variant). +func containsOverriddenVariant(includeVariants map[string][]string, key, value string) bool { + for k, v := range includeVariants { + if k != key { + continue + } + for _, vv := range v { + if vv == value { + return true + } + } + } + return false +} + +// copyIncludeVariantsAndRemoveOverrides copies includeVariants and removes +// overridden variant values. For the default query (currOverride=-1), all +// overridden values are removed. For an override query at index i, only +// other overrides' values are removed. Returns true if the query should be +// skipped because all values for a variant were removed. +func copyIncludeVariantsAndRemoveOverrides( + overrides []configv1.VariantJunitTableOverride, + currOverride int, + includeVariants map[string][]string) (map[string][]string, bool) { + + cp := make(map[string][]string) + for key, values := range includeVariants { + var newSlice []string + for _, v := range values { + if !shouldSkipVariant(overrides, currOverride, key, v) { + newSlice = append(newSlice, v) + } + } + if len(newSlice) == 0 { + return cp, true + } + cp[key] = newSlice + } + return cp, false +} + +func shouldSkipVariant(overrides []configv1.VariantJunitTableOverride, currOverride int, key, value string) bool { + for i, override := range overrides { + if i == currOverride { + return false + } + if override.VariantName == key && override.VariantValue == value { + return true + } } - sort.Strings(keys) - return keys + return false } diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/provider_test.go b/pkg/api/componentreadiness/dataprovider/bigquery/provider_test.go new file mode 100644 index 0000000000..159170e21b --- /dev/null +++ b/pkg/api/componentreadiness/dataprovider/bigquery/provider_test.go @@ -0,0 +1,105 @@ +package bigquery + +import ( + "reflect" + "testing" + + configv1 "github.com/openshift/sippy/pkg/apis/config/v1" + "github.com/stretchr/testify/assert" +) + +func TestCopyIncludeVariantsAndRemoveOverrides(t *testing.T) { + tests := []struct { + name string + overrides []configv1.VariantJunitTableOverride + currOverride int + includeVariants map[string][]string + expected map[string][]string + expectedSkipQuery bool + }{ + { + name: "No overrides, no variants removed", + overrides: []configv1.VariantJunitTableOverride{}, + currOverride: -1, + includeVariants: map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3"}, + }, + expected: map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3"}, + }, + }, + { + name: "Single override removes matching variant", + overrides: []configv1.VariantJunitTableOverride{ + {VariantName: "key1", VariantValue: "value1"}, + }, + currOverride: -1, + includeVariants: map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3"}, + }, + expected: map[string][]string{ + "key1": {"value2"}, + "key2": {"value3"}, + }, + }, + { + name: "Override does not remove its own variant", + overrides: []configv1.VariantJunitTableOverride{ + {VariantName: "key1", VariantValue: "value1"}, + }, + currOverride: 0, + includeVariants: map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3"}, + }, + expected: map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3"}, + }, + }, + { + name: "Multiple overrides remove multiple variants", + overrides: []configv1.VariantJunitTableOverride{ + {VariantName: "key1", VariantValue: "value1"}, + {VariantName: "key2", VariantValue: "value3"}, + }, + currOverride: -1, + includeVariants: map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3", "value4"}, + }, + expected: map[string][]string{ + "key1": {"value2"}, + "key2": {"value4"}, + }, + }, + { + name: "All variants removed", + overrides: []configv1.VariantJunitTableOverride{ + {VariantName: "key1", VariantValue: "value1"}, + {VariantName: "key1", VariantValue: "value2"}, + {VariantName: "key2", VariantValue: "value3"}, + }, + currOverride: -1, + includeVariants: map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3"}, + }, + expected: map[string][]string{}, + expectedSkipQuery: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, skipQuery := copyIncludeVariantsAndRemoveOverrides(tt.overrides, tt.currOverride, tt.includeVariants) + assert.Equal(t, tt.expectedSkipQuery, skipQuery) + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, result) + } + }) + } +} diff --git a/pkg/api/componentreadiness/query/querygenerators.go b/pkg/api/componentreadiness/dataprovider/bigquery/querygenerators.go similarity index 99% rename from pkg/api/componentreadiness/query/querygenerators.go rename to pkg/api/componentreadiness/dataprovider/bigquery/querygenerators.go index 059b6c8e2e..12acf89c26 100644 --- a/pkg/api/componentreadiness/query/querygenerators.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/querygenerators.go @@ -1,4 +1,4 @@ -package query +package bigquery import ( "context" diff --git a/pkg/api/componentreadiness/query/querygenerators_test.go b/pkg/api/componentreadiness/dataprovider/bigquery/querygenerators_test.go similarity index 99% rename from pkg/api/componentreadiness/query/querygenerators_test.go rename to pkg/api/componentreadiness/dataprovider/bigquery/querygenerators_test.go index d98834dd38..bc35d6ec5a 100644 --- a/pkg/api/componentreadiness/query/querygenerators_test.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/querygenerators_test.go @@ -1,4 +1,4 @@ -package query +package bigquery import ( "strings" diff --git a/pkg/api/componentreadiness/query/releasedates.go b/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go similarity index 98% rename from pkg/api/componentreadiness/query/releasedates.go rename to pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go index d18c06d4dd..749414234a 100644 --- a/pkg/api/componentreadiness/query/releasedates.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go @@ -1,4 +1,4 @@ -package query +package bigquery import ( "context" diff --git a/pkg/api/componentreadiness/dataprovider/interface.go b/pkg/api/componentreadiness/dataprovider/interface.go index 106309e02d..37b40f91e2 100644 --- a/pkg/api/componentreadiness/dataprovider/interface.go +++ b/pkg/api/componentreadiness/dataprovider/interface.go @@ -18,12 +18,10 @@ type TestStatusQuerier interface { allJobVariants crtest.JobVariants) (map[string]crstatus.TestStatus, []error) // QuerySampleTestStatus returns test status for the sample release. - // includeVariants may differ from reqOptions for junit table overrides. QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, includeVariants map[string][]string, - start, end time.Time, - dataSource string) (map[string]crstatus.TestStatus, []error) + start, end time.Time) (map[string]crstatus.TestStatus, []error) } // TestDetailsQuerier fetches per-job-run test breakdowns used for test details reports. @@ -34,8 +32,7 @@ type TestDetailsQuerier interface { QuerySampleJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, includeVariants map[string][]string, - start, end time.Time, - dataSource string) (map[string][]crstatus.TestJobRunRows, []error) + start, end time.Time) (map[string][]crstatus.TestJobRunRows, []error) } // MetadataQuerier fetches reference data used to configure and parameterize reports. diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 73ada4e14d..547dc6ad12 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -450,8 +450,7 @@ func (p *PostgresProvider) QueryBaseTestStatus(_ context.Context, reqOptions req func (p *PostgresProvider) QuerySampleTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, includeVariants map[string][]string, - start, end time.Time, - _ string) (map[string]crstatus.TestStatus, []error) { + start, end time.Time) (map[string]crstatus.TestStatus, []error) { dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) for _, k := range reqOptions.VariantOption.DBGroupBy.List() { @@ -648,8 +647,7 @@ func (p *PostgresProvider) QueryBaseJobRunTestStatus(_ context.Context, reqOptio func (p *PostgresProvider) QuerySampleJobRunTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, includeVariants map[string][]string, - start, end time.Time, - _ string) (map[string][]crstatus.TestJobRunRows, []error) { + start, end time.Time) (map[string][]crstatus.TestJobRunRows, []error) { return p.queryTestDetails( reqOptions.SampleRelease.Name, diff --git a/pkg/api/componentreadiness/regressiontracker.go b/pkg/api/componentreadiness/regressiontracker.go index 287f5eb227..326c68ed35 100644 --- a/pkg/api/componentreadiness/regressiontracker.go +++ b/pkg/api/componentreadiness/regressiontracker.go @@ -15,7 +15,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - configv1 "github.com/openshift/sippy/pkg/apis/config/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" @@ -166,34 +165,31 @@ func NewRegressionTracker( releases []v1.Release, backend RegressionStore, views []crview.View, - overrides []configv1.VariantJunitTableOverride, dryRun bool) *RegressionTracker { return &RegressionTracker{ - dataProvider: dataProvider, - dbc: dbc, - cacheOpts: cacheOptions, - releases: releases, - backend: backend, - views: views, - variantJunitTableOverrides: overrides, - dryRun: dryRun, - logger: log.WithField("daemon", "regression-tracker"), + dataProvider: dataProvider, + dbc: dbc, + cacheOpts: cacheOptions, + releases: releases, + backend: backend, + views: views, + dryRun: dryRun, + logger: log.WithField("daemon", "regression-tracker"), } } // RegressionTracker is the primary object for managing regression tracking logic. type RegressionTracker struct { - backend RegressionStore - dataProvider dataprovider.DataProvider - dbc *db.DB - cacheOpts cache.RequestOptions - releases []v1.Release - dryRun bool - views []crview.View - logger log.FieldLogger - variantJunitTableOverrides []configv1.VariantJunitTableOverride - errors []error + backend RegressionStore + dataProvider dataprovider.DataProvider + dbc *db.DB + cacheOpts cache.RequestOptions + releases []v1.Release + dryRun bool + views []crview.View + logger log.FieldLogger + errors []error } func (rt *RegressionTracker) Name() string { @@ -255,7 +251,7 @@ func (rt *RegressionTracker) SyncRegressionsForRelease(ctx context.Context, rele } report, reportErrs := GetComponentReport( - ctx, rt.dataProvider, rt.dbc, reportOpts, rt.variantJunitTableOverrides, "") + ctx, rt.dataProvider, rt.dbc, reportOpts, "") if len(reportErrs) > 0 { var strErrors []string for _, err := range reportErrs { diff --git a/pkg/api/componentreadiness/test_details.go b/pkg/api/componentreadiness/test_details.go index ee6b8d24ae..97d065e243 100644 --- a/pkg/api/componentreadiness/test_details.go +++ b/pkg/api/componentreadiness/test_details.go @@ -21,15 +21,12 @@ import ( "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" - "github.com/openshift/sippy/pkg/api/componentreadiness/query" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" - configv1 "github.com/openshift/sippy/pkg/apis/config/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" - "github.com/openshift/sippy/pkg/util" ) func GetTestDetails(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, reqOptions reqopts.RequestOptions, releases []v1.Release, baseURL string) (testdetails.Report, []error) { - generator := NewComponentReportGenerator(provider, reqOptions, dbc, nil, releases, baseURL) + generator := NewComponentReportGenerator(provider, reqOptions, dbc, releases, baseURL) if os.Getenv("DEV_MODE") == "1" { return generator.GenerateTestDetailsReport(ctx) } @@ -352,10 +349,9 @@ func (c *ComponentReportGenerator) getSampleJobRunTestStatus( ctx context.Context, allJobVariants crtest.JobVariants, includeVariants map[string][]string, - start, end time.Time, - junitTable string) (map[string][]crstatus.TestJobRunRows, []error) { + start, end time.Time) (map[string][]crstatus.TestJobRunRows, []error) { - return c.dataProvider.QuerySampleJobRunTestStatus(ctx, c.ReqOptions, allJobVariants, includeVariants, start, end, junitTable) + return c.dataProvider.QuerySampleJobRunTestStatus(ctx, c.ReqOptions, allJobVariants, includeVariants, start, end) } func (c *ComponentReportGenerator) getJobRunTestStatus(ctx context.Context) (crstatus.TestJobRunStatuses, []error) { @@ -366,14 +362,10 @@ func (c *ComponentReportGenerator) getJobRunTestStatus(ctx context.Context) (crs return crstatus.TestJobRunStatuses{}, errs } var baseStatus, sampleStatus map[string][]crstatus.TestJobRunRows - var baseErrs, baseOverrideErrs, sampleErrs []error + var baseErrs, sampleErrs []error wg := sync.WaitGroup{} - // channels for status as we may collect status from multiple queries run in separate goroutines - statusCh := make(chan map[string][]crstatus.TestJobRunRows) errCh := make(chan error) - statusDoneCh := make(chan struct{}) // To signal when all processing is done - statusErrsDoneCh := make(chan struct{}) // To signal when all processing is done c.middlewares.QueryTestDetails(ctx, &wg, errCh, allJobVariants) @@ -387,7 +379,6 @@ func (c *ComponentReportGenerator) getJobRunTestStatus(ctx context.Context) (crs default: baseStatus, baseErrs = c.getBaseJobRunTestStatus(ctx, allJobVariants, c.ReqOptions.BaseRelease.Name, c.ReqOptions.BaseRelease.Start, c.ReqOptions.BaseRelease.End) } - }() wg.Add(1) @@ -398,101 +389,30 @@ func (c *ComponentReportGenerator) getJobRunTestStatus(ctx context.Context) (crs logrus.Infof("Context canceled while fetching sample job run test status") return default: - includeVariants, skipQuery := copyIncludeVariantsAndRemoveOverrides(c.variantJunitTableOverrides, -1, c.ReqOptions.VariantOption.IncludeVariants) - if skipQuery { - fLog.Infof("skipping default status query as all values for a variant were overridden") - return - } - fLog.Infof("running default status query with includeVariants: %+v", includeVariants) - status, errs := c.getSampleJobRunTestStatus(ctx, allJobVariants, includeVariants, - c.ReqOptions.SampleRelease.Start, c.ReqOptions.SampleRelease.End, query.DefaultJunitTable) - fLog.Infof("received %d test statuses and %d errors from default query", len(status), len(errs)) - statusCh <- status - for _, err := range errs { - errCh <- err - } + fLog.Infof("running sample status query with includeVariants: %+v", c.ReqOptions.VariantOption.IncludeVariants) + status, errs := c.getSampleJobRunTestStatus(ctx, allJobVariants, c.ReqOptions.VariantOption.IncludeVariants, + c.ReqOptions.SampleRelease.Start, c.ReqOptions.SampleRelease.End) + fLog.Infof("received %d test statuses and %d errors from sample query", len(status), len(errs)) + sampleStatus = status + sampleErrs = errs } - }() - // fork additional sample queries for the overrides - for i, or := range c.variantJunitTableOverrides { - if !utils.ContainsOverriddenVariant(c.ReqOptions.VariantOption.IncludeVariants, or.VariantName, or.VariantValue) { - continue - } - // only do this additional query if the specified override variant is actually included in this request - wg.Add(1) - go func(i int, or configv1.VariantJunitTableOverride) { - defer wg.Done() - select { - case <-ctx.Done(): - return - default: - includeVariants, skipQuery := copyIncludeVariantsAndRemoveOverrides(c.variantJunitTableOverrides, i, c.ReqOptions.VariantOption.IncludeVariants) - if skipQuery { - fLog.Infof("skipping override status query as all values for a variant were overridden") - return - } - fLog.Infof("running override status query for %+v with includeVariants: %+v", or, includeVariants) - // Calculate a start time relative to the requested end time: (i.e. for rarely run jobs) - end := c.ReqOptions.SampleRelease.End - start, err := util.ParseCRReleaseTime([]v1.Release{}, "", or.RelativeStart, - true, &c.ReqOptions.SampleRelease.End, c.ReqOptions.CacheOption.CRTimeRoundingFactor) - if err != nil { - errCh <- err - return - } - status, errs := c.getSampleJobRunTestStatus(ctx, allJobVariants, includeVariants, - start, end, or.TableName) - fLog.Infof("received %d job run test statuses and %d errors from override query", len(status), len(errs)) - statusCh <- status - for _, err := range errs { - errCh <- err - } - } - - }(i, or) - } - go func() { wg.Wait() - close(statusCh) close(errCh) }() - go func() { - - for status := range statusCh { - fLog.Infof("received %d job run test statuses over channel", len(status)) - for k, v := range status { - if sampleStatus == nil { - fLog.Warnf("initializing sampleStatus map") - sampleStatus = make(map[string][]crstatus.TestJobRunRows) - } - if v2, ok := sampleStatus[k]; ok { - fLog.Warnf("sampleStatus already had key: %+v", k) - fLog.Warnf("sampleStatus new value: %+v", v) - fLog.Warnf("sampleStatus old value: %+v", v2) - } - sampleStatus[k] = v - } - } - close(statusDoneCh) - }() - - go func() { - for err := range errCh { - sampleErrs = append(sampleErrs, err) - } - close(statusErrsDoneCh) - }() + var middlewareErrs []error + for err := range errCh { + middlewareErrs = append(middlewareErrs, err) + } - <-statusDoneCh - <-statusErrsDoneCh fLog.Infof("total test statuses: %d", len(sampleStatus)) - if len(baseErrs) != 0 || len(baseOverrideErrs) != 0 { + if len(baseErrs) != 0 || len(sampleErrs) != 0 || len(middlewareErrs) != 0 { errs = append(errs, baseErrs...) - errs = append(errs, baseOverrideErrs...) + errs = append(errs, sampleErrs...) + errs = append(errs, middlewareErrs...) } return crstatus.TestJobRunStatuses{BaseStatus: baseStatus, SampleStatus: sampleStatus}, errs diff --git a/pkg/componentreadiness/jiraautomator/jiraautomator.go b/pkg/componentreadiness/jiraautomator/jiraautomator.go index 5f2f07221e..45d3add4af 100644 --- a/pkg/componentreadiness/jiraautomator/jiraautomator.go +++ b/pkg/componentreadiness/jiraautomator/jiraautomator.go @@ -19,7 +19,6 @@ import ( "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" "github.com/openshift/sippy/pkg/apis/cache" - configv1 "github.com/openshift/sippy/pkg/apis/config/v1" jiratype "github.com/openshift/sippy/pkg/apis/jira/v1" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" bqclient "github.com/openshift/sippy/pkg/bigquery" @@ -66,8 +65,7 @@ type JiraAutomator struct { includeComponents sets.String jiraAccount string dryRun bool - variantToJiraComponents map[Variant]JiraComponent - variantJunitTableOverrides []configv1.VariantJunitTableOverride + variantToJiraComponents map[Variant]JiraComponent } func NewJiraAutomator( @@ -83,23 +81,21 @@ func NewJiraAutomator( columnThresholds map[Variant]int, dryRun bool, variantToJiraComponents map[Variant]JiraComponent, - variantJunitTableOverrides []configv1.VariantJunitTableOverride, ) (JiraAutomator, error) { j := JiraAutomator{ - jiraClient: jiraClient, - bqClient: bqClient, - dataProvider: provider, - dbc: dbc, - cacheOptions: cacheOptions, - releases: releases, - sippyURL: sippyURL, - columnThresholds: columnThresholds, - includeComponents: includeComponents, - jiraAccount: jiraAccount, - dryRun: dryRun, - variantToJiraComponents: variantToJiraComponents, - variantJunitTableOverrides: variantJunitTableOverrides, + jiraClient: jiraClient, + bqClient: bqClient, + dataProvider: provider, + dbc: dbc, + cacheOptions: cacheOptions, + releases: releases, + sippyURL: sippyURL, + columnThresholds: columnThresholds, + includeComponents: includeComponents, + jiraAccount: jiraAccount, + dryRun: dryRun, + variantToJiraComponents: variantToJiraComponents, } if bqClient == nil || bqClient.BQ == nil { return j, fmt.Errorf("we don't have a bigquery client for jira integrator") @@ -152,7 +148,7 @@ func (j JiraAutomator) getComponentReportForView(view crview.View) (crtype.Compo } // Passing empty gcs bucket and prow URL, they are not needed outside test details reports - report, errs := componentreadiness.GetComponentReport(context.Background(), j.dataProvider, j.dbc, reportOpts, j.variantJunitTableOverrides, "") + report, errs := componentreadiness.GetComponentReport(context.Background(), j.dataProvider, j.dbc, reportOpts, "") if len(errs) > 0 { var strErrors []string for _, err := range errs { diff --git a/pkg/dataloader/crcacheloader/crcacheloader.go b/pkg/dataloader/crcacheloader/crcacheloader.go index 90725038b1..f71d10af42 100644 --- a/pkg/dataloader/crcacheloader/crcacheloader.go +++ b/pkg/dataloader/crcacheloader/crcacheloader.go @@ -57,7 +57,7 @@ func New( views: views, releases: releases, bqClient: bqClient, - dataProvider: bqprovider.NewBigQueryProvider(bqClient), + dataProvider: bqprovider.NewBigQueryProvider(bqClient, config.ComponentReadinessConfig.VariantJunitTableOverrides), config: config, crTimeRoundingFactor: crTimeRoundingFactor, } @@ -265,6 +265,6 @@ func (l *ComponentReadinessCacheLoader) buildGenerator( // Making a generator directly as we are going to bypass the caching to ensure we get fresh report, // explicitly set our reports in the cache, thus resetting the timer for all expiry and keeping the cache // primed. - generator := componentreadiness.NewComponentReportGenerator(l.dataProvider, reqOpts, l.dbc, l.config.ComponentReadinessConfig.VariantJunitTableOverrides, l.releases, "") + generator := componentreadiness.NewComponentReportGenerator(l.dataProvider, reqOpts, l.dbc, l.releases, "") return &generator, nil } diff --git a/pkg/sippyserver/metrics/metrics.go b/pkg/sippyserver/metrics/metrics.go index f3fc7f92f2..758dc940e7 100644 --- a/pkg/sippyserver/metrics/metrics.go +++ b/pkg/sippyserver/metrics/metrics.go @@ -15,8 +15,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" log "github.com/sirupsen/logrus" - configv1 "github.com/openshift/sippy/pkg/apis/config/v1" - "github.com/openshift/sippy/pkg/api/componentreadiness" "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" @@ -117,7 +115,7 @@ func getReleaseStatus(releases []v1.Release, release string) string { // presume in a historical context there won't be scraping of these metrics // pinning the time just to be consistent -func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crProvider dataprovider.DataProvider, reportEnd time.Time, cacheOptions cache.RequestOptions, views []crview.View, variantJunitTableOverrides []configv1.VariantJunitTableOverride) error { +func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crProvider dataprovider.DataProvider, reportEnd time.Time, cacheOptions cache.RequestOptions, views []crview.View) error { start := time.Now() log.Info("beginning refresh metrics") @@ -168,7 +166,7 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crP } if crProvider != nil { - refreshComponentReadinessMetrics(ctx, crProvider, dbc, cacheOptions, views, releases, variantJunitTableOverrides) + refreshComponentReadinessMetrics(ctx, crProvider, dbc, cacheOptions, views, releases) } // BigQuery-only metrics @@ -184,10 +182,10 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crP } func refreshComponentReadinessMetrics(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, - cacheOptions cache.RequestOptions, views []crview.View, releases []v1.Release, variantJunitTableOverrides []configv1.VariantJunitTableOverride) { + cacheOptions cache.RequestOptions, views []crview.View, releases []v1.Release) { for _, view := range views { if view.Metrics.Enabled { - err := updateComponentReadinessMetricsForView(ctx, provider, dbc, cacheOptions, view, releases, variantJunitTableOverrides) + err := updateComponentReadinessMetricsForView(ctx, provider, dbc, cacheOptions, view, releases) if err != nil { log.WithError(err).Error("error") log.WithError(err).WithField("view", view.Name).Error("error refreshing metrics/regressions for view") @@ -198,7 +196,7 @@ func refreshComponentReadinessMetrics(ctx context.Context, provider dataprovider } // updateComponentReadinessTrackingForView queries the report for the given view, and then updates metrics. -func updateComponentReadinessMetricsForView(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []v1.Release, overrides []configv1.VariantJunitTableOverride) error { +func updateComponentReadinessMetricsForView(ctx context.Context, provider dataprovider.DataProvider, dbc *db.DB, cacheOptions cache.RequestOptions, view crview.View, releases []v1.Release) error { logger := log.WithField("view", view.Name) logger.Info("generating report for view") @@ -227,7 +225,7 @@ func updateComponentReadinessMetricsForView(ctx context.Context, provider datapr CacheOption: cacheOptions, } - report, errs := componentreadiness.GetComponentReport(ctx, provider, dbc, reportOpts, overrides, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, dbc, reportOpts, "") if len(errs) > 0 { var strErrors []string for _, err := range errs { diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 9df846c2f1..9b91f55f20 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -1032,7 +1032,6 @@ func (s *Server) getComponentReportFromRequest(req *http.Request) (componentrepo s.crDataProvider, s.db, options, - s.config.ComponentReadinessConfig.VariantJunitTableOverrides, baseURL, ) if len(errs) > 0 { diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 0880201ae4..420b73243f 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -25,6 +25,11 @@ clean_up () { echo "Generating coverage report..." go tool covdata percent -i="$COVDIR" go tool covdata textfmt -i="$COVDIR" -o=e2e-coverage.out + # Merge test binary coverage (from -coverprofile) into server binary coverage + if [ -f e2e-test-coverage.out ]; then + echo "Merging test binary coverage into server coverage..." + tail -n +2 e2e-test-coverage.out >> e2e-coverage.out + fi echo "Coverage data written to e2e-coverage.out" echo "View HTML report: go tool cover -html=e2e-coverage.out -o=e2e-coverage.html" fi @@ -117,6 +122,6 @@ done echo "Cache priming complete" # Run our tests that request against the API, args ensure serially and fresh test code compile: -gotestsum ./test/e2e/... -count 1 -p 1 +gotestsum ./test/e2e/... -count 1 -p 1 -coverprofile=e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... # WARNING: do not place more commands here without addressing return code from go test not being overridden by the cleanup func diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/componentreadiness_test.go index a70bb39aab..ca99847592 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/componentreadiness_test.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "testing" + "time" "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" @@ -85,6 +86,92 @@ func TestTestDetails(t *testing.T) { } } +func TestTestDetailsWithFallback(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // Request test details for test-fallback-improves with testBasisRelease=4.20. + // This exercises the releasefallback middleware's QueryTestDetails path which + // queries base job run status for the override release. + params := url.Values{} + params.Set("view", viewName) + params.Set("testId", "test-fallback-improves") + params.Set("component", "comp-FallbackImproves") + params.Set("capability", "cap1") + params.Set("testBasisRelease", "4.20") + // All DBGroupBy variants must be specified for test details + params.Set("Architecture", "amd64") + params.Set("Platform", "aws") + params.Set("Network", "ovn") + params.Set("Topology", "ha") + params.Set("Installer", "ipi") + params.Set("FeatureSet", "default") + params.Set("Suite", "parallel") + params.Set("Upgrade", "none") + params.Set("LayeredProduct", "none") + + var details testdetails.Report + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/test_details?%s", params.Encode()), &details) + require.NoError(t, err, "error getting fallback test details") + + assert.Equal(t, "test-fallback-improves", details.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // The fallback path should produce analyses with sample data + for _, analysis := range details.Analyses { + sampleTotal := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, sampleTotal, 0, "sample stats should have run data") + } +} + +func TestTestDetailsForNewTestAPI(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // Request test details for test-new-test-pass-rate-fail which has no base data. + // This exercises the GenerateDetailsReportForTest path where BaseStats is nil. + params := url.Values{} + params.Set("view", viewName) + params.Set("testId", "test-new-test-pass-rate-fail") + params.Set("component", "comp-NewTestPassRate") + params.Set("capability", "cap1") + params.Set("Architecture", "amd64") + params.Set("Platform", "aws") + params.Set("Network", "ovn") + params.Set("Topology", "ha") + params.Set("Installer", "ipi") + params.Set("FeatureSet", "default") + params.Set("Suite", "parallel") + params.Set("Upgrade", "none") + params.Set("LayeredProduct", "none") + + var details testdetails.Report + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/test_details?%s", params.Encode()), &details) + require.NoError(t, err, "error getting new test details") + + assert.Equal(t, "test-new-test-pass-rate-fail", details.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + for _, analysis := range details.Analyses { + sampleTotal := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, sampleTotal, 0, "sample stats should have run data") + } +} + +func TestReportWithIncludeVariantsAPI(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&includeVariant=Platform:aws", viewName), &report) + require.NoError(t, err, "error getting filtered report") + require.NotEmpty(t, report.Rows, "filtered report should have rows") + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Equal(t, "aws", col.ColumnIdentification.Variants["Platform"], + "all columns should be aws when filtering by Platform:aws") + } + } +} + func TestVariants(t *testing.T) { // /api/component_readiness/variants returns CacheVariants (legacy BQ column names) var variants map[string][]string @@ -221,6 +308,38 @@ func TestColumnGroupByAndDBGroupBy(t *testing.T) { }) } +func TestComponentReadinessWithExplicitParams(t *testing.T) { + // Test using explicit date/release params instead of a view. + // This exercises parseDateRange and parseAdvancedOptions. + now := time.Now().UTC().Truncate(time.Hour) + params := url.Values{} + params.Set("baseRelease", "4.21") + params.Set("sampleRelease", "4.22") + params.Set("baseStartTime", now.Add(-60*24*time.Hour).Format(time.RFC3339)) + params.Set("baseEndTime", now.Add(-30*24*time.Hour).Format(time.RFC3339)) + params.Set("sampleStartTime", now.Add(-3*24*time.Hour).Format(time.RFC3339)) + params.Set("sampleEndTime", now.Format(time.RFC3339)) + params.Set("confidence", "95") + params.Set("minFail", "3") + params.Set("pity", "5") + params.Set("passRateNewTests", "90") + params.Set("columnGroupBy", "Network,Platform,Topology") + params.Set("dbGroupBy", "Architecture,FeatureSet,Installer,Network,Platform,Suite,Topology,Upgrade,LayeredProduct") + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?%s", params.Encode()), &report) + require.NoError(t, err, "explicit params request should succeed") + assert.NotEmpty(t, report.Rows, "report with explicit params should have rows") + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Contains(t, col.ColumnIdentification.Variants, "Topology") + } + } +} + func TestVariantCrossCompare(t *testing.T) { viewName := fmt.Sprintf("%s-main", util.Release) diff --git a/test/e2e/componentreadiness/report/report_test.go b/test/e2e/componentreadiness/report/report_test.go index 43b5b06530..bff375a34a 100644 --- a/test/e2e/componentreadiness/report/report_test.go +++ b/test/e2e/componentreadiness/report/report_test.go @@ -42,6 +42,7 @@ func setupProvider(t *testing.T) (*pgprovider.PostgresProvider, reqopts.RequestO Confidence: 95, PityFactor: 5, MinimumFailure: 3, + PassRateRequiredNewTests: 90, IncludeMultiReleaseAnalysis: true, }, CacheOption: cache.RequestOptions{}, @@ -54,7 +55,7 @@ func TestReportStatuses(t *testing.T) { provider, reqOptions := setupProvider(t) ctx := context.Background() - report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") require.Empty(t, errs, "GetComponentReport returned errors: %v", errs) require.NotEmpty(t, report.Rows, "report should have rows") @@ -120,7 +121,7 @@ func TestTestDetails(t *testing.T) { require.NoError(t, err) // Generate report to find a regressed test - report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") require.Empty(t, errs) var testID reqopts.TestIdentification @@ -167,7 +168,7 @@ func TestFallback(t *testing.T) { provider, reqOptions := setupProvider(t) ctx := context.Background() - report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") require.Empty(t, errs, "GetComponentReport returned errors: %v", errs) type regressedInfo struct { @@ -217,7 +218,7 @@ func TestFallbackInsufficientRuns(t *testing.T) { provider, reqOptions := setupProvider(t) ctx := context.Background() - report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") require.Empty(t, errs) found := false @@ -241,7 +242,7 @@ func TestMissingBasis(t *testing.T) { provider, reqOptions := setupProvider(t) ctx := context.Background() - report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") require.Empty(t, errs) hasMissingBasis := false @@ -257,15 +258,171 @@ func TestMissingBasis(t *testing.T) { } } - assert.True(t, hasMissingBasis, "report should have at least one MissingBasis cell") + assert.True(t, hasMissingBasis, "report should have at least one MissingBasis cell (new test with good pass rate)") assert.True(t, hasMissingSample, "report should have at least one MissingSample cell") } +func TestNewTestPassRateRegression(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + // The new flaky test (80% pass rate) should be flagged as a regression + // via buildPassRateTestStats since it's below the 90% PassRateRequiredNewTests threshold. + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-new-test-pass-rate-fail" { + found = true + assert.Equal(t, crtest.ExtremeRegression, rt.ReportStatus, + "new test with 70%% pass rate should be an extreme regression (below 80%% extreme threshold)") + assert.Equal(t, crtest.PassRate, rt.Comparison, + "new test regression should use pass rate comparison, not fisher exact") + } + } + } + } + assert.True(t, found, "test-new-test-pass-rate-fail should appear as a regressed test") +} + +func TestTestDetailsForFallbackTest(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + releases, err := provider.QueryReleases(ctx) + require.NoError(t, err) + + // test-fallback-improves has data in 4.21 (worse) and 4.20 (better), + // so the report should override the base to 4.20. + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + var testID reqopts.TestIdentification + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-fallback-improves" { + testID = reqopts.TestIdentification{ + Component: rt.RowIdentification.Component, + Capability: rt.RowIdentification.Capability, + TestID: rt.RowIdentification.TestID, + RequestedVariants: rt.ColumnIdentification.Variants, + BaseOverrideRelease: "4.20", + } + found = true + } + } + } + } + require.True(t, found, "test-fallback-improves should be in the report") + + detailReqOpts := reqOptions + detailReqOpts.TestIDOptions = []reqopts.TestIdentification{testID} + + details, detailErrs := componentreadiness.GetTestDetails(ctx, provider, nil, detailReqOpts, releases, "") + require.Empty(t, detailErrs, "GetTestDetails returned errors: %v", detailErrs) + + assert.Equal(t, "test-fallback-improves", details.Identification.RowIdentification.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // The fallback path should produce an analysis with base override data + require.GreaterOrEqual(t, len(details.Analyses), 1) + for _, analysis := range details.Analyses { + total := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, total, 0, "sample stats should have run data") + } +} + +func TestTestDetailsForNewTest(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + releases, err := provider.QueryReleases(ctx) + require.NoError(t, err) + + // test-new-test-pass-rate-fail only exists in 4.22 sample, no base data. + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + var testID reqopts.TestIdentification + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-new-test-pass-rate-fail" { + testID = reqopts.TestIdentification{ + Component: rt.RowIdentification.Component, + Capability: rt.RowIdentification.Capability, + TestID: rt.RowIdentification.TestID, + RequestedVariants: rt.ColumnIdentification.Variants, + } + found = true + } + } + } + } + require.True(t, found, "test-new-test-pass-rate-fail should be in the report") + + detailReqOpts := reqOptions + detailReqOpts.TestIDOptions = []reqopts.TestIdentification{testID} + + details, detailErrs := componentreadiness.GetTestDetails(ctx, provider, nil, detailReqOpts, releases, "") + require.Empty(t, detailErrs, "GetTestDetails returned errors: %v", detailErrs) + + assert.Equal(t, "test-new-test-pass-rate-fail", details.Identification.RowIdentification.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // New test should have sample data but no base data + for _, analysis := range details.Analyses { + sampleTotal := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, sampleTotal, 0, "sample stats should have run data") + if analysis.BaseStats != nil { + baseTotal := analysis.BaseStats.SuccessCount + analysis.BaseStats.FailureCount + analysis.BaseStats.FlakeCount + assert.Equal(t, 0, baseTotal, "base stats should have no run data for a new test") + } + } +} + +func TestReportDevMode(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + t.Setenv("DEV_MODE", "1") + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs, "GetComponentReport in DEV_MODE returned errors: %v", errs) + assert.NotEmpty(t, report.Rows, "DEV_MODE report should still have rows") +} + +func TestReportWithIncludeVariants(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + reqOptions.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + } + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs, "GetComponentReport with IncludeVariants returned errors: %v", errs) + require.NotEmpty(t, report.Rows, "filtered report should have rows") + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Equal(t, "aws", col.ColumnIdentification.Variants["Platform"], + "all columns should be aws when filtering by Platform:aws") + } + } +} + func TestSignificantImprovement(t *testing.T) { provider, reqOptions := setupProvider(t) ctx := context.Background() - report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, nil, "") + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") require.Empty(t, errs) hasImprovement := false diff --git a/test/e2e/componentreadiness/triage/triageapi_test.go b/test/e2e/componentreadiness/triage/triageapi_test.go index bbdf010e96..d6762ac1ce 100644 --- a/test/e2e/componentreadiness/triage/triageapi_test.go +++ b/test/e2e/componentreadiness/triage/triageapi_test.go @@ -83,6 +83,22 @@ func Test_TriageAPI(t *testing.T) { require.Error(t, err) }) + t.Run("create fails with non-existent regression ID", func(t *testing.T) { + defer cleanupAllTriages(dbc) + triage := models.Triage{ + URL: jiraBug.URL, + Type: models.TriageTypeProduct, + Regressions: []models.TestRegression{ + {ID: testRegression1.ID}, + {ID: 999999}, // non-existent + }, + } + + var triageResponse models.Triage + err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) + require.Error(t, err, "should fail when a regression ID does not exist") + }) + t.Run("create generates audit_log record", func(t *testing.T) { defer cleanupAllTriages(dbc) triage1 := models.Triage{ From 6a89578a233d206a20283a87cee70129059f90c0 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 14:51:58 -0400 Subject: [PATCH 08/25] Fix gofmt formatting in jiraautomator and server Co-Authored-By: Claude Opus 4.6 --- pkg/componentreadiness/jiraautomator/jiraautomator.go | 8 ++++---- pkg/sippyserver/server.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/componentreadiness/jiraautomator/jiraautomator.go b/pkg/componentreadiness/jiraautomator/jiraautomator.go index 45d3add4af..39e1ebe9ed 100644 --- a/pkg/componentreadiness/jiraautomator/jiraautomator.go +++ b/pkg/componentreadiness/jiraautomator/jiraautomator.go @@ -61,10 +61,10 @@ type JiraAutomator struct { // columnThresholds defines a threshold for the number of red cells in a column. // When the number of red cells of a column is over this threshold, a jira card will be created for the // Variant (column) based jira component. No other jira cards will be created per component row. - columnThresholds map[Variant]int - includeComponents sets.String - jiraAccount string - dryRun bool + columnThresholds map[Variant]int + includeComponents sets.String + jiraAccount string + dryRun bool variantToJiraComponents map[Variant]JiraComponent } diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 9b91f55f20..8f6939c71d 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -12,11 +12,11 @@ import ( "os" "os/signal" "regexp" - "syscall" sorting "sort" "strconv" "strings" "sync" + "syscall" "time" "cloud.google.com/go/storage" From 1acebbc3ea5558f18b34dbe1cea56b41f819c7e1 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 15:01:14 -0400 Subject: [PATCH 09/25] Add coverage instrumentation to CI e2e test script Add coverage collection to the Prow CI e2e script: build with sippy-cover, set GOCOVERDIR, collect server + test binary coverage profiles and merge them into a single report. Co-Authored-By: Claude Opus 4.6 --- .../sippy-e2e-sippy-e2e-test-commands.sh | 75 +++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index b8302420f1..feb3c0d83e 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -22,7 +22,7 @@ echo "The sippy CI image: ${SIPPY_IMAGE}" KUBECTL_CMD="${KUBECTL_CMD:=oc}" echo "The kubectl command is: ${KUBECTL_CMD}" -# Launch the sippy api server pod. +# Launch the sippy api server pod with coverage instrumentation. cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 kind: Pod @@ -54,7 +54,7 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File command: - - /bin/sippy + - /bin/sippy-cover args: - serve - --listen @@ -72,8 +72,18 @@ spec: - ocp - --views - ./config/e2e-views.yaml + env: + - name: GOCOVERDIR + value: /tmp/coverage + volumeMounts: + - mountPath: /tmp/coverage + name: coverage imagePullSecrets: - name: regcred + volumes: + - name: coverage + persistentVolumeClaim: + claimName: sippy-coverage dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler @@ -121,7 +131,62 @@ ${KUBECTL_CMD} -n sippy-e2e port-forward pod/redis1 ${SIPPY_REDIS_PORT}:6379 & ${KUBECTL_CMD} -n sippy-e2e get svc,ep -${KUBECTL_CMD} -n sippy-e2e delete secret regcred - # only 1 in parallel, some tests will clash if run at the same time -gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e.xml -- ./test/e2e/... -v -p 1 +gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e.xml -- ./test/e2e/... -v -p 1 -coverprofile=${ARTIFACT_DIR}/e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... +TEST_EXIT=$? + +# Collect coverage data. Coverage counters are flushed when the server exits. +# Pod deletion sends SIGTERM during graceful termination (terminationGracePeriodSeconds: 30), +# so we just delete the pod directly — no need for a separate exec kill. +echo "Stopping sippy-server to flush coverage data..." +${KUBECTL_CMD} -n sippy-e2e delete pod sippy-server --wait=true --timeout=60s || true + +# Launch a minimal helper pod to access the coverage PVC. +cat << END | ${KUBECTL_CMD} apply -f - +apiVersion: v1 +kind: Pod +metadata: + name: coverage-helper + namespace: sippy-e2e +spec: + containers: + - name: helper + image: ${SIPPY_IMAGE} + command: ["sleep", "300"] + volumeMounts: + - mountPath: /tmp/coverage + name: coverage + readOnly: true + imagePullSecrets: + - name: regcred + volumes: + - name: coverage + persistentVolumeClaim: + claimName: sippy-coverage + restartPolicy: Never +END + +${KUBECTL_CMD} -n sippy-e2e wait --for=condition=Ready pod/coverage-helper --timeout=60s + +COVDIR=$(mktemp -d) +${KUBECTL_CMD} -n sippy-e2e cp coverage-helper:/tmp/coverage "${COVDIR}" -c helper || true + +if find "${COVDIR}" -name 'covcounters.*' -print -quit 2>/dev/null | grep -q .; then + echo "Generating coverage report..." + go tool covdata percent -i="${COVDIR}" + go tool covdata textfmt -i="${COVDIR}" -o="${ARTIFACT_DIR}/e2e-coverage.out" + # Merge test binary coverage (from -coverprofile) into server binary coverage + if [ -f "${ARTIFACT_DIR}/e2e-test-coverage.out" ]; then + echo "Merging test binary coverage into server coverage..." + tail -n +2 "${ARTIFACT_DIR}/e2e-test-coverage.out" >> "${ARTIFACT_DIR}/e2e-coverage.out" + fi + go tool cover -html="${ARTIFACT_DIR}/e2e-coverage.out" -o="${ARTIFACT_DIR}/e2e-coverage.html" + echo "Coverage report written to ${ARTIFACT_DIR}/e2e-coverage.html" +else + echo "WARNING: No coverage data found" +fi +rm -rf "${COVDIR}" + +${KUBECTL_CMD} -n sippy-e2e delete secret regcred || true + +exit ${TEST_EXIT} From 80edd646539cf305462966d4abd02e4c38a4d661 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 15:29:18 -0400 Subject: [PATCH 10/25] Add sippy-cover binary to Dockerfile and coverage PVC to CI setup Build a coverage-instrumented sippy-cover binary in the Dockerfile and create a PVC in the CI setup script to persist coverage data across pod restarts. Co-Authored-By: Claude Opus 4.6 --- Dockerfile | 2 ++ e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/Dockerfile b/Dockerfile index 78f92688e9..d7cc5b814d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,13 @@ COPY . . ENV PATH="/go/bin:${PATH}" ENV GOPATH="/go" RUN dnf module enable nodejs:18 -y && dnf install -y go make npm && make build +RUN go build -cover -coverpkg=./cmd/...,./pkg/... -mod=vendor -o ./sippy-cover ./cmd/sippy FROM registry.access.redhat.com/ubi9/ubi:latest AS base RUN mkdir -p /historical-data RUN mkdir -p /config COPY --from=builder /go/src/sippy/sippy /bin/sippy +COPY --from=builder /go/src/sippy/sippy-cover /bin/sippy-cover COPY --from=builder /go/src/sippy/sippy-daemon /bin/sippy-daemon COPY --from=builder /go/src/sippy/scripts/fetchdata.sh /bin/fetchdata.sh COPY --from=builder /go/src/sippy/config/*.yaml /config/ diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh index ab60a3214d..470818d9b4 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh @@ -202,6 +202,21 @@ ${KUBECTL_CMD} -n sippy-e2e get svc,ep # Get the registry credentials for all build farm clusters out to the cluster-pool cluster. ${KUBECTL_CMD} -n sippy-e2e create secret generic regcred --from-file=.dockerconfigjson=${DOCKERCONFIGJSON} --type=kubernetes.io/dockerconfigjson +# Create a PVC for coverage data that outlives the server pod. +cat << END | ${KUBECTL_CMD} apply -f - +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sippy-coverage + namespace: sippy-e2e +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +END + # Seed synthetic data into postgres. cat << END | ${KUBECTL_CMD} apply -f - apiVersion: batch/v1 From 25783e039fb36f5bbfb090827aeb942de235a454 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 15:36:24 -0400 Subject: [PATCH 11/25] Address CodeRabbit review feedback - automatejira: include error details in GetJobVariants failure - seed_data: make test_failures UPDATE failure fatal - serve: initialize GCS client independently of data provider - component_report: recompute cell status from post-analysis results so middleware (triage) can improve cells, not just worsen them - postgres provider: propagate context into query methods, add job name filter (periodic/release/aggregator) for BQ parity - metrics: guard against nil crProvider to prevent panic - triage tests: sort regressions by TestID for deterministic ordering - setup script: quote shell variables Co-Authored-By: Claude Opus 4.6 --- cmd/sippy/automatejira.go | 2 +- cmd/sippy/seed_data.go | 2 +- cmd/sippy/serve.go | 16 ++++++++-------- .../sippy-e2e-sippy-e2e-setup-commands.sh | 4 ++-- pkg/api/componentreadiness/component_report.go | 13 +++++++------ .../dataprovider/postgres/provider.go | 11 +++++++---- pkg/sippyserver/metrics/metrics.go | 10 +++++++--- .../componentreadiness/triage/triageapi_test.go | 4 ++++ 8 files changed, 37 insertions(+), 25 deletions(-) diff --git a/cmd/sippy/automatejira.go b/cmd/sippy/automatejira.go index d0b0289c10..8f08b5f9ef 100644 --- a/cmd/sippy/automatejira.go +++ b/cmd/sippy/automatejira.go @@ -173,7 +173,7 @@ func NewAutomateJiraCommand() *cobra.Command { provider := bqprovider.NewBigQueryProvider(bigQueryClient, config.ComponentReadinessConfig.VariantJunitTableOverrides) allVariants, errs := componentreadiness.GetJobVariants(ctx, provider) if len(errs) > 0 { - return fmt.Errorf("failed to get job variants") + return fmt.Errorf("failed to get job variants: %v", errs) } variantToJiraComponents, err := jiraautomator.GetVariantJiraMap(ctx, bigQueryClient) if err != nil { diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index 764ea188ce..698a379b60 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -672,7 +672,7 @@ func seedRunsForJob(dbc *db.DB, suite *models.Suite, prowJob models.ProwJob, jrK SELECT COUNT(*) FROM prow_job_run_tests WHERE prow_job_run_id = prow_job_runs.id AND status = 12 ), 0) WHERE prow_job_id = ?`, prowJob.ID).Error; err != nil { - log.WithError(err).Warn("failed to update test_failures count") + return 0, 0, fmt.Errorf("updating test_failures for prow job %s: %w", prowJob.Name, err) } return runCount, totalResults, nil diff --git a/cmd/sippy/serve.go b/cmd/sippy/serve.go index 8e02bd823a..631a85d5ea 100644 --- a/cmd/sippy/serve.go +++ b/cmd/sippy/serve.go @@ -134,14 +134,6 @@ func NewServeCommand() *cobra.Command { } crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient, config.ComponentReadinessConfig.VariantJunitTableOverrides) - - gcsClient, err = gcs.NewGCSClient(context.TODO(), - f.GoogleCloudFlags.ServiceAccountCredentialFile, - f.GoogleCloudFlags.OAuthClientCredentialFile, - ) - if err != nil { - log.WithError(err).Warn("unable to create GCS client, some APIs may not work") - } } case "postgres": @@ -152,6 +144,14 @@ func NewServeCommand() *cobra.Command { return fmt.Errorf("unknown --data-provider %q, must be bigquery or postgres", f.DataProvider) } + gcsClient, err = gcs.NewGCSClient(context.TODO(), + f.GoogleCloudFlags.ServiceAccountCredentialFile, + f.GoogleCloudFlags.OAuthClientCredentialFile, + ) + if err != nil { + log.WithError(err).Warn("unable to create GCS client, some APIs may not work") + } + // Make sure the db is initialized, otherwise let the user know: prowJobs := []models.ProwJob{} res := dbc.DB.Find(&prowJobs).Limit(1) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh index 470818d9b4..ae3cc0488d 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh @@ -258,12 +258,12 @@ ${KUBECTL_CMD} -n sippy-e2e describe job sippy-seed-job set +e echo "Waiting up to ${SIPPY_LOAD_TIMEOUT:=1200s} for the sippy-seed-job to complete..." -${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-seed-job --timeout ${SIPPY_LOAD_TIMEOUT} +${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-seed-job --timeout "${SIPPY_LOAD_TIMEOUT}" retVal=$? set -e job_pod=$(${KUBECTL_CMD} -n sippy-e2e get pod --selector=job-name=sippy-seed-job --output=jsonpath='{.items[0].metadata.name}') -${KUBECTL_CMD} -n sippy-e2e logs ${job_pod} > ${ARTIFACT_DIR}/sippy-seed.log +${KUBECTL_CMD} -n sippy-e2e logs "${job_pod}" > "${ARTIFACT_DIR}/sippy-seed.log" if [ ${retVal} -ne 0 ]; then echo "sippy seeding never finished on time." diff --git a/pkg/api/componentreadiness/component_report.go b/pkg/api/componentreadiness/component_report.go index eed74c9709..a75a3cb772 100644 --- a/pkg/api/componentreadiness/component_report.go +++ b/pkg/api/componentreadiness/component_report.go @@ -126,10 +126,9 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport) // Give middleware their chance to adjust the result for ri, row := range report.Rows { for ci, col := range row.Columns { - // Track the worst (most negative) status across all regressed tests after PostAnalysis. - // We only hit this loop if there are regressed tests, which is good because we know the - // cell status can't be improved or missing basis/sample. - worstStatus := report.Rows[ri].Columns[ci].Status + // Recompute cell status from post-analysis results. We start fresh so + // middleware (e.g. triage) can improve a cell's status, not just worsen it. + var worstStatus crtest.Status for rti := range col.RegressedTests { testKey := crtest.Identification{ RowIdentification: col.RegressedTests[rti].RowIdentification, @@ -138,11 +137,13 @@ func (c *ComponentReportGenerator) PostAnalysis(report *crtype.ComponentReport) if err := c.middlewares.PostAnalysis(testKey, &report.Rows[ri].Columns[ci].RegressedTests[rti].TestComparison); err != nil { return err } - if newStatus := report.Rows[ri].Columns[ci].RegressedTests[rti].TestComparison.ReportStatus; newStatus < worstStatus { + if newStatus := report.Rows[ri].Columns[ci].RegressedTests[rti].TestComparison.ReportStatus; rti == 0 || newStatus < worstStatus { worstStatus = newStatus } } - report.Rows[ri].Columns[ci].Status = worstStatus + if len(col.RegressedTests) > 0 { + report.Rows[ri].Columns[ci].Status = worstStatus + } } } diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 547dc6ad12..ef09207645 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -331,12 +331,12 @@ WHERE tow.staff_approved_obsolete = false GROUP BY tow.unique_id, t.name, s.name, tow.component, tow.capabilities, d.prow_job_id ` -func (p *PostgresProvider) queryTestStatus(release string, start, end time.Time, +func (p *PostgresProvider) queryTestStatus(ctx context.Context, release string, start, end time.Time, _ crtest.JobVariants, includeVariants map[string][]string, dbGroupBy map[string]bool) (map[string]crstatus.TestStatus, []error) { var rows []testStatusRow - if err := p.dbc.DB.Raw(testStatusQuery, release, start, end).Scan(&rows).Error; err != nil { + if err := p.dbc.DB.WithContext(ctx).Raw(testStatusQuery, release, start, end).Scan(&rows).Error; err != nil { return nil, []error{fmt.Errorf("querying test status: %w", err)} } @@ -424,7 +424,7 @@ func (p *PostgresProvider) fetchJobVariants(rows []testStatusRow) map[uint]map[s return result } -func (p *PostgresProvider) QueryBaseTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, +func (p *PostgresProvider) QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants) (map[string]crstatus.TestStatus, []error) { dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) @@ -438,6 +438,7 @@ func (p *PostgresProvider) QueryBaseTestStatus(_ context.Context, reqOptions req } return p.queryTestStatus( + ctx, reqOptions.BaseRelease.Name, reqOptions.BaseRelease.Start, reqOptions.BaseRelease.End, @@ -447,7 +448,7 @@ func (p *PostgresProvider) QueryBaseTestStatus(_ context.Context, reqOptions req ) } -func (p *PostgresProvider) QuerySampleTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, +func (p *PostgresProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, allJobVariants crtest.JobVariants, includeVariants map[string][]string, start, end time.Time) (map[string]crstatus.TestStatus, []error) { @@ -462,6 +463,7 @@ func (p *PostgresProvider) QuerySampleTestStatus(_ context.Context, reqOptions r } return p.queryTestStatus( + ctx, reqOptions.SampleRelease.Name, start, end, allJobVariants, @@ -679,6 +681,7 @@ func (p *PostgresProvider) QueryJobRuns(_ context.Context, reqOptions reqopts.Re WHERE pj.release = ? AND pjr.timestamp >= ? AND pjr.timestamp < ? AND pj.deleted_at IS NULL AND pjr.deleted_at IS NULL + AND (pj.name LIKE 'periodic-%%' OR pj.name LIKE 'release-%%' OR pj.name LIKE 'aggregator-%%') GROUP BY pj.name ORDER BY pj.name `, release, start, end).Scan(&rows).Error diff --git a/pkg/sippyserver/metrics/metrics.go b/pkg/sippyserver/metrics/metrics.go index 758dc940e7..745b17505a 100644 --- a/pkg/sippyserver/metrics/metrics.go +++ b/pkg/sippyserver/metrics/metrics.go @@ -119,9 +119,13 @@ func RefreshMetricsDB(ctx context.Context, dbc *db.DB, bqc *bqclient.Client, crP start := time.Now() log.Info("beginning refresh metrics") - releases, err := crProvider.QueryReleases(ctx) - if err != nil { - return err + var releases []v1.Release + if crProvider != nil { + var err error + releases, err = crProvider.QueryReleases(ctx) + if err != nil { + return err + } } // Local DB metrics diff --git a/test/e2e/componentreadiness/triage/triageapi_test.go b/test/e2e/componentreadiness/triage/triageapi_test.go index d6762ac1ce..bf429da3d8 100644 --- a/test/e2e/componentreadiness/triage/triageapi_test.go +++ b/test/e2e/componentreadiness/triage/triageapi_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "os" + "sort" "strings" "testing" "time" @@ -965,6 +966,9 @@ func getSeedDataRegressions(t *testing.T) []models.TestRegression { seedRegressions = append(seedRegressions, r) } } + sort.Slice(seedRegressions, func(i, j int) bool { + return seedRegressions[i].TestID < seedRegressions[j].TestID + }) return seedRegressions } From de9f1ec1277d670f3f65a076fd9eb763103fe794 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 15:36:56 -0400 Subject: [PATCH 12/25] Add .out --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6a9ae9fa38..a4905127eb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ report.sh /sippy-ng/build/* .env *.log +*.out /e2e-coverage/ /e2e-coverage.out /e2e-coverage.html From 1467315f3f027afd95a46116559897a10ebc0eb6 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 19:02:52 -0400 Subject: [PATCH 13/25] Mem and remove artifact --- e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh | 3 ++- scripts/e2e.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index feb3c0d83e..afc28f5471 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -50,7 +50,7 @@ spec: - "Wait for a short time" resources: limits: - memory: 5Gi + memory: 8Gi terminationMessagePath: /dev/termination-log terminationMessagePolicy: File command: @@ -179,6 +179,7 @@ if find "${COVDIR}" -name 'covcounters.*' -print -quit 2>/dev/null | grep -q .; if [ -f "${ARTIFACT_DIR}/e2e-test-coverage.out" ]; then echo "Merging test binary coverage into server coverage..." tail -n +2 "${ARTIFACT_DIR}/e2e-test-coverage.out" >> "${ARTIFACT_DIR}/e2e-coverage.out" + rm -f "${ARTIFACT_DIR}/e2e-test-coverage.out" fi go tool cover -html="${ARTIFACT_DIR}/e2e-coverage.out" -o="${ARTIFACT_DIR}/e2e-coverage.html" echo "Coverage report written to ${ARTIFACT_DIR}/e2e-coverage.html" diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 420b73243f..7f2047e4dd 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -29,6 +29,7 @@ clean_up () { if [ -f e2e-test-coverage.out ]; then echo "Merging test binary coverage into server coverage..." tail -n +2 e2e-test-coverage.out >> e2e-coverage.out + rm -f e2e-test-coverage.out fi echo "Coverage data written to e2e-coverage.out" echo "View HTML report: go tool cover -html=e2e-coverage.out -o=e2e-coverage.html" From 2366f7303e6f415e9299b0267a16f67f48fcb108 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 20:45:45 -0400 Subject: [PATCH 14/25] Use coverage-instrumented binary for datasync e2e test Prefer sippy-cover over sippy when available, and propagate GOCOVERDIR to the subprocess so load command coverage is captured. Co-Authored-By: Claude Opus 4.6 --- test/e2e/datasync/datasync_test.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/e2e/datasync/datasync_test.go b/test/e2e/datasync/datasync_test.go index dbbf2e95a4..5f57853a37 100644 --- a/test/e2e/datasync/datasync_test.go +++ b/test/e2e/datasync/datasync_test.go @@ -28,7 +28,22 @@ func TestDataSync(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - cmd := exec.CommandContext(ctx, repoRoot+"/sippy", "load", // #nosec G204 + // Prefer the coverage-instrumented binary if available + sippyBin := "" + for _, candidate := range []string{ + repoRoot + "/sippy-cover", + "/bin/sippy-cover", + repoRoot + "/sippy", + "/bin/sippy", + } { + if _, err := os.Stat(candidate); err == nil { + sippyBin = candidate + break + } + } + require.NotEmpty(t, sippyBin, "could not find sippy binary") + + cmd := exec.CommandContext(ctx, sippyBin, "load", // #nosec G204 "--loader", "prow", "--release", util.Release, "--prow-load-since", "2h", @@ -41,6 +56,9 @@ func TestDataSync(t *testing.T) { cmd.Dir = repoRoot cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + if coverDir := os.Getenv("GOCOVERDIR"); coverDir != "" { + cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir) + } err := cmd.Run() require.NoError(t, err, "sippy load command should complete without error") From 0f47bcd664ad143fef67a4a0e880148cce267641 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 20:51:56 -0400 Subject: [PATCH 15/25] Add TestTriageAffectsReportStatus e2e test Exercises the regression tracker PostAnalysis middleware path where triages adjust report cell status from SignificantRegression to SignificantTriagedRegression (or Extreme equivalent). This improves regressiontracker middleware coverage from ~20% to ~71% for PostAnalysis. Co-Authored-By: Claude Opus 4.6 --- .../componentreadiness_test.go | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/componentreadiness_test.go index ca99847592..cfafb1487f 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/componentreadiness_test.go @@ -377,3 +377,84 @@ func TestVariantCrossCompare(t *testing.T) { t.Logf("cross-compare (no compareVariant) report: %d rows", len(report.Rows)) }) } + +func TestTriageAffectsReportStatus(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // Get a report and find a regressed test + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &report) + require.NoError(t, err) + + var regressionID uint + var originalStatus crtest.Status + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.ReportStatus == crtest.SignificantRegression || rt.ReportStatus == crtest.ExtremeRegression { + if rt.Regression != nil { + regressionID = rt.Regression.ID + originalStatus = rt.ReportStatus + found = true + } + } + if found { + break + } + } + if found { + break + } + } + if found { + break + } + } + require.True(t, found, "should find a regressed test with a regression ID") + t.Logf("found regression ID %d with status %d", regressionID, originalStatus) + + // Create a triage for this regression + triage := models.Triage{ + URL: "https://issues.redhat.com/browse/OCPBUGS-99999", + Type: models.TriageTypeProduct, + Regressions: []models.TestRegression{ + {ID: regressionID}, + }, + } + var triageResponse models.Triage + err = util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) + require.NoError(t, err, "creating triage should succeed") + t.Logf("created triage ID %d", triageResponse.ID) + + // Clean up triage when done + defer func() { + _ = util.SippyDelete(fmt.Sprintf("/api/component_readiness/triages/%d", triageResponse.ID)) + }() + + // Re-fetch the report — PostAnalysis runs outside the cache, so triaged status should be reflected + var updatedReport componentreport.ComponentReport + err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &updatedReport) + require.NoError(t, err) + + // Find the same regression and verify its status changed to triaged + foundTriaged := false + for _, row := range updatedReport.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.Regression == nil || rt.Regression.ID != regressionID { + continue + } + expectedStatus := crtest.SignificantTriagedRegression + if originalStatus == crtest.ExtremeRegression { + expectedStatus = crtest.ExtremeTriagedRegression + } + assert.Equal(t, expectedStatus, rt.ReportStatus, + "regression should have triaged status after triage creation") + assert.NotEmpty(t, rt.Explanations, "triaged test should have explanations") + foundTriaged = true + } + } + } + assert.True(t, foundTriaged, "should find the triaged regression in the updated report") +} From 23af877142fbe7dcb77c759b31e4fdc4da6cf338 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 21:29:02 -0400 Subject: [PATCH 16/25] Restore GCS credentials for datasync e2e test in CI The coverage instrumentation changes inadvertently removed the GCS credential setup, causing the datasync test to skip in CI. Restore GCS_CRED, GCS_SA_JSON_PATH, and SIPPY_E2E_REPO_ROOT exports so the test runner can access them. Co-Authored-By: Claude Opus 4.6 --- e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index afc28f5471..505e55307f 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -16,12 +16,21 @@ trap cleanup EXIT # When running locally, the user has to define SIPPY_IMAGE. echo "The sippy CI image: ${SIPPY_IMAGE}" +# The GCS_CRED allows us to pull artifacts from GCS when importing prow jobs. +# Redefine GCS_CRED to use your own. +GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" +echo "The GCS cred is: ${GCS_CRED}" + # If you're using Openshift, we use oc, if you're using plain Kubernetes, # we use kubectl. # KUBECTL_CMD="${KUBECTL_CMD:=oc}" echo "The kubectl command is: ${KUBECTL_CMD}" +# Make GCS credentials available to the test runner for the datasync test +export GCS_SA_JSON_PATH="${GCS_CRED}" +export SIPPY_E2E_REPO_ROOT="/go/src/sippy" + # Launch the sippy api server pod with coverage instrumentation. cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 From aae77ac486c8a0585872b92f687ddbadc7674c56 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 7 Apr 2026 22:05:01 -0400 Subject: [PATCH 17/25] Add coverage instrumentation to seed job and fix datasync test for CI - Seed job now uses sippy-cover with GOCOVERDIR and coverage PVC - Datasync test runs sippy load as a k8s Job using the sippy image, with GCS credentials and coverage PVC mounted - Restore gcs-cred secret creation in CI test script Co-Authored-By: Claude Opus 4.6 --- .../sippy-e2e-sippy-e2e-setup-commands.sh | 12 +- .../sippy-e2e-sippy-e2e-test-commands.sh | 10 +- test/e2e/datasync/datasync_test.go | 120 ++++++++++++------ 3 files changed, 102 insertions(+), 40 deletions(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh index ae3cc0488d..c43f8b06c3 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh @@ -238,9 +238,19 @@ spec: terminationMessagePolicy: File command: ["/bin/sh", "-c"] args: - - /bin/sippy seed-data --init-database --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres + - /bin/sippy-cover seed-data --init-database --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres + env: + - name: GOCOVERDIR + value: /tmp/coverage + volumeMounts: + - mountPath: /tmp/coverage + name: coverage imagePullSecrets: - name: regcred + volumes: + - name: coverage + persistentVolumeClaim: + claimName: sippy-coverage dnsPolicy: ClusterFirst restartPolicy: Never schedulerName: default-scheduler diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index 505e55307f..ae2c255aa3 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -27,9 +27,13 @@ echo "The GCS cred is: ${GCS_CRED}" KUBECTL_CMD="${KUBECTL_CMD:=oc}" echo "The kubectl command is: ${KUBECTL_CMD}" -# Make GCS credentials available to the test runner for the datasync test -export GCS_SA_JSON_PATH="${GCS_CRED}" -export SIPPY_E2E_REPO_ROOT="/go/src/sippy" +# Get the gcs credentials out to the cluster-pool cluster. +# These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations). +# See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa +${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred="${GCS_CRED}" -n sippy-e2e + +# The datasync test runs sippy load as a k8s Job, so it needs these to create the pod. +export SIPPY_E2E_SIPPY_IMAGE="${SIPPY_IMAGE}" # Launch the sippy api server pod with coverage instrumentation. cat << END | ${KUBECTL_CMD} apply -f - diff --git a/test/e2e/datasync/datasync_test.go b/test/e2e/datasync/datasync_test.go index 5f57853a37..f3ff3facef 100644 --- a/test/e2e/datasync/datasync_test.go +++ b/test/e2e/datasync/datasync_test.go @@ -2,8 +2,10 @@ package datasync import ( "context" + "fmt" "os" "os/exec" + "strings" "testing" "time" @@ -12,8 +14,9 @@ import ( ) func TestDataSync(t *testing.T) { - if os.Getenv("GCS_SA_JSON_PATH") == "" { - t.Skip("GCS_SA_JSON_PATH not set, skipping data sync test") + sippyImage := os.Getenv("SIPPY_E2E_SIPPY_IMAGE") + if sippyImage == "" { + t.Skip("SIPPY_E2E_SIPPY_IMAGE not set, skipping data sync test") } dbc := util.CreateE2EPostgresConnection(t) @@ -22,46 +25,91 @@ func TestDataSync(t *testing.T) { require.NoError(t, dbc.DB.Table("prow_job_runs").Count(&countBefore).Error) t.Logf("prow_job_runs before sync: %d", countBefore) - repoRoot := os.Getenv("SIPPY_E2E_REPO_ROOT") - require.NotEmpty(t, repoRoot, "SIPPY_E2E_REPO_ROOT must be set") - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - // Prefer the coverage-instrumented binary if available - sippyBin := "" - for _, candidate := range []string{ - repoRoot + "/sippy-cover", - "/bin/sippy-cover", - repoRoot + "/sippy", - "/bin/sippy", - } { - if _, err := os.Stat(candidate); err == nil { - sippyBin = candidate - break - } + kubectl := os.Getenv("KUBECTL_CMD") + if kubectl == "" { + kubectl = "oc" } - require.NotEmpty(t, sippyBin, "could not find sippy binary") - cmd := exec.CommandContext(ctx, sippyBin, "load", // #nosec G204 - "--loader", "prow", - "--release", util.Release, - "--prow-load-since", "2h", - "--config", "config/e2e-openshift.yaml", - "--google-service-account-credential-file", os.Getenv("GCS_SA_JSON_PATH"), - "--database-dsn", os.Getenv("SIPPY_E2E_DSN"), - "--skip-matview-refresh", - "--log-level", "debug", - ) - cmd.Dir = repoRoot - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if coverDir := os.Getenv("GOCOVERDIR"); coverDir != "" { - cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir) - } + // Create a Job that runs sippy load as a pod on the cluster, with GCS + // credentials and coverage instrumentation. + jobManifest := fmt.Sprintf(`apiVersion: batch/v1 +kind: Job +metadata: + name: sippy-datasync-job + namespace: sippy-e2e +spec: + template: + spec: + containers: + - name: sippy + image: %s + resources: + limits: + memory: 8Gi + command: ["/bin/sippy-cover"] + args: + - load + - --loader + - prow + - --release + - "%s" + - --prow-load-since + - 2h + - --config + - config/e2e-openshift.yaml + - --google-service-account-credential-file + - /tmp/secrets/gcs-cred + - --database-dsn + - postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres + - --skip-matview-refresh + - --log-level + - debug + env: + - name: GOCOVERDIR + value: /tmp/coverage + volumeMounts: + - mountPath: /tmp/secrets + name: gcs-cred + readOnly: true + - mountPath: /tmp/coverage + name: coverage + imagePullSecrets: + - name: regcred + volumes: + - name: gcs-cred + secret: + secretName: gcs-cred + - name: coverage + persistentVolumeClaim: + claimName: sippy-coverage + restartPolicy: Never + backoffLimit: 0`, sippyImage, util.Release) + + // Apply the job manifest + applyCmd := exec.CommandContext(ctx, kubectl, "apply", "-f", "-") // #nosec G204 + applyCmd.Stdin = strings.NewReader(jobManifest) + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + require.NoError(t, applyCmd.Run(), "failed to create datasync job") + + // Wait for the job to complete + waitCmd := exec.CommandContext(ctx, kubectl, "-n", "sippy-e2e", "wait", // #nosec G204 + "--for=condition=complete", "job/sippy-datasync-job", "--timeout=600s") + waitCmd.Stdout = os.Stdout + waitCmd.Stderr = os.Stderr + waitErr := waitCmd.Run() + + // Collect logs regardless of success/failure + logCmd := exec.CommandContext(ctx, kubectl, "-n", "sippy-e2e", "logs", // #nosec G204 + "--selector=job-name=sippy-datasync-job") + logCmd.Stdout = os.Stdout + logCmd.Stderr = os.Stderr + _ = logCmd.Run() - err := cmd.Run() - require.NoError(t, err, "sippy load command should complete without error") + require.NoError(t, waitErr, "sippy load job should complete successfully") var countAfter int64 require.NoError(t, dbc.DB.Table("prow_job_runs").Count(&countAfter).Error) From 685b6ee304cacab4493aa34e3969fbaa9436d95a Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Wed, 15 Apr 2026 15:00:56 -0400 Subject: [PATCH 18/25] Add missing JobLabels and TestFailures fields to crstatus.TestJobRunRows Co-Authored-By: Claude Opus 4.6 --- pkg/apis/api/componentreport/crstatus/types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/apis/api/componentreport/crstatus/types.go b/pkg/apis/api/componentreport/crstatus/types.go index 71d6ff3010..8dd0e44cff 100644 --- a/pkg/apis/api/componentreport/crstatus/types.go +++ b/pkg/apis/api/componentreport/crstatus/types.go @@ -63,6 +63,8 @@ type TestJobRunRows struct { crtest.Count JiraComponent string `bigquery:"jira_component"` JiraComponentID *big.Rat `bigquery:"jira_component_id"` + JobLabels []string `bigquery:"-" json:"job_labels,omitempty"` + TestFailures int `bigquery:"-" json:"test_failures"` } // JobVariant defines a variant and the possible values. From 4af328d55350a6ef91b293726858b92bb4d3a9a0 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 17 Apr 2026 11:22:26 -0400 Subject: [PATCH 19/25] Split CR e2e tests into postgres and bigquery suites Move existing component readiness e2e tests under postgres/ and restore the upstream BigQuery TestComponentReadinessViews test in bigquery/, gated on GCS_SA_JSON_PATH. The e2e script now runs two phases: postgres tests first, then restarts the server with --data-provider bigquery for the BQ suite when credentials are available. Co-Authored-By: Claude Opus 4.6 --- scripts/e2e.sh | 98 ++++++++++++++----- .../bigquery/componentreadiness_test.go | 31 ++++++ .../{ => postgres}/componentreadiness_test.go | 2 +- .../regressiontracker_test.go | 0 .../{ => postgres}/report/report_test.go | 0 .../{ => postgres}/triage/triageapi_test.go | 0 6 files changed, 103 insertions(+), 28 deletions(-) create mode 100644 test/e2e/componentreadiness/bigquery/componentreadiness_test.go rename test/e2e/componentreadiness/{ => postgres}/componentreadiness_test.go (99%) rename test/e2e/componentreadiness/{ => postgres}/regressiontracker/regressiontracker_test.go (100%) rename test/e2e/componentreadiness/{ => postgres}/report/report_test.go (100%) rename test/e2e/componentreadiness/{ => postgres}/triage/triageapi_test.go (100%) diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 19ea32a8d3..7bb23473f1 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -12,12 +12,16 @@ REDIS_CONTAINER="sippy-e2e-test-redis" REDIS_PORT="23479" if [ -z "$GCS_SA_JSON_PATH" ]; then - echo "WARNING: GCS_SA_JSON_PATH not set, data sync test will be skipped" 1>&2 + echo "WARNING: GCS_SA_JSON_PATH not set, data sync and BigQuery tests will be skipped" 1>&2 fi +E2E_EXIT_CODE=0 clean_up () { ARG=$? + if [ $ARG -ne 0 ]; then + E2E_EXIT_CODE=$ARG + fi echo "Stopping sippy API child process: $CHILD_PID" kill $CHILD_PID 2>/dev/null && wait $CHILD_PID 2>/dev/null # Generate coverage report from the server's coverage data @@ -26,11 +30,13 @@ clean_up () { go tool covdata percent -i="$COVDIR" go tool covdata textfmt -i="$COVDIR" -o=e2e-coverage.out # Merge test binary coverage (from -coverprofile) into server binary coverage - if [ -f e2e-test-coverage.out ]; then - echo "Merging test binary coverage into server coverage..." - tail -n +2 e2e-test-coverage.out >> e2e-coverage.out - rm -f e2e-test-coverage.out - fi + for f in e2e-test-coverage.out e2e-bq-test-coverage.out; do + if [ -f "$f" ]; then + echo "Merging $f into server coverage..." + tail -n +2 "$f" >> e2e-coverage.out + rm -f "$f" + fi + done echo "Coverage data written to e2e-coverage.out" echo "View HTML report: go tool cover -html=e2e-coverage.out -o=e2e-coverage.html" fi @@ -40,7 +46,23 @@ clean_up () { echo "Tearing down container $REDIS_CONTAINER" $DOCKER stop -i $REDIS_CONTAINER $DOCKER rm -i $REDIS_CONTAINER - exit $ARG + exit $E2E_EXIT_CODE +} + +wait_for_sippy() { + echo "Waiting for sippy API to start on port $SIPPY_API_PORT..." + TIMEOUT=600 + ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + if curl -s "http://localhost:$SIPPY_API_PORT/api/health" > /dev/null 2>&1; then + echo "Sippy API is ready after ${ELAPSED}s" + return 0 + fi + sleep 2 + ELAPSED=$((ELAPSED + 2)) + done + echo "Timeout waiting for sippy API to start after ${TIMEOUT}s" + return 1 } trap clean_up EXIT @@ -95,23 +117,7 @@ GOCOVERDIR="$COVDIR" ./sippy serve \ --data-provider postgres > e2e.log 2>&1 & CHILD_PID=$! -# Give it time to start up, and fill the redis cache -echo "Waiting for sippy API to start on port $SIPPY_API_PORT, see e2e.log for output..." -TIMEOUT=600 -ELAPSED=0 -while [ $ELAPSED -lt $TIMEOUT ]; do - if curl -s "http://localhost:$SIPPY_API_PORT/api/health" > /dev/null 2>&1; then - echo "Sippy API is ready after ${ELAPSED}s" - break - fi - sleep 2 - ELAPSED=$((ELAPSED + 2)) -done - -if [ $ELAPSED -ge $TIMEOUT ]; then - echo "Timeout waiting for sippy API to start after ${TIMEOUT}s" - exit 1 -fi +wait_for_sippy || exit 1 # Prime the component readiness cache so triage tests can find cached reports echo "Priming component readiness cache..." @@ -122,7 +128,45 @@ for VIEW in $(echo "$VIEWS" | jq -r '.[].name'); do done echo "Cache priming complete" -# Run our tests that request against the API, args ensure serially and fresh test code compile: -gotestsum ./test/e2e/... -count 1 -p 1 -coverprofile=e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... +# Phase 1: Run postgres-backed tests +echo "=== Phase 1: Running postgres-backed e2e tests ===" +gotestsum \ + ./test/e2e/componentreadiness/postgres/... \ + ./test/e2e/componentreadiness/bugs/... \ + ./test/e2e/datasync/... \ + ./test/e2e/ \ + -count 1 -p 1 -coverprofile=e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... +POSTGRES_EXIT=$? +if [ $POSTGRES_EXIT -ne 0 ]; then + E2E_EXIT_CODE=$POSTGRES_EXIT +fi + +# Phase 2: Run BigQuery-backed tests (if credentials are available) +if [ -n "$GCS_SA_JSON_PATH" ]; then + echo "=== Phase 2: Running BigQuery-backed e2e tests ===" + echo "Stopping postgres-backed server..." + kill $CHILD_PID 2>/dev/null && wait $CHILD_PID 2>/dev/null -# WARNING: do not place more commands here without addressing return code from go test not being overridden by the cleanup func + GOCOVERDIR="$COVDIR" ./sippy serve \ + --listen ":$SIPPY_API_PORT" \ + --listen-metrics ":12112" \ + --database-dsn="$SIPPY_E2E_DSN" \ + --log-level debug \ + --views config/e2e-views.yaml \ + --google-service-account-credential-file $GCS_SA_JSON_PATH \ + --redis-url="$REDIS_URL" \ + --data-provider bigquery > e2e-bq.log 2>&1 & + CHILD_PID=$! + + wait_for_sippy || exit 1 + + gotestsum \ + ./test/e2e/componentreadiness/bigquery/... \ + -count 1 -p 1 -coverprofile=e2e-bq-test-coverage.out -coverpkg=./pkg/...,./cmd/... + BQ_EXIT=$? + if [ $BQ_EXIT -ne 0 ]; then + E2E_EXIT_CODE=$BQ_EXIT + fi +else + echo "=== Phase 2: Skipping BigQuery tests (GCS_SA_JSON_PATH not set) ===" +fi diff --git a/test/e2e/componentreadiness/bigquery/componentreadiness_test.go b/test/e2e/componentreadiness/bigquery/componentreadiness_test.go new file mode 100644 index 0000000000..6bad2a5fec --- /dev/null +++ b/test/e2e/componentreadiness/bigquery/componentreadiness_test.go @@ -0,0 +1,31 @@ +package bigquery + +import ( + "fmt" + "os" + "testing" + + "github.com/openshift/sippy/pkg/apis/api/componentreport" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" + "github.com/openshift/sippy/test/e2e/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestComponentReadinessViews(t *testing.T) { + if os.Getenv("GCS_SA_JSON_PATH") == "" { + t.Skip("GCS_SA_JSON_PATH not set, skipping BigQuery tests") + } + + var views []crview.View + err := util.SippyGet("/api/component_readiness/views", &views) + require.NoError(t, err, "error making http request") + t.Logf("found %d views", len(views)) + require.Greater(t, len(views), 0, "no views returned, check server cli params") + + // Make a basic request for the first view and ensure we get some data back + var report componentreport.ComponentReport + err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", views[0].Name), &report) + require.NoError(t, err, "error making http request") + assert.Greater(t, len(report.Rows), 25, "component report does not have rows we would expect") +} diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/postgres/componentreadiness_test.go similarity index 99% rename from test/e2e/componentreadiness/componentreadiness_test.go rename to test/e2e/componentreadiness/postgres/componentreadiness_test.go index cfafb1487f..5ba45278e7 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/postgres/componentreadiness_test.go @@ -1,4 +1,4 @@ -package componentreadiness +package postgres import ( "fmt" diff --git a/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go b/test/e2e/componentreadiness/postgres/regressiontracker/regressiontracker_test.go similarity index 100% rename from test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go rename to test/e2e/componentreadiness/postgres/regressiontracker/regressiontracker_test.go diff --git a/test/e2e/componentreadiness/report/report_test.go b/test/e2e/componentreadiness/postgres/report/report_test.go similarity index 100% rename from test/e2e/componentreadiness/report/report_test.go rename to test/e2e/componentreadiness/postgres/report/report_test.go diff --git a/test/e2e/componentreadiness/triage/triageapi_test.go b/test/e2e/componentreadiness/postgres/triage/triageapi_test.go similarity index 100% rename from test/e2e/componentreadiness/triage/triageapi_test.go rename to test/e2e/componentreadiness/postgres/triage/triageapi_test.go From 6edb2b8d5e29b7b568b718796c5005254ca91d42 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 17 Apr 2026 11:23:15 -0400 Subject: [PATCH 20/25] Update CI e2e script to run both postgres and bigquery test suites Refactor the CI test script into two phases: first run postgres-backed tests, then restart the sippy-server pod with --data-provider bigquery and run the BigQuery test suite. Extract server lifecycle into launch_sippy_server/stop_sippy_server helpers. Coverage data from both server runs is collected and merged. Co-Authored-By: Claude Opus 4.6 --- .../sippy-e2e-sippy-e2e-test-commands.sh | 126 ++++++++++++------ 1 file changed, 86 insertions(+), 40 deletions(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index 5998c92fbc..5fe69f13e4 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -35,8 +35,11 @@ ${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred="${GCS_CRED}" # The datasync test runs sippy load as a k8s Job, so it needs these to create the pod. export SIPPY_E2E_SIPPY_IMAGE="${SIPPY_IMAGE}" -# Launch the sippy api server pod with coverage instrumentation. -cat << END | ${KUBECTL_CMD} apply -f - +launch_sippy_server() { + local DATA_PROVIDER=$1 + local EXTRA_ARGS="${2:-}" + + cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 kind: Pod metadata: @@ -77,7 +80,7 @@ spec: - --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres - --redis-url=redis://redis.sippy-e2e.svc.cluster.local:6379 - --data-provider - - postgres + - ${DATA_PROVIDER} - --log-level - debug - --enable-write-endpoints @@ -85,6 +88,8 @@ spec: - ocp - --views - ./config/e2e-views.yaml + - --google-service-account-credential-file + - /tmp/secrets/gcs-cred env: - name: GCS_SA_JSON_PATH value: /tmp/secrets/gcs-cred @@ -112,27 +117,39 @@ spec: terminationGracePeriodSeconds: 30 END -# The basic readiness probe will give us at least 10 seconds before declaring the pod as ready. -echo "Waiting for sippy api server pod to be Ready ..." -set +e -${KUBECTL_CMD} -n sippy-e2e wait --for=condition=Ready pod/sippy-server --timeout=600s -server_retVal=$? -set -e - -${KUBECTL_CMD} -n sippy-e2e get pod -o wide -${KUBECTL_CMD} -n sippy-e2e logs sippy-server > ${ARTIFACT_DIR}/sippy-server.log 2>&1 - -if [ ${server_retVal} -ne 0 ]; then - echo - echo "=== SIPPY SERVER FAILURE DIAGNOSTICS ===" - ${KUBECTL_CMD} -n sippy-e2e describe pod/sippy-server - echo "=== Namespace events ===" - ${KUBECTL_CMD} -n sippy-e2e get events --sort-by='.lastTimestamp' - echo "=== END SIPPY SERVER FAILURE DIAGNOSTICS ===" - echo - echo "ERROR: sippy-server pod never became Ready (timed out after 600s)" - exit 1 -fi + echo "Waiting for sippy api server pod (${DATA_PROVIDER}) to be Ready ..." + set +e + ${KUBECTL_CMD} -n sippy-e2e wait --for=condition=Ready pod/sippy-server --timeout=600s + local retVal=$? + set -e + + ${KUBECTL_CMD} -n sippy-e2e get pod -o wide + ${KUBECTL_CMD} -n sippy-e2e logs sippy-server > ${ARTIFACT_DIR}/sippy-server-${DATA_PROVIDER}.log 2>&1 + + if [ ${retVal} -ne 0 ]; then + echo + echo "=== SIPPY SERVER FAILURE DIAGNOSTICS (${DATA_PROVIDER}) ===" + ${KUBECTL_CMD} -n sippy-e2e describe pod/sippy-server + echo "=== Namespace events ===" + ${KUBECTL_CMD} -n sippy-e2e get events --sort-by='.lastTimestamp' + echo "=== END SIPPY SERVER FAILURE DIAGNOSTICS ===" + echo + echo "ERROR: sippy-server pod (${DATA_PROVIDER}) never became Ready (timed out after 600s)" + return 1 + fi + return 0 +} + +stop_sippy_server() { + local DATA_PROVIDER=$1 + echo "Stopping sippy-server (${DATA_PROVIDER}) to flush coverage data..." + ${KUBECTL_CMD} -n sippy-e2e logs sippy-server > ${ARTIFACT_DIR}/sippy-server-${DATA_PROVIDER}.log 2>&1 || true + ${KUBECTL_CMD} -n sippy-e2e delete pod sippy-server --wait=true --timeout=60s || true + ${KUBECTL_CMD} -n sippy-e2e delete svc sippy-server || true +} + +# Phase 1: Launch postgres-backed server +launch_sippy_server postgres || exit 1 echo "Setup services and port forwarding for the sippy api server ..." @@ -146,6 +163,7 @@ export SIPPY_API_PORT # Setup port forward for random port to get to the sippy-server pod ${KUBECTL_CMD} -n sippy-e2e expose pod sippy-server ${KUBECTL_CMD} -n sippy-e2e port-forward pod/sippy-server ${SIPPY_API_PORT}:8080 & +PF_PID_SERVER=$! # Random port for postgres as well, between 18500 and 19000 # Direct postgres access is used for some e2e test to seed data and cleanup things we don't expose on the api, @@ -167,17 +185,44 @@ ${KUBECTL_CMD} -n sippy-e2e port-forward pod/redis1 ${SIPPY_REDIS_PORT}:6379 & ${KUBECTL_CMD} -n sippy-e2e get svc,ep -# only 1 in parallel, some tests will clash if run at the same time -gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e.xml -- ./test/e2e/... -v -p 1 -coverprofile=${ARTIFACT_DIR}/e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... -TEST_EXIT=$? +E2E_EXIT_CODE=0 + +echo "=== Phase 1: Running postgres-backed e2e tests ===" +gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e_postgres.xml -- \ + ./test/e2e/componentreadiness/postgres/... \ + ./test/e2e/componentreadiness/bugs/... \ + ./test/e2e/datasync/... \ + ./test/e2e/ \ + -v -p 1 -coverprofile=${ARTIFACT_DIR}/e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... +POSTGRES_EXIT=$? +if [ ${POSTGRES_EXIT} -ne 0 ]; then + E2E_EXIT_CODE=${POSTGRES_EXIT} +fi + +# Stop the postgres server, kill the port-forward, and restart with bigquery +kill ${PF_PID_SERVER} 2>/dev/null || true +stop_sippy_server postgres + +# Phase 2: Launch bigquery-backed server +echo "=== Phase 2: Running BigQuery-backed e2e tests ===" +launch_sippy_server bigquery || exit 1 + +${KUBECTL_CMD} -n sippy-e2e expose pod sippy-server +${KUBECTL_CMD} -n sippy-e2e port-forward pod/sippy-server ${SIPPY_API_PORT}:8080 & +PF_PID_SERVER=$! + +gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e_bigquery.xml -- \ + ./test/e2e/componentreadiness/bigquery/... \ + -v -p 1 -coverprofile=${ARTIFACT_DIR}/e2e-bq-test-coverage.out -coverpkg=./pkg/...,./cmd/... +BQ_EXIT=$? +if [ ${BQ_EXIT} -ne 0 ]; then + E2E_EXIT_CODE=${BQ_EXIT} +fi -# Collect coverage data. Coverage counters are flushed when the server exits. -# Pod deletion sends SIGTERM during graceful termination (terminationGracePeriodSeconds: 30), -# so we just delete the pod directly — no need for a separate exec kill. -echo "Stopping sippy-server to flush coverage data..." -${KUBECTL_CMD} -n sippy-e2e delete pod sippy-server --wait=true --timeout=60s || true +kill ${PF_PID_SERVER} 2>/dev/null || true +stop_sippy_server bigquery -# Launch a minimal helper pod to access the coverage PVC. +# Collect coverage data from both server runs cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 kind: Pod @@ -211,12 +256,13 @@ if find "${COVDIR}" -name 'covcounters.*' -print -quit 2>/dev/null | grep -q .; echo "Generating coverage report..." go tool covdata percent -i="${COVDIR}" go tool covdata textfmt -i="${COVDIR}" -o="${ARTIFACT_DIR}/e2e-coverage.out" - # Merge test binary coverage (from -coverprofile) into server binary coverage - if [ -f "${ARTIFACT_DIR}/e2e-test-coverage.out" ]; then - echo "Merging test binary coverage into server coverage..." - tail -n +2 "${ARTIFACT_DIR}/e2e-test-coverage.out" >> "${ARTIFACT_DIR}/e2e-coverage.out" - rm -f "${ARTIFACT_DIR}/e2e-test-coverage.out" - fi + for f in ${ARTIFACT_DIR}/e2e-test-coverage.out ${ARTIFACT_DIR}/e2e-bq-test-coverage.out; do + if [ -f "$f" ]; then + echo "Merging $f into server coverage..." + tail -n +2 "$f" >> "${ARTIFACT_DIR}/e2e-coverage.out" + rm -f "$f" + fi + done go tool cover -html="${ARTIFACT_DIR}/e2e-coverage.out" -o="${ARTIFACT_DIR}/e2e-coverage.html" echo "Coverage report written to ${ARTIFACT_DIR}/e2e-coverage.html" else @@ -226,4 +272,4 @@ rm -rf "${COVDIR}" ${KUBECTL_CMD} -n sippy-e2e delete secret regcred || true -exit ${TEST_EXIT} +exit ${E2E_EXIT_CODE} From 7c5f0f8c20b962791ce8559bb92d374504f67a75 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 17 Apr 2026 13:48:24 -0400 Subject: [PATCH 21/25] Fix CI e2e cache priming, coverage dir detection, and test comment Add cache-warming loop before running tests in the CI script so triage tests find cached reports. Fix coverage directory detection after kubectl cp which nests files in a subdirectory. Correct comment about pass rate to match seed data (70%, not 80%). Co-Authored-By: Claude Opus 4.6 --- .../sippy-e2e-sippy-e2e-test-commands.sh | 19 +++++++++++++++---- .../postgres/report/report_test.go | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index 5fe69f13e4..190c0e5e60 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -187,6 +187,15 @@ ${KUBECTL_CMD} -n sippy-e2e get svc,ep E2E_EXIT_CODE=0 +# Prime the component readiness cache so triage tests can find cached reports +echo "Priming component readiness cache..." +VIEWS=$(curl -sf "http://localhost:${SIPPY_API_PORT}/api/component_readiness/views") || { echo "Failed to fetch views"; exit 1; } +for VIEW in $(echo "$VIEWS" | jq -r '.[].name'); do + echo " Priming cache for view: $VIEW" + curl -sf "http://localhost:${SIPPY_API_PORT}/api/component_readiness?view=$VIEW" > /dev/null || { echo "Failed to prime cache for view: $VIEW"; exit 1; } +done +echo "Cache priming complete" + echo "=== Phase 1: Running postgres-backed e2e tests ===" gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e_postgres.xml -- \ ./test/e2e/componentreadiness/postgres/... \ @@ -252,10 +261,12 @@ ${KUBECTL_CMD} -n sippy-e2e wait --for=condition=Ready pod/coverage-helper --tim COVDIR=$(mktemp -d) ${KUBECTL_CMD} -n sippy-e2e cp coverage-helper:/tmp/coverage "${COVDIR}" -c helper || true -if find "${COVDIR}" -name 'covcounters.*' -print -quit 2>/dev/null | grep -q .; then - echo "Generating coverage report..." - go tool covdata percent -i="${COVDIR}" - go tool covdata textfmt -i="${COVDIR}" -o="${ARTIFACT_DIR}/e2e-coverage.out" +COVERAGE_ROOT=$(find "${COVDIR}" -name 'covcounters.*' -print -quit 2>/dev/null | xargs -r dirname) +COVERAGE_ROOT="${COVERAGE_ROOT:-${COVDIR}}" +if find "${COVERAGE_ROOT}" -name 'covcounters.*' -print -quit 2>/dev/null | grep -q .; then + echo "Generating coverage report from ${COVERAGE_ROOT}..." + go tool covdata percent -i="${COVERAGE_ROOT}" + go tool covdata textfmt -i="${COVERAGE_ROOT}" -o="${ARTIFACT_DIR}/e2e-coverage.out" for f in ${ARTIFACT_DIR}/e2e-test-coverage.out ${ARTIFACT_DIR}/e2e-bq-test-coverage.out; do if [ -f "$f" ]; then echo "Merging $f into server coverage..." diff --git a/test/e2e/componentreadiness/postgres/report/report_test.go b/test/e2e/componentreadiness/postgres/report/report_test.go index bff375a34a..1eaf8e43bd 100644 --- a/test/e2e/componentreadiness/postgres/report/report_test.go +++ b/test/e2e/componentreadiness/postgres/report/report_test.go @@ -269,7 +269,7 @@ func TestNewTestPassRateRegression(t *testing.T) { report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") require.Empty(t, errs) - // The new flaky test (80% pass rate) should be flagged as a regression + // The new flaky test (70% pass rate) should be flagged as a regression // via buildPassRateTestStats since it's below the 90% PassRateRequiredNewTests threshold. found := false for _, row := range report.Rows { From 0c0c0326671ccf92e5de8a9c24f621867f7211ed Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 17 Apr 2026 13:58:52 -0400 Subject: [PATCH 22/25] Replace destructive cleanupAllTriages with targeted cleanupTriages The old cleanupAllTriages used `DELETE FROM triage_regressions WHERE 1=1` which would destroy all triage data including seed data. The new cleanupTriages only deletes specific triages passed as arguments, matching the pattern used by cleanupRegressions in the regression tracker tests. Co-Authored-By: Claude Opus 4.6 --- .../postgres/triage/triageapi_test.go | 87 +++++++++---------- 1 file changed, 41 insertions(+), 46 deletions(-) diff --git a/test/e2e/componentreadiness/postgres/triage/triageapi_test.go b/test/e2e/componentreadiness/postgres/triage/triageapi_test.go index bf429da3d8..5c626214f6 100644 --- a/test/e2e/componentreadiness/postgres/triage/triageapi_test.go +++ b/test/e2e/componentreadiness/postgres/triage/triageapi_test.go @@ -42,12 +42,19 @@ var view = crview.View{ }, } -func cleanupAllTriages(dbc *db.DB) { - // Delete all triage and test regressions in the e2e postgres db. - dbc.DB.Exec("DELETE FROM triage_regressions WHERE 1=1") - res := dbc.DB.Where("1 = 1").Delete(&models.Triage{}) - if res.Error != nil { - log.Errorf("error deleting triage records: %v", res.Error) +// cleanupTriages deletes only the specified triages and their associated +// regression links. Tests should clean up only what they create to avoid +// destroying seed data. +func cleanupTriages(dbc *db.DB, triages ...*models.Triage) { + for _, tr := range triages { + if tr == nil || tr.ID == 0 { + continue + } + dbc.DB.Exec("DELETE FROM triage_regressions WHERE triage_id = ?", tr.ID) + res := dbc.DB.Delete(tr) + if res.Error != nil { + log.Errorf("error deleting triage %d: %v", tr.ID, res.Error) + } } } @@ -65,7 +72,6 @@ func Test_TriageAPI(t *testing.T) { defer dbc.DB.Delete(testRegression2) t.Run("create requires a valid triage type", func(t *testing.T) { - defer cleanupAllTriages(dbc) triage1 := models.Triage{ URL: jiraBug.URL, Regressions: []models.TestRegression{ @@ -85,7 +91,6 @@ func Test_TriageAPI(t *testing.T) { }) t.Run("create fails with non-existent regression ID", func(t *testing.T) { - defer cleanupAllTriages(dbc) triage := models.Triage{ URL: jiraBug.URL, Type: models.TriageTypeProduct, @@ -101,7 +106,6 @@ func Test_TriageAPI(t *testing.T) { }) t.Run("create generates audit_log record", func(t *testing.T) { - defer cleanupAllTriages(dbc) triage1 := models.Triage{ URL: jiraBug.URL, Regressions: []models.TestRegression{ @@ -115,6 +119,7 @@ func Test_TriageAPI(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage1, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) var auditLog models.AuditLog res := dbc.DB. @@ -135,8 +140,8 @@ func Test_TriageAPI(t *testing.T) { }) t.Run("get", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // ensure hateoas links are present require.NotEmpty(t, triageResponse.Links["self"]) @@ -150,8 +155,6 @@ func Test_TriageAPI(t *testing.T) { triageResponse.Links["audit_logs"]) }) t.Run("get with expanded regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Use real regressions from seed data instead of injecting fake data into cache. // We filter to seed data regressions (test IDs starting with "test-") because // other subtests in this function create synthetic regressions that won't appear @@ -172,6 +175,7 @@ func Test_TriageAPI(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) require.Equal(t, 2, len(triageResponse.Regressions)) // Validate that the expanded regressions are present @@ -199,8 +203,8 @@ func Test_TriageAPI(t *testing.T) { } }) t.Run("list", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var allTriages []models.Triage err := util.SippyGet("/api/component_readiness/triages", &allTriages) @@ -230,8 +234,8 @@ func Test_TriageAPI(t *testing.T) { } }) t.Run("update to add regression", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // Update with a new regression: var triageResponse2 models.Triage @@ -254,8 +258,6 @@ func Test_TriageAPI(t *testing.T) { triageResponse2.Links["audit_logs"]) }) t.Run("update to remove a regression", func(t *testing.T) { - defer cleanupAllTriages(dbc) - triage := models.Triage{ URL: jiraBug.URL, Type: models.TriageTypeProduct, @@ -268,6 +270,7 @@ func Test_TriageAPI(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) assert.Equal(t, 2, len(triageResponse.Regressions)) // Update to remove one regression - keep only testRegression1 @@ -281,8 +284,8 @@ func Test_TriageAPI(t *testing.T) { assert.NotEqual(t, triageResponse.UpdatedAt, triageResponse2.UpdatedAt) }) t.Run("update to remove all regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage triageResponse.Regressions = []models.TestRegression{} @@ -291,8 +294,8 @@ func Test_TriageAPI(t *testing.T) { assert.Equal(t, 0, len(triageResponse2.Regressions)) }) t.Run("update to resolve triage sets resolution reason to user", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // Resolve the triage by setting the Resolved timestamp resolvedTime := time.Now() @@ -308,8 +311,8 @@ func Test_TriageAPI(t *testing.T) { assert.Equal(t, models.User, updateResponse.ResolutionReason, "Resolution reason should be set to 'user'") }) t.Run("update fails if resource has no ID", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage triageResponse.ID = 0 @@ -317,8 +320,8 @@ func Test_TriageAPI(t *testing.T) { require.Error(t, err) }) t.Run("update fails if URL has no ID", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage // No ID specified in URL should not work for an update @@ -326,16 +329,16 @@ func Test_TriageAPI(t *testing.T) { require.Error(t, err) }) t.Run("update fails if URL ID and resource ID do not match", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage err := util.SippyPut("/api/component_readiness/triages/128736182736128736", &triageResponse, &triageResponse2) require.Error(t, err) }) t.Run("update generates audit_log record", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) originalTriage := deepCopyTriage(t, triageResponse) // Update with a new regression, and a changed description: @@ -368,8 +371,8 @@ func Test_TriageAPI(t *testing.T) { assertTriageDataMatches(t, originalTriage, oldTriageData, "OldData") }) t.Run("delete generates audit_log record", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) originalTriage := deepCopyTriage(t, triageResponse) // Delete the triage record @@ -395,8 +398,6 @@ func Test_TriageAPI(t *testing.T) { }) t.Run("audit endpoint returns full lifecycle operations", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a triage triage := models.Triage{ URL: "https://issues.redhat.com/browse/OCPBUGS-8888", @@ -410,6 +411,7 @@ func Test_TriageAPI(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) require.True(t, triageResponse.ID > 0) // Small delay to ensure different timestamps @@ -552,8 +554,8 @@ func Test_RegressionAPI(t *testing.T) { release := view.SampleRelease.Release.Name t.Run("list regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) - _ = createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // Test listing regressions by release (release is required) var allRegressions []models.TestRegression @@ -582,8 +584,6 @@ func Test_RegressionAPI(t *testing.T) { assert.Contains(t, testDetailsHREF, "testId=", "test_details link should contain testId parameter") }) t.Run("error when both view and release are specified", func(t *testing.T) { - defer cleanupAllTriages(dbc) - var regressions []models.TestRegression err := util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions?view=%s-main&release=%s", util.Release, util.Release), ®ressions) require.Error(t, err, "Expected error when both view and release are provided") @@ -620,8 +620,6 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { defer dbc.DB.Delete(noMatchRegression.Regression) t.Run("find potential matching triages", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create triages with the matching regressions triage1 := models.Triage{ URL: jiraBug.URL, @@ -633,6 +631,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponse1 models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage1, &triageResponse1) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse1) triage2 := models.Triage{ URL: jiraBug.URL, @@ -644,6 +643,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponse2 models.Triage err = util.SippyPost("/api/component_readiness/triages", &triage2, &triageResponse2) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse2) // Create a triage with the no-match regression (should not appear in results) triageNoMatch := models.Triage{ @@ -656,6 +656,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponseNoMatch models.Triage err = util.SippyPost("/api/component_readiness/triages", &triageNoMatch, &triageResponseNoMatch) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponseNoMatch) // Query for potential matches for the target regression var potentialMatches []componentreadiness.PotentialMatchingTriage @@ -696,8 +697,6 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { }) t.Run("no potential matches found", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a triage with the no-match regression (different name and time) triage := models.Triage{ URL: jiraBug.URL, @@ -709,6 +708,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) // Query for potential matches for the target regression var potentialMatches []componentreadiness.PotentialMatchingTriage @@ -721,8 +721,6 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { }) t.Run("resolved triage confidence level capped at 5", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a regression with the exact same test name (edit distance 0, would normally give confidence 6) exactMatchRegression := createTestRegressionWithDetails(t, tracker, view, "exact-match", "component-e", "capability-v", "TestTargetFunction", &differentFailureTime, crtest.ExtremeRegression) defer dbc.DB.Delete(exactMatchRegression.Regression) @@ -738,6 +736,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponseExactMatch models.Triage err := util.SippyPost("/api/component_readiness/triages", &triageExactMatch, &triageResponseExactMatch) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponseExactMatch) // Resolve the triage resolvedTime := time.Now() @@ -822,8 +821,6 @@ func Test_TriageRawDB(t *testing.T) { defer dbc.DB.Delete(testRegression) t.Run("test Triage model in postgres", func(t *testing.T) { - defer cleanupAllTriages(dbc) - triage1 := models.Triage{ URL: "http://myjira", Regressions: []models.TestRegression{ @@ -832,6 +829,7 @@ func Test_TriageRawDB(t *testing.T) { } res := dbWithContext.Create(&triage1) require.NoError(t, res.Error) + defer cleanupTriages(dbc, &triage1) testRegression.Triages = append(testRegression.Triages, triage1) res = dbWithContext.Save(&testRegression) require.NoError(t, res.Error) @@ -869,6 +867,7 @@ func Test_TriageRawDB(t *testing.T) { } res = dbWithContext.Create(&triage2) require.NoError(t, res.Error) + defer cleanupTriages(dbc, &triage2) testRegression.Triages = append(testRegression.Triages, triage2) res = dbWithContext.Save(&testRegression) require.NoError(t, res.Error) @@ -891,8 +890,6 @@ func Test_TriageRawDB(t *testing.T) { }) t.Run("test Triage model Bug relationship", func(t *testing.T) { - defer cleanupAllTriages(dbc) - jiraBug := createBug(t, dbWithContext) defer dbWithContext.Delete(jiraBug) @@ -902,6 +899,7 @@ func Test_TriageRawDB(t *testing.T) { } res := dbWithContext.Create(&triage1) require.NoError(t, res.Error) + defer cleanupTriages(dbc, &triage1) // Lookup the Triage again to ensure we persisted what we expect: res = dbWithContext.First(&triage1, triage1.ID) @@ -1037,8 +1035,6 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { defer dbc.DB.Delete(testRegressions[9].Regression) t.Run("find potential matching regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Use real regressions from seed data that appear in the component report. // Synthetic regressions with fake test IDs won't appear in the report, so // GetTriagePotentialMatches would skip them entirely. @@ -1057,6 +1053,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) require.Equal(t, 1, len(triageResponse.Regressions)) // Query for potential matches — the other real regressions should appear @@ -1098,8 +1095,6 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { }) t.Run("empty potential matches when no regressions exist", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a triage with one linked regression triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-1234", @@ -1112,6 +1107,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) // Query for potential matches var potentialMatches []componentreadiness.PotentialMatchingRegression @@ -1133,8 +1129,6 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { }) t.Run("empty potential matches when release pair does not match any view", func(t *testing.T) { - defer cleanupAllTriages(dbc) - triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-9999", Type: models.TriageTypeProduct, @@ -1146,6 +1140,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) var potentialMatches []componentreadiness.PotentialMatchingRegression endpoint := fmt.Sprintf("/api/component_readiness/triages/%d/matches?baseRelease=no-such-base&sampleRelease=no-such-sample", triageResponse.ID) @@ -1163,7 +1158,6 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { }) t.Run("verify status values in triage responses", func(t *testing.T) { - defer cleanupAllTriages(dbc) // Use real regressions that appear in the component report so we can verify // status transformation via the expand endpoint @@ -1182,6 +1176,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) require.Equal(t, 2, len(triageResponse.Regressions)) // Use the expand endpoint to get status values from the component report From 508cadb03417efc89164d981c69aaa613db07b93 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 17 Apr 2026 14:33:25 -0400 Subject: [PATCH 23/25] Add unit tests for 6 pure-logic packages with zero prior coverage Tests cover pass rate math, set operations, HTTP param sanitization, BigQuery label rules, triage key serialization, and request option helpers. 151 tests total, all table-driven. Co-Authored-By: Claude Opus 4.6 --- .../api/componentreport/crtest/count_test.go | 308 ++++++++ .../api/componentreport/reqopts/types_test.go | 83 ++ pkg/bigquery/bqlabel/labels_test.go | 84 ++ .../resolvedissues/types_test.go | 153 ++++ pkg/util/param/param_test.go | 440 +++++++++++ pkg/util/sets/string_test.go | 729 ++++++++++++++++++ 6 files changed, 1797 insertions(+) create mode 100644 pkg/apis/api/componentreport/crtest/count_test.go create mode 100644 pkg/apis/api/componentreport/reqopts/types_test.go create mode 100644 pkg/bigquery/bqlabel/labels_test.go create mode 100644 pkg/componentreadiness/resolvedissues/types_test.go create mode 100644 pkg/util/param/param_test.go create mode 100644 pkg/util/sets/string_test.go diff --git a/pkg/apis/api/componentreport/crtest/count_test.go b/pkg/apis/api/componentreport/crtest/count_test.go new file mode 100644 index 0000000000..def694331a --- /dev/null +++ b/pkg/apis/api/componentreport/crtest/count_test.go @@ -0,0 +1,308 @@ +package crtest + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCountFailures(t *testing.T) { + tests := []struct { + name string + count Count + expected int + }{ + { + name: "normal case", + count: Count{TotalCount: 10, SuccessCount: 7, FlakeCount: 1}, + expected: 2, + }, + { + name: "all success", + count: Count{TotalCount: 5, SuccessCount: 5, FlakeCount: 0}, + expected: 0, + }, + { + name: "all flakes", + count: Count{TotalCount: 5, SuccessCount: 0, FlakeCount: 5}, + expected: 0, + }, + { + name: "all failures", + count: Count{TotalCount: 5, SuccessCount: 0, FlakeCount: 0}, + expected: 5, + }, + { + name: "zero total", + count: Count{TotalCount: 0, SuccessCount: 0, FlakeCount: 0}, + expected: 0, + }, + { + name: "data inconsistency floors at zero", + count: Count{TotalCount: 5, SuccessCount: 3, FlakeCount: 3}, + expected: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.count.Failures()) + }) + } +} + +func TestCalculatePassRate(t *testing.T) { + tests := []struct { + name string + success int + failure int + flake int + treatFlakeAsFailure bool + expected float64 + }{ + { + name: "zero total without flake flag", + success: 0, + failure: 0, + flake: 0, + treatFlakeAsFailure: false, + expected: 0.0, + }, + { + name: "zero total with flake flag", + success: 0, + failure: 0, + flake: 0, + treatFlakeAsFailure: true, + expected: 0.0, + }, + { + name: "flakes as success", + success: 8, + failure: 0, + flake: 2, + treatFlakeAsFailure: false, + expected: 1.0, + }, + { + name: "flakes as failure", + success: 8, + failure: 0, + flake: 2, + treatFlakeAsFailure: true, + expected: 0.8, + }, + { + name: "mixed results flakes as success", + success: 6, + failure: 2, + flake: 2, + treatFlakeAsFailure: false, + expected: 0.8, + }, + { + name: "mixed results flakes as failure", + success: 6, + failure: 2, + flake: 2, + treatFlakeAsFailure: true, + expected: 0.6, + }, + { + name: "all failures", + success: 0, + failure: 10, + flake: 0, + treatFlakeAsFailure: false, + expected: 0.0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := CalculatePassRate(tt.success, tt.failure, tt.flake, tt.treatFlakeAsFailure) + assert.InDelta(t, tt.expected, result, 1e-9) + }) + } +} + +func TestCountAdd(t *testing.T) { + a := Count{TotalCount: 3, SuccessCount: 2, FlakeCount: 1} + b := Count{TotalCount: 7, SuccessCount: 4, FlakeCount: 2} + result := a.Add(b) + + assert.Equal(t, 10, result.TotalCount) + assert.Equal(t, 6, result.SuccessCount) + assert.Equal(t, 3, result.FlakeCount) + + // receiver should not be mutated (value receiver) + assert.Equal(t, 3, a.TotalCount) +} + +func TestCountToTestStats(t *testing.T) { + count := Count{TotalCount: 10, SuccessCount: 6, FlakeCount: 2} + + t.Run("flake as success", func(t *testing.T) { + stats := count.ToTestStats(false) + assert.Equal(t, 6, stats.SuccessCount) + assert.Equal(t, 2, stats.FailureCount) + assert.Equal(t, 2, stats.FlakeCount) + assert.InDelta(t, 0.8, stats.SuccessRate, 1e-9) // (6+2)/10 + }) + + t.Run("flake as failure", func(t *testing.T) { + stats := count.ToTestStats(true) + assert.Equal(t, 6, stats.SuccessCount) + assert.Equal(t, 2, stats.FailureCount) + assert.Equal(t, 2, stats.FlakeCount) + assert.InDelta(t, 0.6, stats.SuccessRate, 1e-9) // 6/10 + }) +} + +func TestStatsAdd(t *testing.T) { + a := NewTestStats(5, 3, 2, false) + b := NewTestStats(3, 1, 1, false) + + t.Run("recalculates rate on add", func(t *testing.T) { + result := a.Add(b, false) + assert.Equal(t, 8, result.SuccessCount) + assert.Equal(t, 4, result.FailureCount) + assert.Equal(t, 3, result.FlakeCount) + // (8+3)/15 = 11/15 + assert.InDelta(t, 11.0/15.0, result.SuccessRate, 1e-9) + }) + + t.Run("recalculates with flakes as failure", func(t *testing.T) { + result := a.Add(b, true) + assert.Equal(t, 8, result.SuccessCount) + assert.Equal(t, 4, result.FailureCount) + assert.Equal(t, 3, result.FlakeCount) + // 8/15 + assert.InDelta(t, 8.0/15.0, result.SuccessRate, 1e-9) + }) +} + +func TestStatsFailPassWithFlakes(t *testing.T) { + stats := NewTestStats(5, 3, 2, false) + + t.Run("flakes as failure", func(t *testing.T) { + fail, pass := stats.FailPassWithFlakes(true) + assert.Equal(t, 5, fail) // 3 failures + 2 flakes + assert.Equal(t, 5, pass) // 5 success only + }) + + t.Run("flakes as success", func(t *testing.T) { + fail, pass := stats.FailPassWithFlakes(false) + assert.Equal(t, 3, fail) // 3 failures only + assert.Equal(t, 7, pass) // 5 success + 2 flakes + }) +} + +func TestStatsAddTestCount(t *testing.T) { + stats := NewTestStats(5, 2, 1, false) + count := Count{TotalCount: 10, SuccessCount: 6, FlakeCount: 2} + // count.Failures() = 10 - 6 - 2 = 2 + + t.Run("flakes as success", func(t *testing.T) { + result := stats.AddTestCount(count, false) + assert.Equal(t, 11, result.SuccessCount) // 5 + 6 + assert.Equal(t, 4, result.FailureCount) // 2 + 2 + assert.Equal(t, 3, result.FlakeCount) // 1 + 2 + // (11+3)/18 + assert.InDelta(t, 14.0/18.0, result.SuccessRate, 1e-9) + }) + + t.Run("flakes as failure", func(t *testing.T) { + result := stats.AddTestCount(count, true) + assert.Equal(t, 11, result.SuccessCount) + assert.Equal(t, 4, result.FailureCount) + assert.Equal(t, 3, result.FlakeCount) + // 11/18 + assert.InDelta(t, 11.0/18.0, result.SuccessRate, 1e-9) + }) + + t.Run("with inconsistent count data", func(t *testing.T) { + bad := Count{TotalCount: 5, SuccessCount: 3, FlakeCount: 3} + // bad.Failures() should be 0, not negative + result := stats.AddTestCount(bad, false) + assert.Equal(t, 8, result.SuccessCount) // 5 + 3 + assert.Equal(t, 2, result.FailureCount) // 2 + 0 + assert.Equal(t, 4, result.FlakeCount) // 1 + 3 + }) +} + +func TestStatsPasses(t *testing.T) { + stats := NewTestStats(5, 3, 2, false) + + t.Run("without flakes as failure", func(t *testing.T) { + assert.Equal(t, 7, stats.Passes(false)) // 5 + 2 + }) + + t.Run("with flakes as failure", func(t *testing.T) { + assert.Equal(t, 5, stats.Passes(true)) // 5 only + }) +} + +func TestStatsTotal(t *testing.T) { + stats := NewTestStats(5, 3, 2, false) + assert.Equal(t, 10, stats.Total()) + + empty := NewTestStats(0, 0, 0, false) + assert.Equal(t, 0, empty.Total()) +} + +func TestStringForStatus(t *testing.T) { + tests := []struct { + name string + status Status + expected string + }{ + {"ExtremeRegression", ExtremeRegression, "Extreme"}, + {"SignificantRegression", SignificantRegression, "Significant"}, + {"ExtremeTriagedRegression", ExtremeTriagedRegression, "ExtremeTriaged"}, + {"SignificantTriagedRegression", SignificantTriagedRegression, "SignificantTriaged"}, + {"MissingSample", MissingSample, "MissingSample"}, + {"FixedRegression", FixedRegression, "Fixed"}, + {"FailedFixedRegression", FailedFixedRegression, "FailedFixed"}, + {"NotSignificant falls through to Unknown", NotSignificant, "Unknown"}, + {"MissingBasis falls through to Unknown", MissingBasis, "Unknown"}, + {"MissingBasisAndSample falls through to Unknown", MissingBasisAndSample, "Unknown"}, + {"SignificantImprovement falls through to Unknown", SignificantImprovement, "Unknown"}, + {"undefined status", Status(9999), "Unknown"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, StringForStatus(tt.status)) + }) + } +} + +func TestKeyWithVariantsKeyOrDie(t *testing.T) { + t.Run("stable serialization regardless of insertion order", func(t *testing.T) { + k1 := KeyWithVariants{ + TestID: "test-1", + Variants: map[string]string{"b": "2", "a": "1"}, + } + k2 := KeyWithVariants{ + TestID: "test-1", + Variants: map[string]string{"a": "1", "b": "2"}, + } + assert.Equal(t, k1.KeyOrDie(), k2.KeyOrDie()) + }) + + t.Run("different test IDs produce different keys", func(t *testing.T) { + k1 := KeyWithVariants{TestID: "test-1", Variants: map[string]string{"a": "1"}} + k2 := KeyWithVariants{TestID: "test-2", Variants: map[string]string{"a": "1"}} + assert.NotEqual(t, k1.KeyOrDie(), k2.KeyOrDie()) + }) + + t.Run("different variants produce different keys", func(t *testing.T) { + k1 := KeyWithVariants{TestID: "test-1", Variants: map[string]string{"a": "1"}} + k2 := KeyWithVariants{TestID: "test-1", Variants: map[string]string{"a": "2"}} + assert.NotEqual(t, k1.KeyOrDie(), k2.KeyOrDie()) + }) + + t.Run("nil variants", func(t *testing.T) { + k := KeyWithVariants{TestID: "test-1"} + result := k.KeyOrDie() + assert.Contains(t, result, "test-1") + }) +} diff --git a/pkg/apis/api/componentreport/reqopts/types_test.go b/pkg/apis/api/componentreport/reqopts/types_test.go new file mode 100644 index 0000000000..d8dccf17d4 --- /dev/null +++ b/pkg/apis/api/componentreport/reqopts/types_test.go @@ -0,0 +1,83 @@ +package reqopts + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAnyAreBaseOverrides(t *testing.T) { + tests := []struct { + name string + opts []TestIdentification + expected bool + }{ + { + name: "nil slice returns false", + opts: nil, + expected: false, + }, + { + name: "empty slice returns false", + opts: []TestIdentification{}, + expected: false, + }, + { + name: "single item with empty BaseOverrideRelease returns false", + opts: []TestIdentification{ + {TestID: "test-1"}, + }, + expected: false, + }, + { + name: "single item with BaseOverrideRelease set returns true", + opts: []TestIdentification{ + {TestID: "test-1", BaseOverrideRelease: "4.20"}, + }, + expected: true, + }, + { + name: "multiple items none have override returns false", + opts: []TestIdentification{ + {TestID: "test-1"}, + {TestID: "test-2"}, + {TestID: "test-3"}, + }, + expected: false, + }, + { + name: "multiple items only last has override returns true", + opts: []TestIdentification{ + {TestID: "test-1"}, + {TestID: "test-2"}, + {TestID: "test-3", BaseOverrideRelease: "4.20"}, + }, + expected: true, + }, + { + name: "multiple items first has override returns true", + opts: []TestIdentification{ + {TestID: "test-1", BaseOverrideRelease: "4.20"}, + {TestID: "test-2"}, + {TestID: "test-3"}, + }, + expected: true, + }, + { + name: "multiple items all have override returns true", + opts: []TestIdentification{ + {TestID: "test-1", BaseOverrideRelease: "4.19"}, + {TestID: "test-2", BaseOverrideRelease: "4.20"}, + {TestID: "test-3", BaseOverrideRelease: "4.21"}, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := AnyAreBaseOverrides(tt.opts) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/bigquery/bqlabel/labels_test.go b/pkg/bigquery/bqlabel/labels_test.go new file mode 100644 index 0000000000..4ab1e09940 --- /dev/null +++ b/pkg/bigquery/bqlabel/labels_test.go @@ -0,0 +1,84 @@ +package bqlabel + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeLabelValue(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "lowercase and replace spaces", + input: "Hello World", + expected: "hello_world", + }, + { + name: "already valid chars unchanged", + input: "test-value_123", + expected: "test-value_123", + }, + { + name: "empty string stays empty", + input: "", + expected: "", + }, + { + name: "truncate 64 chars to 63", + input: strings.Repeat("a", 64), + expected: strings.Repeat("a", 63), + }, + { + name: "exactly 63 chars unchanged", + input: strings.Repeat("b", 63), + expected: strings.Repeat("b", 63), + }, + { + name: "uppercase folded to lowercase", + input: "UPPER", + expected: "upper", + }, + { + name: "special chars replaced", + input: "foo@bar.com", + expected: "foo_bar_com", + }, + { + name: "non-latin chars replaced", + input: "日本語", + expected: "___", + }, + { + name: "slashes in URI paths replaced", + input: "/api/v1/test", + expected: "_api_v1_test", + }, + { + name: "dots in IP addresses replaced", + input: "192.168.1.1", + expected: "192_168_1_1", + }, + { + name: "spaces replaced with underscores", + input: "with spaces", + expected: "with_spaces", + }, + { + name: "mixed case with valid special chars", + input: "MiXeD-CaSe_123", + expected: "mixed-case_123", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := sanitizeLabelValue(tc.input) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/pkg/componentreadiness/resolvedissues/types_test.go b/pkg/componentreadiness/resolvedissues/types_test.go new file mode 100644 index 0000000000..f64180aad2 --- /dev/null +++ b/pkg/componentreadiness/resolvedissues/types_test.go @@ -0,0 +1,153 @@ +package resolvedissues + +import ( + "encoding/json" + "testing" + + "github.com/openshift/sippy/pkg/variantregistry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTriagedIssueKey_MarshalUnmarshalRoundTrip(t *testing.T) { + tests := []struct { + name string + key TriagedIssueKey + }{ + { + name: "basic key", + key: TriagedIssueKey{TestID: "openshift-tests.sig-auth", Variants: "aws-amd64-ovn-ha"}, + }, + { + name: "special chars in TestID", + key: TriagedIssueKey{TestID: "test:with-special_chars.v2", Variants: "aws-amd64"}, + }, + { + name: "empty Variants", + key: TriagedIssueKey{TestID: "some-test-id", Variants: ""}, + }, + { + name: "empty TestID", + key: TriagedIssueKey{TestID: "", Variants: "aws-amd64-ovn"}, + }, + { + name: "both fields empty", + key: TriagedIssueKey{TestID: "", Variants: ""}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + marshaled, err := tc.key.MarshalText() + require.NoError(t, err) + + var got TriagedIssueKey + err = got.UnmarshalText(marshaled) + require.NoError(t, err) + + assert.Equal(t, tc.key, got) + }) + } +} + +func TestTriagedIssueKey_DifferentKeysProdduceDifferentSerializations(t *testing.T) { + t.Run("differ only in Variants", func(t *testing.T) { + key1 := TriagedIssueKey{TestID: "same-test", Variants: "aws-amd64"} + key2 := TriagedIssueKey{TestID: "same-test", Variants: "gcp-arm64"} + + m1, err := key1.MarshalText() + require.NoError(t, err) + m2, err := key2.MarshalText() + require.NoError(t, err) + + assert.NotEqual(t, string(m1), string(m2)) + }) + + t.Run("differ only in TestID", func(t *testing.T) { + key1 := TriagedIssueKey{TestID: "test-alpha", Variants: "same-variant"} + key2 := TriagedIssueKey{TestID: "test-beta", Variants: "same-variant"} + + m1, err := key1.MarshalText() + require.NoError(t, err) + m2, err := key2.MarshalText() + require.NoError(t, err) + + assert.NotEqual(t, string(m1), string(m2)) + }) +} + +func TestTriagedIssueKey_JSONMapKey(t *testing.T) { + key := TriagedIssueKey{TestID: "test:with-special_chars.v2", Variants: "aws-amd64-ovn"} + + original := map[TriagedIssueKey]int{key: 42} + data, err := json.Marshal(original) + require.NoError(t, err) + + var restored map[TriagedIssueKey]int + err = json.Unmarshal(data, &restored) + require.NoError(t, err) + + val, ok := restored[key] + require.True(t, ok, "key should be present in restored map") + assert.Equal(t, 42, val) +} + +func TestTriagedIssueKey_JSONMapKeyMultipleEntries(t *testing.T) { + key1 := TriagedIssueKey{TestID: "test-a", Variants: "aws"} + key2 := TriagedIssueKey{TestID: "test-b", Variants: "gcp"} + + original := map[TriagedIssueKey]int{key1: 1, key2: 2} + data, err := json.Marshal(original) + require.NoError(t, err) + + var restored map[TriagedIssueKey]int + err = json.Unmarshal(data, &restored) + require.NoError(t, err) + + assert.Len(t, restored, 2) + assert.Equal(t, 1, restored[key1]) + assert.Equal(t, 2, restored[key2]) +} + +func TestBuildTriageMatchVariants(t *testing.T) { + t.Run("empty slice returns nil", func(t *testing.T) { + result := buildTriageMatchVariants([]string{}) + assert.Nil(t, result) + }) + + t.Run("single item", func(t *testing.T) { + result := buildTriageMatchVariants([]string{"Platform"}) + require.NotNil(t, result) + assert.Equal(t, 1, result.Len()) + assert.True(t, result.Has("Platform")) + }) + + t.Run("multiple items", func(t *testing.T) { + input := []string{"Platform", "Architecture", "Network"} + result := buildTriageMatchVariants(input) + require.NotNil(t, result) + assert.Equal(t, 3, result.Len()) + for _, v := range input { + assert.True(t, result.Has(v), "set should contain %s", v) + } + }) +} + +func TestTriageMatchVariants_ContainsExpectedVariants(t *testing.T) { + expected := []string{ + variantregistry.VariantPlatform, + variantregistry.VariantArch, + variantregistry.VariantNetwork, + variantregistry.VariantTopology, + variantregistry.VariantFeatureSet, + variantregistry.VariantUpgrade, + variantregistry.VariantSuite, + variantregistry.VariantInstaller, + } + + assert.Equal(t, 8, TriageMatchVariants.Len(), "TriageMatchVariants should contain exactly 8 variant names") + + for _, v := range expected { + assert.True(t, TriageMatchVariants.Has(v), "TriageMatchVariants should contain %s", v) + } +} diff --git a/pkg/util/param/param_test.go b/pkg/util/param/param_test.go new file mode 100644 index 0000000000..d028dc3c4d --- /dev/null +++ b/pkg/util/param/param_test.go @@ -0,0 +1,440 @@ +package param + +import ( + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCleanse(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "SQL injection characters stripped", + input: "'; DROP TABLE--", + expected: " DROP TABLE--", + }, + { + name: "spaces are allowed", + input: "hello world", + expected: "hello world", + }, + { + name: "all allowed characters pass through", + input: "test:value-1_2", + expected: "test:value-1_2", + }, + { + name: "unicode stripped entirely", + input: "名前", + expected: "", + }, + { + name: "empty stays empty", + input: "", + expected: "", + }, + { + name: "mixed allowed and disallowed", + input: "abc!@#123", + expected: "abc123", + }, + { + name: "only alphanumeric", + input: "SimpleTest123", + expected: "SimpleTest123", + }, + { + name: "tabs and newlines stripped", + input: "line1\tline2\nline3", + expected: "line1line2line3", + }, + { + name: "parentheses and brackets stripped", + input: "func(arg)[0]", + expected: "funcarg0", + }, + { + name: "percent and equals stripped", + input: "key=value%20encoded", + expected: "keyvalue20encoded", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := Cleanse(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestSafeRead(t *testing.T) { + tests := []struct { + name string + paramName string + value string + expected string + }{ + // release param: ^[\w.-]+$ + { + name: "release valid dotted version", + paramName: "release", + value: "4.16", + expected: "4.16", + }, + { + name: "release rejects SQL injection", + paramName: "release", + value: "4.16; DROP", + expected: "", + }, + { + name: "release valid word", + paramName: "release", + value: "Presubmit", + expected: "Presubmit", + }, + + // baseRelease param: ^\d+\.\d+$ + { + name: "baseRelease valid", + paramName: "baseRelease", + value: "4.16", + expected: "4.16", + }, + { + name: "baseRelease rejects triple dot", + paramName: "baseRelease", + value: "4.16.1", + expected: "", + }, + { + name: "baseRelease rejects v prefix", + paramName: "baseRelease", + value: "v4.16", + expected: "", + }, + + // prow_job_run_id param: ^\d+$ + { + name: "prow_job_run_id valid digits", + paramName: "prow_job_run_id", + value: "12345", + expected: "12345", + }, + { + name: "prow_job_run_id rejects alphanumeric", + paramName: "prow_job_run_id", + value: "123abc", + expected: "", + }, + + // test_id param: ^[\w:. -]+$ + { + name: "test_id valid with colon and hex", + paramName: "test_id", + value: "openshift-tests-upgrade:af8a62c5", + expected: "openshift-tests-upgrade:af8a62c5", + }, + { + name: "test_id valid with space", + paramName: "test_id", + value: "cluster install:0cb1bb27", + expected: "cluster install:0cb1bb27", + }, + + // view param: ^[-.\w]+$ + { + name: "view valid with dash", + paramName: "view", + value: "4.16-main", + expected: "4.16-main", + }, + { + name: "view rejects space", + paramName: "view", + value: "view name", + expected: "", + }, + + // include_success param: ^(true|false)$ + { + name: "include_success true", + paramName: "include_success", + value: "true", + expected: "true", + }, + { + name: "include_success false", + paramName: "include_success", + value: "false", + expected: "false", + }, + { + name: "include_success rejects uppercase True", + paramName: "include_success", + value: "True", + expected: "", + }, + + // empty value always returns "" + { + name: "empty value returns empty string", + paramName: "release", + value: "", + expected: "", + }, + + // prow_job_run_ids param: ^\d+(,\d+)*$ + { + name: "prow_job_run_ids single value", + paramName: "prow_job_run_ids", + value: "12345", + expected: "12345", + }, + { + name: "prow_job_run_ids comma separated", + paramName: "prow_job_run_ids", + value: "123,456,789", + expected: "123,456,789", + }, + { + name: "prow_job_run_ids rejects trailing comma", + paramName: "prow_job_run_ids", + value: "123,", + expected: "", + }, + + // start_date param: ^\d{4}-\d{2}-\d{2}$ + { + name: "start_date valid format", + paramName: "start_date", + value: "2024-01-15", + expected: "2024-01-15", + }, + { + name: "start_date rejects wrong format", + paramName: "start_date", + value: "01-15-2024", + expected: "", + }, + + // test param: ^.+$ (anything non-empty) + { + name: "test allows anything", + paramName: "test", + value: "[sig-cli] oc explain should work", + expected: "[sig-cli] oc explain should work", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := "http://example.com/api" + if tt.value != "" { + u += "?" + tt.paramName + "=" + url.QueryEscape(tt.value) + } + req := httptest.NewRequest("GET", u, nil) + result := SafeRead(req, tt.paramName) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestReadUint(t *testing.T) { + tests := []struct { + name string + paramValue string + limit int + expectedVal int + expectErr bool + }{ + { + name: "missing param returns zero and nil", + paramValue: "", + limit: 100, + expectedVal: 0, + expectErr: false, + }, + { + name: "valid value within limit", + paramValue: "42", + limit: 100, + expectedVal: 42, + expectErr: false, + }, + { + name: "value exceeds limit", + paramValue: "42", + limit: 10, + expectedVal: 0, + expectErr: true, + }, + { + name: "limit zero means no limit", + paramValue: "42", + limit: 0, + expectedVal: 42, + expectErr: false, + }, + { + name: "negative value rejected by regex", + paramValue: "-5", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + { + name: "non-numeric rejected", + paramValue: "abc", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + { + name: "value exactly at limit", + paramValue: "100", + limit: 100, + expectedVal: 100, + expectErr: false, + }, + { + name: "value one over limit", + paramValue: "101", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + { + name: "zero value", + paramValue: "0", + limit: 100, + expectedVal: 0, + expectErr: false, + }, + { + name: "decimal rejected by regex", + paramValue: "3.14", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := "http://example.com/api" + if tt.paramValue != "" { + u += "?count=" + url.QueryEscape(tt.paramValue) + } + req := httptest.NewRequest("GET", u, nil) + val, err := ReadUint(req, "count", tt.limit) + if tt.expectErr { + require.Error(t, err) + assert.Equal(t, 0, val) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedVal, val) + } + }) + } +} + +func TestReadBool(t *testing.T) { + tests := []struct { + name string + paramValue string + defaultValue bool + expectedVal bool + expectErr bool + }{ + { + name: "empty with default true", + paramValue: "", + defaultValue: true, + expectedVal: true, + expectErr: false, + }, + { + name: "empty with default false", + paramValue: "", + defaultValue: false, + expectedVal: false, + expectErr: false, + }, + { + name: "true value", + paramValue: "true", + defaultValue: false, + expectedVal: true, + expectErr: false, + }, + { + name: "false value", + paramValue: "false", + defaultValue: true, + expectedVal: false, + expectErr: false, + }, + { + name: "uppercase TRUE rejected", + paramValue: "TRUE", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "numeric 1 rejected", + paramValue: "1", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "yes rejected", + paramValue: "yes", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "mixed case True rejected", + paramValue: "True", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "numeric 0 rejected", + paramValue: "0", + defaultValue: true, + expectedVal: false, + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := "http://example.com/api" + if tt.paramValue != "" { + u += "?flag=" + url.QueryEscape(tt.paramValue) + } + req := httptest.NewRequest("GET", u, nil) + val, err := ReadBool(req, "flag", tt.defaultValue) + if tt.expectErr { + require.Error(t, err) + assert.Equal(t, false, val) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedVal, val) + } + }) + } +} diff --git a/pkg/util/sets/string_test.go b/pkg/util/sets/string_test.go new file mode 100644 index 0000000000..4522485314 --- /dev/null +++ b/pkg/util/sets/string_test.go @@ -0,0 +1,729 @@ +package sets + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewString(t *testing.T) { + tests := []struct { + name string + items []string + wantLen int + wantList []string + }{ + { + name: "empty", + items: nil, + wantLen: 0, + wantList: []string{}, + }, + { + name: "single item", + items: []string{"a"}, + wantLen: 1, + wantList: []string{"a"}, + }, + { + name: "multiple items", + items: []string{"c", "a", "b"}, + wantLen: 3, + wantList: []string{"a", "b", "c"}, + }, + { + name: "duplicates are collapsed", + items: []string{"a", "b", "a", "b", "a"}, + wantLen: 2, + wantList: []string{"a", "b"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.items...) + assert.Equal(t, tt.wantLen, s.Len()) + assert.Equal(t, tt.wantList, s.List()) + }) + } +} + +func TestInsert(t *testing.T) { + tests := []struct { + name string + initial []string + insert []string + wantLen int + wantItems []string + }{ + { + name: "insert into empty set", + initial: nil, + insert: []string{"x"}, + wantLen: 1, + wantItems: []string{"x"}, + }, + { + name: "insert new item", + initial: []string{"a"}, + insert: []string{"b"}, + wantLen: 2, + wantItems: []string{"a", "b"}, + }, + { + name: "insert duplicate does not increase length", + initial: []string{"a", "b"}, + insert: []string{"a"}, + wantLen: 2, + wantItems: []string{"a", "b"}, + }, + { + name: "insert multiple with overlap", + initial: []string{"a"}, + insert: []string{"a", "b", "c"}, + wantLen: 3, + wantItems: []string{"a", "b", "c"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + s.Insert(tt.insert...) + assert.Equal(t, tt.wantLen, s.Len()) + assert.Equal(t, tt.wantItems, s.List()) + }) + } +} + +func TestDelete(t *testing.T) { + tests := []struct { + name string + initial []string + delete []string + wantLen int + wantItems []string + }{ + { + name: "delete existing item", + initial: []string{"a", "b", "c"}, + delete: []string{"b"}, + wantLen: 2, + wantItems: []string{"a", "c"}, + }, + { + name: "delete non-existent item does not panic", + initial: []string{"a"}, + delete: []string{"z"}, + wantLen: 1, + wantItems: []string{"a"}, + }, + { + name: "delete from empty set does not panic", + initial: nil, + delete: []string{"x"}, + wantLen: 0, + wantItems: []string{}, + }, + { + name: "delete all items", + initial: []string{"a", "b"}, + delete: []string{"a", "b"}, + wantLen: 0, + wantItems: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + s.Delete(tt.delete...) + assert.Equal(t, tt.wantLen, s.Len()) + assert.Equal(t, tt.wantItems, s.List()) + }) + } +} + +func TestHas(t *testing.T) { + tests := []struct { + name string + initial []string + check string + want bool + }{ + { + name: "item present", + initial: []string{"a", "b"}, + check: "a", + want: true, + }, + { + name: "item absent", + initial: []string{"a", "b"}, + check: "c", + want: false, + }, + { + name: "empty set", + initial: nil, + check: "a", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + assert.Equal(t, tt.want, s.Has(tt.check)) + }) + } +} + +func TestHasAll(t *testing.T) { + tests := []struct { + name string + initial []string + check []string + want bool + }{ + { + name: "all present", + initial: []string{"a", "b", "c"}, + check: []string{"a", "c"}, + want: true, + }, + { + name: "some missing", + initial: []string{"a", "b"}, + check: []string{"a", "z"}, + want: false, + }, + { + name: "empty args returns true (vacuous truth)", + initial: []string{"a"}, + check: nil, + want: true, + }, + { + name: "empty set with empty args returns true", + initial: nil, + check: nil, + want: true, + }, + { + name: "empty set with non-empty args returns false", + initial: nil, + check: []string{"a"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + assert.Equal(t, tt.want, s.HasAll(tt.check...)) + }) + } +} + +func TestHasAny(t *testing.T) { + tests := []struct { + name string + initial []string + check []string + want bool + }{ + { + name: "one match", + initial: []string{"a", "b", "c"}, + check: []string{"z", "b"}, + want: true, + }, + { + name: "no match", + initial: []string{"a", "b"}, + check: []string{"x", "y"}, + want: false, + }, + { + name: "empty args returns false", + initial: []string{"a"}, + check: nil, + want: false, + }, + { + name: "empty set returns false", + initial: nil, + check: []string{"a"}, + want: false, + }, + { + name: "empty set with empty args returns false", + initial: nil, + check: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + assert.Equal(t, tt.want, s.HasAny(tt.check...)) + }) + } +} + +func TestDifference(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want []string + }{ + { + name: "standard asymmetric: s1 minus s2", + s1: []string{"a", "b", "c"}, + s2: []string{"b", "c", "d"}, + want: []string{"a"}, + }, + { + name: "reverse direction: s2 minus s1", + s1: []string{"b", "c", "d"}, + s2: []string{"a", "b", "c"}, + want: []string{"d"}, + }, + { + name: "disjoint sets", + s1: []string{"a", "b"}, + s2: []string{"c", "d"}, + want: []string{"a", "b"}, + }, + { + name: "identical sets", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: []string{}, + }, + { + name: "empty s1", + s1: nil, + s2: []string{"a"}, + want: []string{}, + }, + { + name: "empty s2", + s1: []string{"a", "b"}, + s2: nil, + want: []string{"a", "b"}, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + result := s1.Difference(s2) + assert.Equal(t, tt.want, result.List()) + }) + } +} + +func TestDifferenceIsAsymmetric(t *testing.T) { + s1 := NewString("a", "b", "c") + s2 := NewString("b", "c", "d", "e") + + d1 := s1.Difference(s2) + d2 := s2.Difference(s1) + + assert.Equal(t, []string{"a"}, d1.List()) + assert.Equal(t, []string{"d", "e"}, d2.List()) + assert.False(t, d1.Equal(d2), "Difference must be asymmetric") +} + +func TestUnion(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want []string + }{ + { + name: "disjoint", + s1: []string{"a", "b"}, + s2: []string{"c", "d"}, + want: []string{"a", "b", "c", "d"}, + }, + { + name: "overlapping", + s1: []string{"a", "b", "c"}, + s2: []string{"b", "c", "d"}, + want: []string{"a", "b", "c", "d"}, + }, + { + name: "identical", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: []string{"a", "b"}, + }, + { + name: "empty s1", + s1: nil, + s2: []string{"a"}, + want: []string{"a"}, + }, + { + name: "empty s2", + s1: []string{"a"}, + s2: nil, + want: []string{"a"}, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + result := s1.Union(s2) + assert.Equal(t, tt.want, result.List()) + }) + } +} + +func TestUnionIsCommutative(t *testing.T) { + s1 := NewString("a", "b") + s2 := NewString("c", "d") + assert.True(t, s1.Union(s2).Equal(s2.Union(s1))) +} + +func TestIntersection(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want []string + }{ + { + name: "standard overlap", + s1: []string{"a", "b", "c"}, + s2: []string{"b", "c", "d"}, + want: []string{"b", "c"}, + }, + { + name: "disjoint", + s1: []string{"a", "b"}, + s2: []string{"c", "d"}, + want: []string{}, + }, + { + name: "identical", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: []string{"a", "b"}, + }, + { + name: "empty s1", + s1: nil, + s2: []string{"a"}, + want: []string{}, + }, + { + name: "empty s2", + s1: []string{"a"}, + s2: nil, + want: []string{}, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + result := s1.Intersection(s2) + assert.Equal(t, tt.want, result.List()) + }) + } +} + +func TestIntersectionOptimizationPath(t *testing.T) { + // The implementation walks the smaller set. Verify both code paths + // produce the same result regardless of which set is smaller. + small := NewString("b", "c") // len=2 + large := NewString("a", "b", "c", "d") // len=4 + + t.Run("s1 smaller than s2", func(t *testing.T) { + result := small.Intersection(large) + assert.Equal(t, []string{"b", "c"}, result.List()) + }) + + t.Run("s1 larger than s2", func(t *testing.T) { + result := large.Intersection(small) + assert.Equal(t, []string{"b", "c"}, result.List()) + }) + + t.Run("commutative", func(t *testing.T) { + assert.True(t, small.Intersection(large).Equal(large.Intersection(small))) + }) +} + +func TestIsSuperset(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want bool + }{ + { + name: "proper superset", + s1: []string{"a", "b", "c"}, + s2: []string{"a", "b"}, + want: true, + }, + { + name: "equal sets are supersets of each other", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: true, + }, + { + name: "not a superset", + s1: []string{"a", "b"}, + s2: []string{"a", "c"}, + want: false, + }, + { + name: "empty set is superset of empty set", + s1: nil, + s2: nil, + want: true, + }, + { + name: "any set is superset of empty set", + s1: []string{"a"}, + s2: nil, + want: true, + }, + { + name: "empty set is not superset of non-empty set", + s1: nil, + s2: []string{"a"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + assert.Equal(t, tt.want, s1.IsSuperset(s2)) + }) + } +} + +func TestEqual(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want bool + }{ + { + name: "identical contents", + s1: []string{"a", "b", "c"}, + s2: []string{"c", "b", "a"}, + want: true, + }, + { + name: "different sizes", + s1: []string{"a", "b"}, + s2: []string{"a"}, + want: false, + }, + { + name: "same length but different contents", + s1: []string{"a", "b"}, + s2: []string{"a", "c"}, + want: false, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + assert.Equal(t, tt.want, s1.Equal(s2)) + }) + } +} + +func TestList(t *testing.T) { + tests := []struct { + name string + items []string + want []string + }{ + { + name: "sorted output", + items: []string{"c", "a", "b"}, + want: []string{"a", "b", "c"}, + }, + { + name: "already sorted", + items: []string{"a", "b", "c"}, + want: []string{"a", "b", "c"}, + }, + { + name: "empty set returns empty slice", + items: nil, + want: []string{}, + }, + { + name: "single item", + items: []string{"x"}, + want: []string{"x"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.items...) + assert.Equal(t, tt.want, s.List()) + }) + } +} + +func TestListIsDeterministic(t *testing.T) { + s := NewString("z", "a", "m", "b", "y") + first := s.List() + for i := 0; i < 100; i++ { + assert.Equal(t, first, s.List(), "List() must return deterministic sorted order") + } +} + +func TestUnsortedList(t *testing.T) { + s := NewString("a", "b", "c") + result := s.UnsortedList() + assert.Len(t, result, 3) + assert.ElementsMatch(t, []string{"a", "b", "c"}, result) +} + +func TestPopAny(t *testing.T) { + t.Run("pop from non-empty set", func(t *testing.T) { + s := NewString("a", "b", "c") + val, ok := s.PopAny() + require.True(t, ok) + assert.NotEmpty(t, val) + assert.False(t, s.Has(val), "popped item must be removed from the set") + assert.Equal(t, 2, s.Len()) + }) + + t.Run("pop from empty set", func(t *testing.T) { + s := NewString() + val, ok := s.PopAny() + assert.False(t, ok) + assert.Equal(t, "", val) + }) + + t.Run("pop all items", func(t *testing.T) { + s := NewString("x", "y") + seen := map[string]bool{} + for s.Len() > 0 { + val, ok := s.PopAny() + require.True(t, ok) + seen[val] = true + } + assert.Equal(t, map[string]bool{"x": true, "y": true}, seen) + _, ok := s.PopAny() + assert.False(t, ok) + }) +} + +func TestLen(t *testing.T) { + tests := []struct { + name string + items []string + want int + }{ + {"empty", nil, 0}, + {"one", []string{"a"}, 1}, + {"three", []string{"a", "b", "c"}, 3}, + {"duplicates don't count", []string{"a", "a", "a"}, 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.items...) + assert.Equal(t, tt.want, s.Len()) + }) + } +} + +func TestStringKeySet(t *testing.T) { + t.Run("from map[string]int", func(t *testing.T) { + m := map[string]int{"foo": 1, "bar": 2, "baz": 3} + s := StringKeySet(m) + assert.Equal(t, 3, s.Len()) + assert.Equal(t, []string{"bar", "baz", "foo"}, s.List()) + }) + + t.Run("from map[string]string", func(t *testing.T) { + m := map[string]string{"a": "x", "b": "y"} + s := StringKeySet(m) + assert.Equal(t, []string{"a", "b"}, s.List()) + }) + + t.Run("from empty map", func(t *testing.T) { + m := map[string]bool{} + s := StringKeySet(m) + assert.Equal(t, 0, s.Len()) + assert.Equal(t, []string{}, s.List()) + }) + + t.Run("panics with non-map argument", func(t *testing.T) { + assert.Panics(t, func() { + StringKeySet("not a map") + }) + }) + + t.Run("panics with slice argument", func(t *testing.T) { + assert.Panics(t, func() { + StringKeySet([]string{"a", "b"}) + }) + }) +} + +func TestInsertReturnsSelf(t *testing.T) { + s := NewString() + returned := s.Insert("a", "b") + assert.True(t, s.Equal(returned), "Insert should return the same set for chaining") +} + +func TestDeleteReturnsSelf(t *testing.T) { + s := NewString("a", "b") + returned := s.Delete("a") + assert.True(t, s.Equal(returned), "Delete should return the same set for chaining") +} From 3f70be88b24ed19c52eddfd3b5ea99ab238688ab Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 17 Apr 2026 14:33:30 -0400 Subject: [PATCH 24/25] Add unit test phase to e2e scripts for combined coverage reporting Both local (scripts/e2e.sh) and CI (sippy-e2e-test-commands.sh) scripts now run `gotestsum ./pkg/...` after e2e tests and merge unit test coverage into the combined report. Co-Authored-By: Claude Opus 4.6 --- e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh | 11 ++++++++++- scripts/e2e.sh | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index 190c0e5e60..b064e41b52 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -231,6 +231,15 @@ fi kill ${PF_PID_SERVER} 2>/dev/null || true stop_sippy_server bigquery +echo "=== Running unit tests for coverage ===" +gotestsum --junitfile ${ARTIFACT_DIR}/junit_unit.xml -- \ + ./pkg/... \ + -v -coverprofile=${ARTIFACT_DIR}/unit-test-coverage.out -coverpkg=./pkg/...,./cmd/... +UNIT_EXIT=$? +if [ ${UNIT_EXIT} -ne 0 ]; then + E2E_EXIT_CODE=${UNIT_EXIT} +fi + # Collect coverage data from both server runs cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 @@ -267,7 +276,7 @@ if find "${COVERAGE_ROOT}" -name 'covcounters.*' -print -quit 2>/dev/null | grep echo "Generating coverage report from ${COVERAGE_ROOT}..." go tool covdata percent -i="${COVERAGE_ROOT}" go tool covdata textfmt -i="${COVERAGE_ROOT}" -o="${ARTIFACT_DIR}/e2e-coverage.out" - for f in ${ARTIFACT_DIR}/e2e-test-coverage.out ${ARTIFACT_DIR}/e2e-bq-test-coverage.out; do + for f in ${ARTIFACT_DIR}/e2e-test-coverage.out ${ARTIFACT_DIR}/e2e-bq-test-coverage.out ${ARTIFACT_DIR}/unit-test-coverage.out; do if [ -f "$f" ]; then echo "Merging $f into server coverage..." tail -n +2 "$f" >> "${ARTIFACT_DIR}/e2e-coverage.out" diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 7bb23473f1..fb355d6d9a 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -30,7 +30,7 @@ clean_up () { go tool covdata percent -i="$COVDIR" go tool covdata textfmt -i="$COVDIR" -o=e2e-coverage.out # Merge test binary coverage (from -coverprofile) into server binary coverage - for f in e2e-test-coverage.out e2e-bq-test-coverage.out; do + for f in e2e-test-coverage.out e2e-bq-test-coverage.out unit-test-coverage.out; do if [ -f "$f" ]; then echo "Merging $f into server coverage..." tail -n +2 "$f" >> e2e-coverage.out @@ -170,3 +170,12 @@ if [ -n "$GCS_SA_JSON_PATH" ]; then else echo "=== Phase 2: Skipping BigQuery tests (GCS_SA_JSON_PATH not set) ===" fi + +echo "=== Running unit tests for coverage ===" +gotestsum \ + ./pkg/... \ + -count 1 -coverprofile=unit-test-coverage.out -coverpkg=./pkg/...,./cmd/... +UNIT_EXIT=$? +if [ $UNIT_EXIT -ne 0 ]; then + E2E_EXIT_CODE=$UNIT_EXIT +fi From ff7f2da6f737ecc58f4e5a47e5fba610ddc83c87 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Fri, 17 Apr 2026 14:37:36 -0400 Subject: [PATCH 25/25] Move GCS secret creation back to CI setup step The GCS credential file is mounted by CI in the setup step pod, not the test step pod. Creating the k8s secret must happen during setup so it's available when the test step launches the sippy-server. Co-Authored-By: Claude Opus 4.6 --- e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh | 10 ++++++++++ e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh index 28026bae66..327352e49c 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh @@ -245,6 +245,16 @@ fi ${KUBECTL_CMD} -n sippy-e2e get svc,ep +# Get the gcs credentials out to the cluster-pool cluster. +# These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations). +# See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa +GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" +if [ -f "${GCS_CRED}" ]; then + ${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred="${GCS_CRED}" -n sippy-e2e +else + echo "WARNING: GCS credential file ${GCS_CRED} not found, BigQuery tests will fail" +fi + # Get the registry credentials for all build farm clusters out to the cluster-pool cluster. ${KUBECTL_CMD} -n sippy-e2e create secret generic regcred --from-file=.dockerconfigjson=${DOCKERCONFIGJSON} --type=kubernetes.io/dockerconfigjson diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index b064e41b52..1f8603fe4c 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -16,22 +16,12 @@ trap cleanup EXIT # When running locally, the user has to define SIPPY_IMAGE. echo "The sippy CI image: ${SIPPY_IMAGE}" -# The GCS_CRED allows us to pull artifacts from GCS when importing prow jobs. -# Redefine GCS_CRED to use your own. -GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" -echo "The GCS cred is: ${GCS_CRED}" - # If you're using Openshift, we use oc, if you're using plain Kubernetes, # we use kubectl. # KUBECTL_CMD="${KUBECTL_CMD:=oc}" echo "The kubectl command is: ${KUBECTL_CMD}" -# Get the gcs credentials out to the cluster-pool cluster. -# These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations). -# See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa -${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred="${GCS_CRED}" -n sippy-e2e - # The datasync test runs sippy load as a k8s Job, so it needs these to create the pod. export SIPPY_E2E_SIPPY_IMAGE="${SIPPY_IMAGE}"