diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index 6b5e3a6329..698a379b60 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -1,58 +1,49 @@ package main import ( + "context" "fmt" - "math/rand" + "os" + "sort" + "strings" "time" + "github.com/lib/pq" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" - + "gopkg.in/yaml.v3" + + componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" + pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" + apitype "github.com/openshift/sippy/pkg/apis/api" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" v1 "github.com/openshift/sippy/pkg/apis/sippyprocessing/v1" "github.com/openshift/sippy/pkg/db" "github.com/openshift/sippy/pkg/db/models" "github.com/openshift/sippy/pkg/db/models/jobrunscan" "github.com/openshift/sippy/pkg/flags" "github.com/openshift/sippy/pkg/sippyserver" + "github.com/openshift/sippy/pkg/util/sets" ) type SeedDataFlags struct { - DBFlags *flags.PostgresFlags - InitDatabase bool - Releases []string - JobsPerRelease int - TestNames []string - RunsPerJob int + DBFlags *flags.PostgresFlags + InitDatabase bool } func NewSeedDataFlags() *SeedDataFlags { return &SeedDataFlags{ - DBFlags: flags.NewPostgresDatabaseFlags(), - Releases: []string{"5.0", "4.22", "4.21"}, // Default releases - JobsPerRelease: 3, // Default jobs per release - TestNames: []string{ - "install should succeed: infrastructure", - "install should succeed: overall", - "install should succeed: configuration", - "install should succeed: cluster bootstrap", - "install should succeed: other", - "[sig-cluster-lifecycle] Cluster completes upgrade", - "[sig-sippy] upgrade should work", - "[sig-sippy] openshift-tests should work", - }, - RunsPerJob: 20, // Default runs per job + DBFlags: flags.NewPostgresDatabaseFlags(), } } func (f *SeedDataFlags) BindFlags(fs *pflag.FlagSet) { f.DBFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Initialize the DB schema before seeding data") - fs.StringSliceVar(&f.Releases, "release", f.Releases, "Releases to create ProwJobs for (can be specified multiple times)") - fs.IntVar(&f.JobsPerRelease, "jobs", f.JobsPerRelease, "Number of ProwJobs to create for each release") - fs.StringSliceVar(&f.TestNames, "test", f.TestNames, "Test names to create (can be specified multiple times)") - fs.IntVar(&f.RunsPerJob, "runs", f.RunsPerJob, "Number of ProwJobRuns to create for each ProwJob") } func NewSeedDataCommand() *cobra.Command { @@ -62,16 +53,19 @@ func NewSeedDataCommand() *cobra.Command { Use: "seed-data", Short: "Populate test data in the database", Long: `Populate test data in the database for development purposes. -This command creates sample ProwJob and Test records with realistic test data -that can be used for local development and testing. -Test results are randomized with 85% pass rate, 10% flake rate, and 5% failure rate. -All counts, releases, and test names are configurable via command-line flags. +Creates deterministic Component Readiness data covering all CR statuses +(NotSignificant, SignificantRegression, ExtremeRegression, MissingSample, +MissingBasis, BasisOnly, SignificantImprovement, BelowMinFailure) and +fallback scenarios. Use with 'sippy serve --data-provider postgres'. -The command can be re-run as needed to add more runs, or because your old job runs -rolled off the 1 week window. +Drop and recreate the database to re-seed (e.g. docker compose down -v). `, RunE: func(cmd *cobra.Command, args []string) error { + if strings.Contains(f.DBFlags.DSN, "amazonaws.com") { + return fmt.Errorf("refusing to seed synthetic data into a production database") + } + dbc, err := f.DBFlags.GetDBClient() if err != nil { return errors.WithMessage(err, "could not connect to database") @@ -87,247 +81,730 @@ rolled off the 1 week window. } log.Info("Starting to seed test data...") + return seedSyntheticData(dbc) + }, + } - // Create the test suite - if err := createTestSuite(dbc); err != nil { - return errors.WithMessage(err, "failed to create test suite") - } - log.Info("Created test suite 'ourtests'") + f.BindFlags(cmd.Flags()) - // Create ProwJobs for each release - for _, release := range f.Releases { - if err := createProwJobsForRelease(dbc, release, f.JobsPerRelease); err != nil { - return errors.WithMessagef(err, "failed to create ProwJobs for release %s", release) - } - log.Infof("Processed %d ProwJobs for release %s", f.JobsPerRelease, release) - } + return cmd +} - // Create Test models - if err := createTestModels(dbc, f.TestNames); err != nil { - return errors.WithMessage(err, "failed to create Test models") - } - log.Infof("Processed %d Test models", len(f.TestNames)) +// --- Synthetic data seeding --- - // Create labels and symptoms - if err := createLabelsAndSymptoms(dbc); err != nil { - return errors.WithMessage(err, "failed to create labels and symptoms") - } - log.Info("Created sample labels and symptoms") +// syntheticJobDef defines a job with its full 9-key variant map. +type syntheticJobDef struct { + nameTemplate string + variants map[string]string +} - // Create ProwJobRuns for each ProwJob - if err := createProwJobRuns(dbc, f.RunsPerJob); err != nil { - return errors.WithMessage(err, "failed to create ProwJobRuns") - } - log.Info("Created ProwJobRuns and test results for all ProwJobs") +// syntheticTestSpec defines a test with deterministic pass/fail counts per release per job. +type syntheticTestSpec struct { + testID string + testName string + component string + capabilities []string + // Each entry maps a job name template -> per-release counts. + // The job template determines which variants the test runs with. + jobCounts map[string]map[string]testCount // jobTemplate -> release -> counts +} - // Apply labels to job runs - if err := applyLabelsToJobRuns(dbc); err != nil { - return errors.WithMessage(err, "failed to apply labels to job runs") - } - log.Info("Applied labels to ~25% of job runs") +type testCount struct { + total int + success int + flake int +} + +var syntheticReleases = []string{"4.22", "4.21", "4.20", "4.19"} + +var syntheticJobs = []syntheticJobDef{ + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-upgrade-from-stable-4.21-e2e-aws-ovn-upgrade", + variants: map[string]string{ + "Platform": "aws", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "unknown", "Upgrade": "minor", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-amd64", + variants: map[string]string{ + "Platform": "aws", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "parallel", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-arm64", + variants: map[string]string{ + "Platform": "aws", "Architecture": "arm64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "parallel", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-techpreview-serial", + variants: map[string]string{ + "Platform": "aws", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "techpreview", + "Suite": "serial", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-gcp-ovn-amd64", + variants: map[string]string{ + "Platform": "gcp", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "parallel", "Upgrade": "none", "LayeredProduct": "none", + }, + }, + { + nameTemplate: "periodic-ci-openshift-release-master-ci-%s-e2e-gcp-ovn-upgrade-micro", + variants: map[string]string{ + "Platform": "gcp", "Architecture": "amd64", "Network": "ovn", + "Topology": "ha", "Installer": "ipi", "FeatureSet": "default", + "Suite": "unknown", "Upgrade": "micro", "LayeredProduct": "none", + }, + }, +} - totalProwJobs := len(f.Releases) * f.JobsPerRelease - totalRuns := totalProwJobs * f.RunsPerJob - totalTestResults := totalRuns * len(f.TestNames) +// Job template constants for referencing specific jobs in test specs. +const awsAmd64Parallel = "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-amd64" +const awsArm64Parallel = "periodic-ci-openshift-release-master-ci-%s-e2e-aws-ovn-arm64" +const gcpAmd64Parallel = "periodic-ci-openshift-release-master-ci-%s-e2e-gcp-ovn-amd64" + +// allJobTemplates returns name templates from syntheticJobs for use in test specs +// that should run on every job (e.g. install tests). +func allJobTemplates() []string { + templates := make([]string, len(syntheticJobs)) + for i, j := range syntheticJobs { + templates[i] = j.nameTemplate + } + return templates +} - log.Info("Refreshing materialized views...") - sippyserver.RefreshData(dbc, nil, false) +// allJobCounts builds a jobCounts map that assigns the given per-release counts +// to every synthetic job. Used for tests like install indicators that run everywhere. +func allJobCounts(releaseCounts map[string]testCount) map[string]map[string]testCount { + result := make(map[string]map[string]testCount, len(syntheticJobs)) + for _, tpl := range allJobTemplates() { + result[tpl] = releaseCounts + } + return result +} - log.Infof("Successfully seeded test data! Created %d ProwJobs, %d Tests, %d ProwJobRuns, and %d test results", - totalProwJobs, len(f.TestNames), totalRuns, totalTestResults) - return nil +var syntheticTests = []syntheticTestSpec{ + // --- NotSignificant: appears in 3 jobs across 2 platforms --- + { + testID: "test-not-significant", testName: "[sig-arch] Check build pods use all cpu cores", + component: "comp-NotSignificant", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 95, 0}, "4.22": {100, 93, 0}}, + awsArm64Parallel: {"4.21": {80, 76, 0}, "4.22": {80, 75, 0}}, + gcpAmd64Parallel: {"4.21": {100, 97, 0}, "4.22": {100, 95, 0}}, + }, + }, + + // --- SignificantRegression: regressed on aws/amd64, fine elsewhere --- + { + testID: "test-significant-regression", testName: "[sig-network] Services should serve endpoints on same port and different protocol", + component: "comp-SignificantRegression", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 170, 0}}, + awsArm64Parallel: {"4.21": {180, 171, 0}, "4.22": {180, 168, 0}}, + gcpAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 188, 0}}, + }, + }, + + // --- ExtremeRegression: extreme on aws/amd64, significant on others --- + { + testID: "test-extreme-regression", testName: "[sig-etcd] etcd leader changes are not excessive", + component: "comp-ExtremeRegression", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 140, 0}}, + awsArm64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 170, 0}}, + gcpAmd64Parallel: {"4.21": {200, 190, 0}, "4.22": {200, 170, 0}}, + }, + }, + + // --- MissingSample: test in base, 0 sample runs --- + { + testID: "test-missing-sample", testName: "[sig-storage] CSI volumes should be mountable", + component: "comp-MissingSample", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 95, 0}, "4.22": {0, 0, 0}}, + }, + }, + + // --- MissingBasis: test only in sample --- + { + testID: "test-missing-basis", testName: "[sig-node] New pod lifecycle test", + component: "comp-MissingBasis", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.22": {100, 95, 0}}, }, + }, + + // --- NewTestPassRateRegression: new test only in sample, below PassRateRequiredNewTests threshold --- + { + testID: "test-new-test-pass-rate-fail", testName: "[sig-node] New flaky pod readiness test", + component: "comp-NewTestPassRate", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.22": {100, 70, 0}}, + }, + }, + + // --- BasisOnly: test in base, absent from sample --- + { + testID: "test-basis-only", testName: "[sig-apps] Removed deployment test", + component: "comp-BasisOnly", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 95, 0}}, + }, + }, + + // --- SignificantImprovement: 80% -> 95% --- + { + testID: "test-significant-improvement", testName: "[sig-cli] oc adm should handle upgrades gracefully", + component: "comp-SignificantImprovement", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {200, 160, 0}, "4.22": {200, 190, 0}}, + }, + }, + + // --- BelowMinFailure: only 2 failures, below MinimumFailure=3 --- + { + testID: "test-below-min-failure", testName: "[sig-auth] RBAC should allow access with valid token", + component: "comp-BelowMinFailure", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: {"4.21": {100, 100, 0}, "4.22": {100, 98, 0}}, + }, + }, + + // --- Fallback: 4.21 worse, 4.20 better -> swaps to 4.20 --- + { + testID: "test-fallback-improves", testName: "[sig-instrumentation] Metrics should report accurate cpu usage", + component: "comp-FallbackImproves", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: { + "4.21": {200, 180, 0}, + "4.20": {200, 194, 0}, + "4.22": {200, 160, 0}, + }, + }, + }, + + // --- Double fallback: 4.21->4.20->4.19 --- + { + testID: "test-fallback-double", testName: "[sig-scheduling] Scheduler should spread pods evenly", + component: "comp-FallbackDouble", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: { + "4.21": {200, 180, 0}, + "4.20": {200, 186, 0}, + "4.19": {200, 194, 0}, + "4.22": {200, 160, 0}, + }, + }, + }, + + // --- Fallback insufficient runs: 4.20 has <60% of 4.21 count --- + { + testID: "test-fallback-insufficient-runs", testName: "[sig-network] DNS should resolve cluster services", + component: "comp-FallbackInsufficient", capabilities: []string{"cap1"}, + jobCounts: map[string]map[string]testCount{ + awsAmd64Parallel: { + "4.21": {1000, 940, 0}, + "4.20": {100, 99, 0}, + "4.22": {1000, 850, 0}, + }, + }, + }, + + // --- Install / health indicator tests: run on every job, every release --- + { + testID: "test-install-overall", testName: "install should succeed: overall", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 95, 0}, "4.21": {100, 96, 0}, "4.20": {100, 97, 0}, "4.19": {100, 97, 0}, + }), + }, + { + testID: "test-install-config", testName: "install should succeed: configuration", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 97, 0}, "4.21": {100, 98, 0}, "4.20": {100, 98, 0}, "4.19": {100, 98, 0}, + }), + }, + { + testID: "test-install-bootstrap", testName: "install should succeed: cluster bootstrap", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 96, 0}, "4.21": {100, 97, 0}, "4.20": {100, 97, 0}, "4.19": {100, 97, 0}, + }), + }, + { + testID: "test-install-other", testName: "install should succeed: other", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 98, 0}, "4.21": {100, 99, 0}, "4.20": {100, 99, 0}, "4.19": {100, 99, 0}, + }), + }, + { + testID: "test-install-infra", testName: "install should succeed: infrastructure", + component: "comp-Install", capabilities: []string{"install"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 96, 0}, "4.21": {100, 97, 0}, "4.20": {100, 97, 0}, "4.19": {100, 97, 0}, + }), + }, + { + testID: "test-upgrade", testName: "[sig-sippy] upgrade should work", + component: "comp-Install", capabilities: []string{"upgrade"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 94, 0}, "4.21": {100, 95, 0}, "4.20": {100, 96, 0}, "4.19": {100, 96, 0}, + }), + }, + { + testID: "test-openshift-tests", testName: "[sig-sippy] openshift-tests should work", + component: "comp-Install", capabilities: []string{"tests"}, + jobCounts: allJobCounts(map[string]testCount{ + "4.22": {100, 90, 0}, "4.21": {100, 92, 0}, "4.20": {100, 93, 0}, "4.19": {100, 93, 0}, + }), + }, +} + +// releaseTimeWindow returns the start/end times for a release's test data. +func releaseTimeWindow(release string) (start, end time.Time) { + now := time.Now().UTC().Truncate(time.Hour) + switch release { + case "4.22": + return now.Add(-3 * 24 * time.Hour), now + case "4.21": + return now.Add(-60 * 24 * time.Hour), now.Add(-30 * 24 * time.Hour) + case "4.20": + return now.Add(-120 * 24 * time.Hour), now.Add(-90 * 24 * time.Hour) + case "4.19": + return now.Add(-180 * 24 * time.Hour), now.Add(-150 * 24 * time.Hour) + default: + return now.Add(-14 * 24 * time.Hour), now } +} - f.BindFlags(cmd.Flags()) +func seedSyntheticData(dbc *db.DB) error { + // Check if data already exists + var count int64 + if err := dbc.DB.Model(&models.ProwJob{}).Count(&count).Error; err != nil { + return fmt.Errorf("failed to check for existing data: %w", err) + } + if count > 0 { + log.Infof("Database already contains %d ProwJobs, skipping seed. Use --init-database to reset.", count) + return nil + } - return cmd + if err := createTestSuite(dbc, "synthetic"); err != nil { + return errors.WithMessage(err, "failed to create test suite") + } + log.Info("Created test suite 'synthetic'") + + if err := seedProwJobs(dbc); err != nil { + return err + } + + if err := seedTestsAndOwnerships(dbc); err != nil { + return err + } + + totalRuns, totalResults, err := seedJobRunsAndResults(dbc) + if err != nil { + return err + } + + if err := createLabelsAndSymptoms(dbc); err != nil { + return errors.WithMessage(err, "failed to create labels and symptoms") + } + + if err := writeSyntheticViewsFile(); err != nil { + return errors.WithMessage(err, "failed to write views file") + } + + log.Info("Refreshing materialized views...") + sippyserver.RefreshData(dbc, nil, false) + + log.Info("Syncing regressions...") + if err := syncRegressions(dbc); err != nil { + return errors.WithMessage(err, "failed to sync regressions") + } + + log.Infof("Seeded synthetic data: %d ProwJobRuns, %d test results across %d releases", + totalRuns, totalResults, len(syntheticReleases)) + return nil } -func createProwJobsForRelease(dbc *db.DB, release string, jobsPerRelease int) error { - for i := 1; i <= jobsPerRelease; i++ { - // Choose JobTier based on whether i is even or odd - var jobTier = "JobTier:standard" // even number job index = standard - if i%2 != 0 { - jobTier = "JobTier:hidden" // odd = hidden +func seedProwJobs(dbc *db.DB) error { + for _, release := range syntheticReleases { + for _, job := range syntheticJobs { + name := fmt.Sprintf(job.nameTemplate, release) + variants := variantMapToArray(job.variants) + prowJob := models.ProwJob{ + Kind: models.ProwKind("periodic"), + Name: name, + Release: release, + Variants: variants, + } + var existing models.ProwJob + if err := dbc.DB.Where("name = ?", name).FirstOrCreate(&existing, prowJob).Error; err != nil { + return fmt.Errorf("failed to create ProwJob %s: %w", name, err) + } } + } + log.Infof("Created ProwJobs for %d releases x %d jobs", len(syntheticReleases), len(syntheticJobs)) + return nil +} + +type testInfo struct { + name string + uniqueID string + component string + capabilities []string +} + +func seedTestsAndOwnerships(dbc *db.DB) error { + var suite models.Suite + if err := dbc.DB.Where("name = ?", "synthetic").First(&suite).Error; err != nil { + return fmt.Errorf("failed to find suite: %w", err) + } - prowJob := models.ProwJob{ - Kind: models.ProwKind("periodic"), - Name: fmt.Sprintf("sippy-test-job-%s-test-%d", release, i), - Release: release, - // TestGridURL, Bugs, and JobRuns are left empty as requested - Variants: []string{"Platform:aws", "Upgrade:none", jobTier}, + seenTests := map[string]testInfo{} + for _, ts := range syntheticTests { + if _, ok := seenTests[ts.testName]; !ok { + seenTests[ts.testName] = testInfo{ + name: ts.testName, + uniqueID: ts.testID, + component: ts.component, + capabilities: ts.capabilities, + } } + } - // Use FirstOrCreate to avoid duplicates - only creates if a ProwJob with this name doesn't exist - var existingJob models.ProwJob - if err := dbc.DB.Where("name = ?", prowJob.Name).FirstOrCreate(&existingJob, prowJob).Error; err != nil { - return fmt.Errorf("failed to create or find ProwJob %s: %v", prowJob.Name, err) + for _, info := range seenTests { + testModel := models.Test{Name: info.name} + var existingTest models.Test + if err := dbc.DB.Where("name = ?", info.name).FirstOrCreate(&existingTest, testModel).Error; err != nil { + return fmt.Errorf("failed to create Test %s: %w", info.name, err) } - // Log whether we created a new job or found an existing one - if existingJob.CreatedAt.IsZero() || existingJob.CreatedAt.Equal(existingJob.UpdatedAt) { - log.Debugf("Created new ProwJob: %s", prowJob.Name) - } else { - log.Debugf("ProwJob already exists: %s", prowJob.Name) + ownership := models.TestOwnership{ + UniqueID: info.uniqueID, + Name: info.name, + TestID: existingTest.ID, + Suite: "synthetic", + SuiteID: &suite.ID, + Component: info.component, + Capabilities: info.capabilities, + } + var existingOwnership models.TestOwnership + if err := dbc.DB.Where("name = ? AND suite = ?", info.name, "synthetic").FirstOrCreate(&existingOwnership, ownership).Error; err != nil { + return fmt.Errorf("failed to create TestOwnership for %s: %w", info.name, err) } } - + log.Infof("Created %d tests with ownership records", len(seenTests)) return nil } -func createTestModels(dbc *db.DB, testNames []string) error { - for _, testName := range testNames { - testModel := models.Test{ - Name: testName, +type jobReleaseKey struct { + jobTemplate string + release string +} + +func seedJobRunsAndResults(dbc *db.DB) (int, int, error) { + var suite models.Suite + if err := dbc.DB.Where("name = ?", "synthetic").First(&suite).Error; err != nil { + return 0, 0, fmt.Errorf("failed to find suite: %w", err) + } + + maxRuns := map[jobReleaseKey]int{} + for _, ts := range syntheticTests { + for jobTpl, releaseCounts := range ts.jobCounts { + for release, counts := range releaseCounts { + key := jobReleaseKey{jobTpl, release} + if counts.total > maxRuns[key] { + maxRuns[key] = counts.total + } + } } + } - // Use FirstOrCreate to avoid duplicates - only creates if a Test with this name doesn't exist - var existingTest models.Test - if err := dbc.DB.Where("name = ?", testModel.Name).FirstOrCreate(&existingTest, testModel).Error; err != nil { - return fmt.Errorf("failed to create or find Test %s: %v", testModel.Name, err) + testIDsByName := map[string]uint{} + var allTests []models.Test + if err := dbc.DB.Find(&allTests).Error; err != nil { + return 0, 0, fmt.Errorf("failed to fetch tests: %w", err) + } + for _, t := range allTests { + testIDsByName[t.Name] = t.ID + } + + totalRuns := 0 + totalResults := 0 + for jrKey, runCount := range maxRuns { + if runCount == 0 { + continue } - if existingTest.CreatedAt.IsZero() || existingTest.CreatedAt.Equal(existingTest.UpdatedAt) { - log.Debugf("Created new Test: %s", testModel.Name) - } else { - log.Debugf("Test already exists: %s", testModel.Name) + jobName := fmt.Sprintf(jrKey.jobTemplate, jrKey.release) + var prowJob models.ProwJob + if err := dbc.DB.Where("name = ?", jobName).First(&prowJob).Error; err != nil { + return 0, 0, fmt.Errorf("failed to find ProwJob %s: %w", jobName, err) + } + + runs, results, err := seedRunsForJob(dbc, &suite, prowJob, jrKey, runCount, testIDsByName) + if err != nil { + return 0, 0, err } + totalRuns += runs + totalResults += results + + log.Debugf("Created %d runs for %s", runCount, jobName) } - return nil + return totalRuns, totalResults, nil } -func createTestSuite(dbc *db.DB) error { - suite := models.Suite{ - Name: "ourtests", +func seedRunsForJob(dbc *db.DB, suite *models.Suite, prowJob models.ProwJob, jrKey jobReleaseKey, runCount int, testIDsByName map[string]uint) (int, int, error) { + start, end := releaseTimeWindow(jrKey.release) + window := end.Sub(start) + interval := window / time.Duration(runCount) + + runIDs := make([]uint, runCount) + for i := range runCount { + timestamp := start.Add(time.Duration(i) * interval) + run := models.ProwJobRun{ + ProwJobID: prowJob.ID, + Cluster: "build01", + Timestamp: timestamp, + Duration: 3 * time.Hour, + } + if err := dbc.DB.Create(&run).Error; err != nil { + return 0, 0, fmt.Errorf("failed to create ProwJobRun: %w", err) + } + runIDs[i] = run.ID } - // Use FirstOrCreate to avoid duplicates - var existingSuite models.Suite - if err := dbc.DB.Where("name = ?", suite.Name).FirstOrCreate(&existingSuite, suite).Error; err != nil { - return fmt.Errorf("failed to create or find Suite %s: %v", suite.Name, err) + // Runs that get test results (all except the last 2) + testableRuns := runCount + if testableRuns > 2 { + testableRuns = runCount - 2 } - return nil -} + runsWithFailure := map[uint]bool{} + totalResults := 0 + + for _, ts := range syntheticTests { + releaseCounts, hasJob := ts.jobCounts[jrKey.jobTemplate] + if !hasJob { + continue + } + counts, hasRelease := releaseCounts[jrKey.release] + if !hasRelease || counts.total == 0 { + continue + } + + testID, ok := testIDsByName[ts.testName] + if !ok { + return 0, 0, fmt.Errorf("test %q not found in DB", ts.testName) + } + + for i := 0; i < counts.total && i < testableRuns; i++ { + var status int + switch { + case i < counts.success-counts.flake: + status = 1 // pass + case i < counts.success: + status = 13 // flake (counts as success too) + default: + status = 12 // failure + runsWithFailure[runIDs[i]] = true + } -func createProwJobRuns(dbc *db.DB, runsPerJob int) error { - var prowJobs []models.ProwJob - if err := dbc.DB.Find(&prowJobs).Error; err != nil { - return fmt.Errorf("failed to fetch existing ProwJobs: %v", err) + result := models.ProwJobRunTest{ + ProwJobRunID: runIDs[i], + TestID: testID, + SuiteID: &suite.ID, + Status: status, + Duration: 5.0, + CreatedAt: start.Add(time.Duration(i) * interval), + } + if err := dbc.DB.Create(&result).Error; err != nil { + return 0, 0, fmt.Errorf("failed to create ProwJobRunTest: %w", err) + } + totalResults++ + } } - var tests []models.Test - if err := dbc.DB.Find(&tests).Error; err != nil { - return fmt.Errorf("failed to fetch existing Tests: %v", err) + // Set OverallResult on all runs + for i, runID := range runIDs { + var overallResult v1.JobOverallResult + var succeeded, failed bool + + if i >= testableRuns { + overallResult = v1.JobInternalInfrastructureFailure + failed = true + } else if runsWithFailure[runID] { + overallResult = v1.JobTestFailure + failed = true + } else { + overallResult = v1.JobSucceeded + succeeded = true + } + + if err := dbc.DB.Model(&models.ProwJobRun{}).Where("id = ?", runID). + Updates(map[string]any{ + "overall_result": overallResult, + "succeeded": succeeded, + "failed": failed, + }).Error; err != nil { + return 0, 0, fmt.Errorf("failed to update ProwJobRun result: %w", err) + } } - var suite models.Suite - if err := dbc.DB.Where("name = ?", "ourtests").First(&suite).Error; err != nil { - return fmt.Errorf("failed to find Suite 'ourtests': %v", err) + // Update test_failures count + if err := dbc.DB.Exec(` + UPDATE prow_job_runs SET test_failures = COALESCE(( + SELECT COUNT(*) FROM prow_job_run_tests + WHERE prow_job_run_id = prow_job_runs.id AND status = 12 + ), 0) WHERE prow_job_id = ?`, prowJob.ID).Error; err != nil { + return 0, 0, fmt.Errorf("updating test_failures for prow job %s: %w", prowJob.Name, err) } - log.Infof("Found %d ProwJobs, creating %d runs for each", len(prowJobs), runsPerJob) + return runCount, totalResults, nil +} - // Calculate time range: past 2 weeks from now - now := time.Now() - twoWeeksAgo := now.AddDate(0, 0, -14) +func syncRegressions(dbc *db.DB) error { + provider := pgprovider.NewPostgresProvider(dbc, nil) + ctx := context.Background() - // Duration for each run: 3 hours - runDuration := 3 * time.Hour + releases, err := provider.QueryReleases(ctx) + if err != nil { + return fmt.Errorf("querying releases: %w", err) + } - for _, prowJob := range prowJobs { - log.Infof("Creating %d ProwJobRuns for ProwJob: %s", runsPerJob, prowJob.Name) + viewsData, err := os.ReadFile(syntheticViewsFile) + if err != nil { + return fmt.Errorf("reading views file: %w", err) + } + var views apitype.SippyViews + if err := yaml.Unmarshal(viewsData, &views); err != nil { + return fmt.Errorf("parsing views file: %w", err) + } - for i := 0; i < runsPerJob; i++ { - // Log progress every 10 runs to show activity - if (i+1)%10 == 0 { - log.Infof(" Progress: %d/%d runs created for %s", i+1, runsPerJob, prowJob.Name) - } + tracker := componentreadiness.NewRegressionTracker( + provider, dbc, + cache.RequestOptions{}, + releases, + componentreadiness.NewPostgresRegressionStore(dbc, nil), + views.ComponentReadiness, + false, + ) + tracker.Load() + if len(tracker.Errors()) > 0 { + for _, err := range tracker.Errors() { + log.WithError(err).Warn("regression tracker error") + } + return fmt.Errorf("regression tracker encountered %d errors", len(tracker.Errors())) + } + return nil +} - // Calculate timestamp: spread evenly over the past 2 weeks - totalDuration := 14 * 24 * time.Hour - // Time between runs = total duration / runs - timeBetweenRuns := totalDuration / time.Duration(runsPerJob) - timestamp := twoWeeksAgo.Add(time.Duration(i) * timeBetweenRuns) - - prowJobRun := models.ProwJobRun{ - ProwJobID: prowJob.ID, - Cluster: "build01", - Timestamp: timestamp, - Duration: runDuration, - TestCount: len(tests), - } +const syntheticViewsFile = "config/e2e-views.yaml" - if err := dbc.DB.Create(&prowJobRun).Error; err != nil { - return fmt.Errorf("failed to create ProwJobRun for ProwJob %s: %v", prowJob.Name, err) +// writeSyntheticViewsFile generates a views file with include_variants matching the seed data. +func writeSyntheticViewsFile() error { + // Collect all unique variant values from synthetic jobs + allVariants := map[string]map[string]bool{} + for _, job := range syntheticJobs { + for k, v := range job.variants { + if allVariants[k] == nil { + allVariants[k] = map[string]bool{} } + allVariants[k][v] = true + } + } - var testFailures int - for _, test := range tests { - // Determine test status based on random chance - // 5% chance of failure, 10% chance of flake, 85% chance of pass - // nolint: gosec - randNum := rand.Float64() - var status int - if randNum < 0.05 { - status = 12 // failure - testFailures++ - } else if randNum < 0.15 { - status = 13 // flake - } else { - status = 1 // pass - } + includeVariants := map[string][]string{} + for k, vals := range allVariants { + sorted := make([]string, 0, len(vals)) + for v := range vals { + sorted = append(sorted, v) + } + sort.Strings(sorted) + includeVariants[k] = sorted + } - prowJobRunTest := models.ProwJobRunTest{ - ProwJobRunID: prowJobRun.ID, - TestID: test.ID, - SuiteID: &suite.ID, - Status: status, - Duration: 5.0, // 5 seconds - CreatedAt: timestamp, - } + dbGroupBy := sets.NewString("Architecture", "FeatureSet", "Installer", "Network", "Platform", + "Suite", "Topology", "Upgrade", "LayeredProduct") + columnGroupBy := sets.NewString("Network", "Platform", "Topology") + + views := apitype.SippyViews{ + ComponentReadiness: []crview.View{ + { + Name: "4.22-main", + BaseRelease: reqopts.RelativeRelease{ + Release: reqopts.Release{Name: "4.21"}, + RelativeStart: "now-60d", + RelativeEnd: "now-30d", + }, + SampleRelease: reqopts.RelativeRelease{ + Release: reqopts.Release{Name: "4.22"}, + RelativeStart: "now-3d", + RelativeEnd: "now", + }, + VariantOptions: reqopts.Variants{ + ColumnGroupBy: columnGroupBy, + DBGroupBy: dbGroupBy, + IncludeVariants: includeVariants, + }, + AdvancedOptions: reqopts.Advanced{ + Confidence: 95, + PityFactor: 5, + MinimumFailure: 3, + PassRateRequiredNewTests: 90, + IncludeMultiReleaseAnalysis: true, + }, + PrimeCache: crview.PrimeCache{Enabled: true}, + RegressionTracking: crview.RegressionTracking{Enabled: true}, + }, + }, + } - if err := dbc.DB.Create(&prowJobRunTest).Error; err != nil { - return fmt.Errorf("failed to create ProwJobRunTest for test %s: %v", test.Name, err) - } - } + data, err := yaml.Marshal(views) + if err != nil { + return fmt.Errorf("marshaling views: %w", err) + } - // Set overall result based on test failures and random factors - var overallResult v1.JobOverallResult - if testFailures > 0 { - prowJobRun.Failed = true - prowJobRun.Succeeded = false - prowJobRun.TestFailures = testFailures - - // Randomly assign different failure types - // nolint: gosec - failureType := rand.Float64() - if failureType < 0.7 { - overallResult = v1.JobTestFailure // 70% test failures - } else if failureType < 0.85 { - overallResult = v1.JobUpgradeFailure // 15% upgrade failures - } else if failureType < 0.92 { - overallResult = v1.JobInstallFailure // 7% install failures - } else { - overallResult = v1.JobExternalInfrastructureFailure // 8% infrastructure failures - } - } else { - prowJobRun.Failed = false - prowJobRun.Succeeded = true - prowJobRun.TestFailures = 0 - overallResult = v1.JobSucceeded - } - prowJobRun.OverallResult = overallResult + if err := os.WriteFile(syntheticViewsFile, data, 0o600); err != nil { + return fmt.Errorf("writing %s: %w", syntheticViewsFile, err) + } - if err := dbc.DB.Save(&prowJobRun).Error; err != nil { - return fmt.Errorf("failed to update ProwJobRun for ProwJob %s: %v", prowJob.Name, err) - } - } + log.Infof("Generated views file: %s", syntheticViewsFile) + return nil +} + +// variantMapToArray converts a variant map to a pq.StringArray. +func variantMapToArray(m map[string]string) pq.StringArray { + result := make([]string, 0, len(m)) + for k, v := range m { + result = append(result, k+":"+v) + } + return result +} + +func createTestSuite(dbc *db.DB, name string) error { + suite := models.Suite{ + Name: name, + } - log.Infof("Completed creating %d ProwJobRuns for ProwJob: %s", runsPerJob, prowJob.Name) + var existingSuite models.Suite + if err := dbc.DB.Where("name = ?", suite.Name).FirstOrCreate(&existingSuite, suite).Error; err != nil { + return fmt.Errorf("failed to create or find Suite %s: %v", suite.Name, err) } return nil @@ -339,13 +816,12 @@ func createLabelsAndSymptoms(dbc *db.DB) error { UpdatedBy: "seed-data", } - // Create sample labels labels := []jobrunscan.Label{ { LabelContent: jobrunscan.LabelContent{ ID: "InfraFailure", LabelTitle: "Infrastructure failure: omit job from CR", - Explanation: "Job failed due to **infrastructure issues** not related to product code. See [TRT documentation](https://docs.ci.openshift.org/docs/architecture/ci-operator/) for more details.", + Explanation: "Job failed due to **infrastructure issues** not related to product code.", }, Metadata: metadata, }, @@ -353,7 +829,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "ClusterDNSFlake", LabelTitle: "Cluster DNS resolution failure(s)", - Explanation: "Job experienced DNS resolution timeouts in the cluster:\n\n- Check for network issues\n- Review DNS server logs\n- Examine cluster network configuration", + Explanation: "Job experienced DNS resolution timeouts in the cluster.", }, Metadata: metadata, }, @@ -361,7 +837,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "ClusterInstallTimeout", LabelTitle: "Cluster install timeout", - Explanation: "Cluster installation exceeded timeout threshold. This may indicate:\n\n1. Slow infrastructure provisioning\n2. Network connectivity problems\n3. Image pull failures", + Explanation: "Cluster installation exceeded timeout threshold.", }, Metadata: metadata, }, @@ -369,7 +845,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "IntervalFile", LabelTitle: "Has interval file(s)", - Explanation: "Job produced interval monitoring files. Use the `intervals` tool to analyze timing data.", + Explanation: "Job produced interval monitoring files.", }, HideDisplayContexts: []string{jobrunscan.MetricsContext, jobrunscan.JAQOptsContext}, Metadata: metadata, @@ -378,7 +854,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { LabelContent: jobrunscan.LabelContent{ ID: "APIServerTimeout", LabelTitle: "API server timeout", - Explanation: "Requests to the API server timed out. Common causes:\n\n- High API server load\n- Network latency issues\n- Slow etcd responses", + Explanation: "Requests to the API server timed out.", }, Metadata: metadata, }, @@ -389,14 +865,8 @@ func createLabelsAndSymptoms(dbc *db.DB) error { if err := dbc.DB.Where("id = ?", label.ID).FirstOrCreate(&existing, label).Error; err != nil { return fmt.Errorf("failed to create or find label %s: %v", label.ID, err) } - if existing.CreatedAt.IsZero() || existing.CreatedAt.Equal(existing.UpdatedAt) { - log.Debugf("Created new Label: %s", label.ID) - } else { - log.Debugf("Label already exists: %s", label.ID) - } } - // Create sample symptoms symptoms := []jobrunscan.Symptom{ { SymptomContent: jobrunscan.SymptomContent{ @@ -449,71 +919,7 @@ func createLabelsAndSymptoms(dbc *db.DB) error { if err := dbc.DB.Where("id = ?", symptom.ID).FirstOrCreate(&existing, symptom).Error; err != nil { return fmt.Errorf("failed to create or find symptom %s: %v", symptom.ID, err) } - if existing.CreatedAt.IsZero() || existing.CreatedAt.Equal(existing.UpdatedAt) { - log.Debugf("Created new Symptom: %s", symptom.ID) - } else { - log.Debugf("Symptom already exists: %s", symptom.ID) - } - } - - return nil -} - -func applyLabelsToJobRuns(dbc *db.DB) error { - // Fetch all job runs - var jobRuns []models.ProwJobRun - if err := dbc.DB.Find(&jobRuns).Error; err != nil { - return fmt.Errorf("failed to fetch job runs: %v", err) - } - - // Fetch all labels - var labels []jobrunscan.Label - if err := dbc.DB.Find(&labels).Error; err != nil { - return fmt.Errorf("failed to fetch labels: %v", err) - } - - if len(labels) == 0 { - log.Warn("No labels found, skipping label application") - return nil - } - - labelIDs := make([]string, len(labels)) - for i, label := range labels { - labelIDs[i] = label.ID - } - - // Apply labels to approximately 25% of job runs - labeledCount := 0 - for i := range jobRuns { - // nolint: gosec // we do not care that the randomness is weak - if rand.Float64() > 0.25 { - continue - } - // Randomly select 1-3 labels - // nolint: gosec - numLabels := rand.Intn(3) + 1 - selectedLabels := make([]string, 0, numLabels) - - // Randomly pick unique labels - usedIndices := make(map[int]bool) - for len(selectedLabels) < numLabels && len(selectedLabels) < len(labelIDs) { - // nolint: gosec - idx := rand.Intn(len(labelIDs)) - if !usedIndices[idx] { - selectedLabels = append(selectedLabels, labelIDs[idx]) - usedIndices[idx] = true - } - } - - jobRuns[i].Labels = selectedLabels - if err := dbc.DB.Save(&jobRuns[i]).Error; err != nil { - return fmt.Errorf("failed to update job run %d with labels: %v", jobRuns[i].ID, err) - } - labeledCount++ } - log.Infof("Applied labels to %d of %d job runs (%.1f%%)", - labeledCount, len(jobRuns), float64(labeledCount)/float64(len(jobRuns))*100) - return nil } diff --git a/cmd/sippy/serve.go b/cmd/sippy/serve.go index ee16451e68..631a85d5ea 100644 --- a/cmd/sippy/serve.go +++ b/cmd/sippy/serve.go @@ -19,6 +19,7 @@ import ( resources "github.com/openshift/sippy" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" bqprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/bigquery" + pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" "github.com/openshift/sippy/pkg/apis/cache" "github.com/openshift/sippy/pkg/bigquery" "github.com/openshift/sippy/pkg/bigquery/bqlabel" @@ -69,10 +70,13 @@ func (f *ServerFlags) BindFlags(flagSet *pflag.FlagSet) { f.ConfigFlags.BindFlags(flagSet) f.APIFlags.BindFlags(flagSet) f.JiraFlags.BindFlags(flagSet) - flagSet.StringVar(&f.DataProvider, "data-provider", "bigquery", "Data provider for component readiness: bigquery") + flagSet.StringVar(&f.DataProvider, "data-provider", "bigquery", "Data provider for component readiness: bigquery, postgres") } func (f *ServerFlags) Validate() error { + if f.DataProvider == "postgres" { + return nil + } return f.GoogleCloudFlags.Validate() } @@ -132,8 +136,12 @@ func NewServeCommand() *cobra.Command { crDataProvider = bqprovider.NewBigQueryProvider(bigQueryClient, config.ComponentReadinessConfig.VariantJunitTableOverrides) } + case "postgres": + crDataProvider = pgprovider.NewPostgresProvider(dbc, cacheClient) + log.Info("Using Postgres data provider for component readiness") + default: - return fmt.Errorf("unknown --data-provider %q, must be bigquery", f.DataProvider) + return fmt.Errorf("unknown --data-provider %q, must be bigquery or postgres", f.DataProvider) } gcsClient, err = gcs.NewGCSClient(context.TODO(), diff --git a/config/e2e-openshift.yaml b/config/e2e-openshift.yaml index 39814b6b2a..996e0aa39c 100644 --- a/config/e2e-openshift.yaml +++ b/config/e2e-openshift.yaml @@ -1,8 +1,8 @@ prow: url: https://prow.ci.openshift.org/prowjobs.js releases: - "4.20": + "4.22": jobs: - periodic-ci-openshift-release-master-nightly-4.20-e2e-aws-ovn-serial-2of2: true - periodic-ci-openshift-release-main-ci-4.20-e2e-azure-ovn-upgrade: true - periodic-ci-openshift-release-master-nightly-4.20-e2e-gcp-ovn-csi: true + periodic-ci-openshift-release-master-nightly-4.22-e2e-aws-ovn-serial-2of2: true + periodic-ci-openshift-release-main-ci-4.22-e2e-azure-ovn-upgrade: true + periodic-ci-openshift-release-master-nightly-4.22-e2e-gcp-ovn-csi: true diff --git a/config/e2e-views.yaml b/config/e2e-views.yaml index aab16d861c..36b9e3abb6 100644 --- a/config/e2e-views.yaml +++ b/config/e2e-views.yaml @@ -1,81 +1,71 @@ ---- component_readiness: -- name: 4.20-main - base_release: - release: "4.19" - relative_start: ga-30d - relative_end: ga - sample_release: - release: "4.20" - relative_start: now-7d - relative_end: now - variant_options: - column_group_by: - Architecture: {} - Network: {} - Platform: {} - Topology: {} - db_group_by: - Architecture: {} - FeatureSet: {} - Installer: {} - Network: {} - Platform: {} - Suite: {} - Topology: {} - Upgrade: {} - include_variants: - Architecture: - - amd64 - FeatureSet: - - default - - techpreview - Installer: - - ipi - - upi - - hypershift - JobTier: - - blocking - - informing - - standard - LayeredProduct: - - none - - virt - Network: - - ovn - Owner: - - eng - - service-delivery - Platform: - - aws - - azure - - gcp - - metal - - rosa - - vsphere - Topology: - - ha - - microshift - - external - CGroupMode: - - v2 - ContainerRuntime: - - runc - - crun - advanced_options: - minimum_failure: 3 - confidence: 95 - pity_factor: 5 - ignore_missing: false - ignore_disruption: true - flake_as_failure: false - pass_rate_required_new_tests: 95 - include_multi_release_analysis: true - metrics: - enabled: true - regression_tracking: - enabled: true - prime_cache: - enabled: true - automate_jira: - enabled: true + - name: 4.22-main + base_release: + release: "4.21" + relative_start: now-60d + relative_end: now-30d + sample_release: + release: "4.22" + relative_start: now-3d + relative_end: now + test_id_options: {} + test_filters: {} + variant_options: + column_group_by: + Network: {} + Platform: {} + Topology: {} + db_group_by: + Architecture: {} + FeatureSet: {} + Installer: {} + LayeredProduct: {} + Network: {} + Platform: {} + Suite: {} + Topology: {} + Upgrade: {} + include_variants: + Architecture: + - amd64 + - arm64 + FeatureSet: + - default + - techpreview + Installer: + - ipi + LayeredProduct: + - none + Network: + - ovn + Platform: + - aws + - gcp + Suite: + - parallel + - serial + - unknown + Topology: + - ha + Upgrade: + - micro + - minor + - none + advanced_options: + minimum_failure: 3 + confidence: 95 + pity_factor: 5 + pass_rate_required_new_tests: 90 + pass_rate_required_all_tests: 0 + ignore_missing: false + ignore_disruption: false + flake_as_failure: false + include_multi_release_analysis: true + metrics: + enabled: false + regression_tracking: + enabled: true + automate_jira: + enabled: false + prime_cache: + enabled: true diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh index cd210804b5..327352e49c 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-setup-commands.sh @@ -6,11 +6,6 @@ # When running locally, the user has to define SIPPY_IMAGE. echo "The sippy CI image: ${SIPPY_IMAGE}" -# The GCS_CRED allows us to pull artifacts from GCS when importing prow jobs. -# Redefine GCS_CRED to use your own. -GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" -echo "The GCS cred is: ${GCS_CRED}" - # If you're using Openshift, we use oc, if you're using plain Kubernetes, # we use kubectl. # @@ -61,14 +56,6 @@ fi e2e_pause -echo "Checking for presense of GCS credentials ..." -if [ -f ${GCS_CRED} ]; then - ls -l ${GCS_CRED} -else - echo "Aborting: GCS credential file ${GCS_CRED} not found" - exit 1 -fi - echo "Starting postgres on cluster-pool cluster..." # Make the "postgres" namespace and pod. @@ -261,9 +248,12 @@ ${KUBECTL_CMD} -n sippy-e2e get svc,ep # Get the gcs credentials out to the cluster-pool cluster. # These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations). # See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa -# - -${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred=$GCS_CRED -n sippy-e2e +GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" +if [ -f "${GCS_CRED}" ]; then + ${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred="${GCS_CRED}" -n sippy-e2e +else + echo "WARNING: GCS credential file ${GCS_CRED} not found, BigQuery tests will fail" +fi # Get the registry credentials for all build farm clusters out to the cluster-pool cluster. ${KUBECTL_CMD} -n sippy-e2e create secret generic regcred --from-file=.dockerconfigjson=${DOCKERCONFIGJSON} --type=kubernetes.io/dockerconfigjson @@ -283,12 +273,12 @@ spec: storage: 100Mi END -# Make the "sippy loader" pod. +# Seed synthetic data into postgres. cat << END | ${KUBECTL_CMD} apply -f - apiVersion: batch/v1 kind: Job metadata: - name: sippy-load-job + name: sippy-seed-job namespace: sippy-e2e spec: template: @@ -304,20 +294,19 @@ spec: terminationMessagePolicy: File command: ["/bin/sh", "-c"] args: - - /bin/sippy load --init-database --log-level=debug --release 4.20 --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres --redis-url=redis://redis.sippy-e2e.svc.cluster.local:6379 --mode=ocp --config ./config/e2e-openshift.yaml --google-service-account-credential-file /tmp/secrets/gcs-cred + - /bin/sippy-cover seed-data --init-database --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres env: - - name: GCS_SA_JSON_PATH - value: /tmp/secrets/gcs-cred + - name: GOCOVERDIR + value: /tmp/coverage volumeMounts: - - mountPath: /tmp/secrets - name: gcs-cred - readOnly: true + - mountPath: /tmp/coverage + name: coverage imagePullSecrets: - name: regcred volumes: - - name: gcs-cred - secret: - secretName: gcs-cred + - name: coverage + persistentVolumeClaim: + claimName: sippy-coverage dnsPolicy: ClusterFirst restartPolicy: Never schedulerName: default-scheduler @@ -327,33 +316,33 @@ spec: END date -echo "Waiting for sippy loader job to finish ..." -${KUBECTL_CMD} -n sippy-e2e get job sippy-load-job -${KUBECTL_CMD} -n sippy-e2e describe job sippy-load-job +echo "Waiting for sippy seed job to finish ..." +${KUBECTL_CMD} -n sippy-e2e get job sippy-seed-job +${KUBECTL_CMD} -n sippy-e2e describe job sippy-seed-job # We set +e to avoid the script aborting before we can retrieve logs. set +e -echo "Waiting up to ${SIPPY_LOAD_TIMEOUT:=1200s} for the sippy-load-job to complete..." -${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-load-job --timeout ${SIPPY_LOAD_TIMEOUT} +echo "Waiting up to ${SIPPY_LOAD_TIMEOUT:=1200s} for the sippy-seed-job to complete..." +${KUBECTL_CMD} -n sippy-e2e wait --for=condition=complete job/sippy-seed-job --timeout "${SIPPY_LOAD_TIMEOUT}" retVal=$? set -e -job_pod=$(${KUBECTL_CMD} -n sippy-e2e get pod --selector=job-name=sippy-load-job --output=jsonpath='{.items[0].metadata.name}') -${KUBECTL_CMD} -n sippy-e2e logs ${job_pod} > ${ARTIFACT_DIR}/sippy-load.log 2>&1 +job_pod=$(${KUBECTL_CMD} -n sippy-e2e get pod --selector=job-name=sippy-seed-job --output=jsonpath='{.items[0].metadata.name}') +${KUBECTL_CMD} -n sippy-e2e logs "${job_pod}" > "${ARTIFACT_DIR}/sippy-seed.log" 2>&1 if [ ${retVal} -ne 0 ]; then echo - echo "=== SIPPY LOAD JOB FAILURE DIAGNOSTICS ===" + echo "=== SIPPY SEED JOB FAILURE DIAGNOSTICS ===" echo "=== Job status ===" - ${KUBECTL_CMD} -n sippy-e2e describe job sippy-load-job + ${KUBECTL_CMD} -n sippy-e2e describe job sippy-seed-job echo "=== Job pod status ===" ${KUBECTL_CMD} -n sippy-e2e describe pod ${job_pod} echo "=== Recent namespace events ===" ${KUBECTL_CMD} -n sippy-e2e get events --sort-by='.lastTimestamp' - echo "=== END SIPPY LOAD JOB FAILURE DIAGNOSTICS ===" + echo "=== END SIPPY SEED JOB FAILURE DIAGNOSTICS ===" echo - echo "ERROR: sippy-load-job did not complete within ${SIPPY_LOAD_TIMEOUT}" + echo "ERROR: sippy-seed-job did not complete within ${SIPPY_LOAD_TIMEOUT}" exit 1 fi diff --git a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh index 72b05151d9..1f8603fe4c 100755 --- a/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh +++ b/e2e-scripts/sippy-e2e-sippy-e2e-test-commands.sh @@ -16,25 +16,20 @@ trap cleanup EXIT # When running locally, the user has to define SIPPY_IMAGE. echo "The sippy CI image: ${SIPPY_IMAGE}" -# The GCS_CRED allows us to pull artifacts from GCS when importing prow jobs. -# Redefine GCS_CRED to use your own. -GCS_CRED="${GCS_CRED:=/var/run/sippy-bigquery-job-importer/gcs-sa}" -echo "The GCS cred is: ${GCS_CRED}" - # If you're using Openshift, we use oc, if you're using plain Kubernetes, # we use kubectl. # KUBECTL_CMD="${KUBECTL_CMD:=oc}" echo "The kubectl command is: ${KUBECTL_CMD}" -# Get the gcs credentials out to the cluster-pool cluster. -# These credentials are in vault and maintained by the TRT team (e.g. for updates and rotations). -# See https://vault.ci.openshift.org/ui/vault/secrets/kv/show/selfservice/technical-release-team/sippy-ci-gcs-read-sa -# -${KUBECTL_CMD} create secret generic gcs-cred --from-file gcs-cred=$GCS_CRED -n sippy-e2e +# The datasync test runs sippy load as a k8s Job, so it needs these to create the pod. +export SIPPY_E2E_SIPPY_IMAGE="${SIPPY_IMAGE}" -# Launch the sippy api server pod with coverage instrumentation. -cat << END | ${KUBECTL_CMD} apply -f - +launch_sippy_server() { + local DATA_PROVIDER=$1 + local EXTRA_ARGS="${2:-}" + + cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 kind: Pod metadata: @@ -74,8 +69,8 @@ spec: - ":12112" - --database-dsn=postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres - --redis-url=redis://redis.sippy-e2e.svc.cluster.local:6379 - - --google-service-account-credential-file - - /tmp/secrets/gcs-cred + - --data-provider + - ${DATA_PROVIDER} - --log-level - debug - --enable-write-endpoints @@ -83,6 +78,8 @@ spec: - ocp - --views - ./config/e2e-views.yaml + - --google-service-account-credential-file + - /tmp/secrets/gcs-cred env: - name: GCS_SA_JSON_PATH value: /tmp/secrets/gcs-cred @@ -110,27 +107,39 @@ spec: terminationGracePeriodSeconds: 30 END -# The basic readiness probe will give us at least 10 seconds before declaring the pod as ready. -echo "Waiting for sippy api server pod to be Ready ..." -set +e -${KUBECTL_CMD} -n sippy-e2e wait --for=condition=Ready pod/sippy-server --timeout=600s -server_retVal=$? -set -e - -${KUBECTL_CMD} -n sippy-e2e get pod -o wide -${KUBECTL_CMD} -n sippy-e2e logs sippy-server > ${ARTIFACT_DIR}/sippy-server.log 2>&1 - -if [ ${server_retVal} -ne 0 ]; then - echo - echo "=== SIPPY SERVER FAILURE DIAGNOSTICS ===" - ${KUBECTL_CMD} -n sippy-e2e describe pod/sippy-server - echo "=== Namespace events ===" - ${KUBECTL_CMD} -n sippy-e2e get events --sort-by='.lastTimestamp' - echo "=== END SIPPY SERVER FAILURE DIAGNOSTICS ===" - echo - echo "ERROR: sippy-server pod never became Ready (timed out after 600s)" - exit 1 -fi + echo "Waiting for sippy api server pod (${DATA_PROVIDER}) to be Ready ..." + set +e + ${KUBECTL_CMD} -n sippy-e2e wait --for=condition=Ready pod/sippy-server --timeout=600s + local retVal=$? + set -e + + ${KUBECTL_CMD} -n sippy-e2e get pod -o wide + ${KUBECTL_CMD} -n sippy-e2e logs sippy-server > ${ARTIFACT_DIR}/sippy-server-${DATA_PROVIDER}.log 2>&1 + + if [ ${retVal} -ne 0 ]; then + echo + echo "=== SIPPY SERVER FAILURE DIAGNOSTICS (${DATA_PROVIDER}) ===" + ${KUBECTL_CMD} -n sippy-e2e describe pod/sippy-server + echo "=== Namespace events ===" + ${KUBECTL_CMD} -n sippy-e2e get events --sort-by='.lastTimestamp' + echo "=== END SIPPY SERVER FAILURE DIAGNOSTICS ===" + echo + echo "ERROR: sippy-server pod (${DATA_PROVIDER}) never became Ready (timed out after 600s)" + return 1 + fi + return 0 +} + +stop_sippy_server() { + local DATA_PROVIDER=$1 + echo "Stopping sippy-server (${DATA_PROVIDER}) to flush coverage data..." + ${KUBECTL_CMD} -n sippy-e2e logs sippy-server > ${ARTIFACT_DIR}/sippy-server-${DATA_PROVIDER}.log 2>&1 || true + ${KUBECTL_CMD} -n sippy-e2e delete pod sippy-server --wait=true --timeout=60s || true + ${KUBECTL_CMD} -n sippy-e2e delete svc sippy-server || true +} + +# Phase 1: Launch postgres-backed server +launch_sippy_server postgres || exit 1 echo "Setup services and port forwarding for the sippy api server ..." @@ -144,6 +153,7 @@ export SIPPY_API_PORT # Setup port forward for random port to get to the sippy-server pod ${KUBECTL_CMD} -n sippy-e2e expose pod sippy-server ${KUBECTL_CMD} -n sippy-e2e port-forward pod/sippy-server ${SIPPY_API_PORT}:8080 & +PF_PID_SERVER=$! # Random port for postgres as well, between 18500 and 19000 # Direct postgres access is used for some e2e test to seed data and cleanup things we don't expose on the api, @@ -156,7 +166,6 @@ ${KUBECTL_CMD} -n sippy-e2e expose pod postg1 ${KUBECTL_CMD} -n sippy-e2e port-forward pod/postg1 ${SIPPY_PSQL_PORT}:5432 & # Random port for redis as well, between 19000 and 19500 -# Direct redis access is used for e2e tests to manipulate cache during testing. SIPPY_REDIS_PORT=$((RANDOM % 501 + 19000)) export SIPPY_REDIS_PORT export REDIS_URL="redis://localhost:${SIPPY_REDIS_PORT}" @@ -166,17 +175,62 @@ ${KUBECTL_CMD} -n sippy-e2e port-forward pod/redis1 ${SIPPY_REDIS_PORT}:6379 & ${KUBECTL_CMD} -n sippy-e2e get svc,ep -# only 1 in parallel, some tests will clash if run at the same time -gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e.xml -- ./test/e2e/... -v -p 1 -coverprofile=${ARTIFACT_DIR}/e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... -TEST_EXIT=$? +E2E_EXIT_CODE=0 + +# Prime the component readiness cache so triage tests can find cached reports +echo "Priming component readiness cache..." +VIEWS=$(curl -sf "http://localhost:${SIPPY_API_PORT}/api/component_readiness/views") || { echo "Failed to fetch views"; exit 1; } +for VIEW in $(echo "$VIEWS" | jq -r '.[].name'); do + echo " Priming cache for view: $VIEW" + curl -sf "http://localhost:${SIPPY_API_PORT}/api/component_readiness?view=$VIEW" > /dev/null || { echo "Failed to prime cache for view: $VIEW"; exit 1; } +done +echo "Cache priming complete" + +echo "=== Phase 1: Running postgres-backed e2e tests ===" +gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e_postgres.xml -- \ + ./test/e2e/componentreadiness/postgres/... \ + ./test/e2e/componentreadiness/bugs/... \ + ./test/e2e/datasync/... \ + ./test/e2e/ \ + -v -p 1 -coverprofile=${ARTIFACT_DIR}/e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... +POSTGRES_EXIT=$? +if [ ${POSTGRES_EXIT} -ne 0 ]; then + E2E_EXIT_CODE=${POSTGRES_EXIT} +fi -# Collect coverage data. Coverage counters are flushed when the server exits. -# Pod deletion sends SIGTERM during graceful termination (terminationGracePeriodSeconds: 30), -# so we just delete the pod directly — no need for a separate exec kill. -echo "Stopping sippy-server to flush coverage data..." -${KUBECTL_CMD} -n sippy-e2e delete pod sippy-server --wait=true --timeout=60s || true +# Stop the postgres server, kill the port-forward, and restart with bigquery +kill ${PF_PID_SERVER} 2>/dev/null || true +stop_sippy_server postgres + +# Phase 2: Launch bigquery-backed server +echo "=== Phase 2: Running BigQuery-backed e2e tests ===" +launch_sippy_server bigquery || exit 1 + +${KUBECTL_CMD} -n sippy-e2e expose pod sippy-server +${KUBECTL_CMD} -n sippy-e2e port-forward pod/sippy-server ${SIPPY_API_PORT}:8080 & +PF_PID_SERVER=$! + +gotestsum --junitfile ${ARTIFACT_DIR}/junit_e2e_bigquery.xml -- \ + ./test/e2e/componentreadiness/bigquery/... \ + -v -p 1 -coverprofile=${ARTIFACT_DIR}/e2e-bq-test-coverage.out -coverpkg=./pkg/...,./cmd/... +BQ_EXIT=$? +if [ ${BQ_EXIT} -ne 0 ]; then + E2E_EXIT_CODE=${BQ_EXIT} +fi + +kill ${PF_PID_SERVER} 2>/dev/null || true +stop_sippy_server bigquery + +echo "=== Running unit tests for coverage ===" +gotestsum --junitfile ${ARTIFACT_DIR}/junit_unit.xml -- \ + ./pkg/... \ + -v -coverprofile=${ARTIFACT_DIR}/unit-test-coverage.out -coverpkg=./pkg/...,./cmd/... +UNIT_EXIT=$? +if [ ${UNIT_EXIT} -ne 0 ]; then + E2E_EXIT_CODE=${UNIT_EXIT} +fi -# Launch a minimal helper pod to access the coverage PVC. +# Collect coverage data from both server runs cat << END | ${KUBECTL_CMD} apply -f - apiVersion: v1 kind: Pod @@ -206,16 +260,19 @@ ${KUBECTL_CMD} -n sippy-e2e wait --for=condition=Ready pod/coverage-helper --tim COVDIR=$(mktemp -d) ${KUBECTL_CMD} -n sippy-e2e cp coverage-helper:/tmp/coverage "${COVDIR}" -c helper || true -if find "${COVDIR}" -name 'covcounters.*' -print -quit 2>/dev/null | grep -q .; then - echo "Generating coverage report..." - go tool covdata percent -i="${COVDIR}" - go tool covdata textfmt -i="${COVDIR}" -o="${ARTIFACT_DIR}/e2e-coverage.out" - # Merge test binary coverage (from -coverprofile) into server binary coverage - if [ -f "${ARTIFACT_DIR}/e2e-test-coverage.out" ]; then - echo "Merging test binary coverage into server coverage..." - tail -n +2 "${ARTIFACT_DIR}/e2e-test-coverage.out" >> "${ARTIFACT_DIR}/e2e-coverage.out" - rm -f "${ARTIFACT_DIR}/e2e-test-coverage.out" - fi +COVERAGE_ROOT=$(find "${COVDIR}" -name 'covcounters.*' -print -quit 2>/dev/null | xargs -r dirname) +COVERAGE_ROOT="${COVERAGE_ROOT:-${COVDIR}}" +if find "${COVERAGE_ROOT}" -name 'covcounters.*' -print -quit 2>/dev/null | grep -q .; then + echo "Generating coverage report from ${COVERAGE_ROOT}..." + go tool covdata percent -i="${COVERAGE_ROOT}" + go tool covdata textfmt -i="${COVERAGE_ROOT}" -o="${ARTIFACT_DIR}/e2e-coverage.out" + for f in ${ARTIFACT_DIR}/e2e-test-coverage.out ${ARTIFACT_DIR}/e2e-bq-test-coverage.out ${ARTIFACT_DIR}/unit-test-coverage.out; do + if [ -f "$f" ]; then + echo "Merging $f into server coverage..." + tail -n +2 "$f" >> "${ARTIFACT_DIR}/e2e-coverage.out" + rm -f "$f" + fi + done go tool cover -html="${ARTIFACT_DIR}/e2e-coverage.out" -o="${ARTIFACT_DIR}/e2e-coverage.html" echo "Coverage report written to ${ARTIFACT_DIR}/e2e-coverage.html" else @@ -225,4 +282,4 @@ rm -rf "${COVDIR}" ${KUBECTL_CMD} -n sippy-e2e delete secret regcred || true -exit ${TEST_EXIT} +exit ${E2E_EXIT_CODE} diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go new file mode 100644 index 0000000000..ef09207645 --- /dev/null +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -0,0 +1,791 @@ +package postgres + +import ( + "context" + "fmt" + "math/big" + "slices" + "sort" + "strings" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + + "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" + "github.com/openshift/sippy/pkg/api/componentreadiness/utils" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + "github.com/openshift/sippy/pkg/db" +) + +var _ dataprovider.DataProvider = &PostgresProvider{} + +// PostgresProvider implements dataprovider.DataProvider using PostgreSQL. +// Designed for local development and testing — not optimized for production scale. +type PostgresProvider struct { + dbc *db.DB + cache cache.Cache +} + +func NewPostgresProvider(dbc *db.DB, c cache.Cache) *PostgresProvider { + if c == nil { + c = &noOpCache{} + } + return &PostgresProvider{dbc: dbc, cache: c} +} + +// noOpCache never stores or returns data; no Redis needed for local dev. +type noOpCache struct{} + +func (n *noOpCache) Get(_ context.Context, _ string, _ time.Duration) ([]byte, error) { + return nil, fmt.Errorf("cache miss") +} +func (n *noOpCache) Set(_ context.Context, _ string, _ []byte, _ time.Duration) error { return nil } + +func (p *PostgresProvider) Cache() cache.Cache { + return p.cache +} + +// --- Variant helpers --- + +// parseVariants splits a pq.StringArray like ["Platform:aws", "Upgrade:none"] into a map. +func parseVariants(variants pq.StringArray) map[string]string { + result := make(map[string]string, len(variants)) + for _, v := range variants { + if k, val, ok := strings.Cut(v, ":"); ok { + result[k] = val + } + } + return result +} + +// variantMapToSlice converts a map to sorted "Key:Value" strings. +func variantMapToSlice(m map[string]string) []string { + result := make([]string, 0, len(m)) + for k, v := range m { + result = append(result, k+":"+v) + } + sort.Strings(result) + return result +} + +// filterByDBGroupBy returns a copy of the variant map keeping only keys in dbGroupBy. +func filterByDBGroupBy(variants map[string]string, dbGroupBy map[string]bool) map[string]string { + filtered := make(map[string]string, len(dbGroupBy)) + for k, v := range variants { + if dbGroupBy[k] { + filtered[k] = v + } + } + return filtered +} + +// matchesIncludeVariants checks if a variant map passes the include filter. +func matchesIncludeVariants(variants map[string]string, includeVariants map[string][]string) bool { + for key, allowed := range includeVariants { + val, exists := variants[key] + if !exists { + return false + } + if !slices.Contains(allowed, val) { + return false + } + } + return true +} + +// --- MetadataQuerier --- + +func (p *PostgresProvider) QueryJobVariants(_ context.Context) (crtest.JobVariants, []error) { + variants := crtest.JobVariants{Variants: map[string][]string{}} + + var pairs []string + err := p.dbc.DB.Raw(`SELECT DISTINCT unnest(variants) AS pair FROM prow_jobs WHERE deleted_at IS NULL`). + Pluck("pair", &pairs).Error + if err != nil { + return variants, []error{fmt.Errorf("querying job variants: %w", err)} + } + + grouped := map[string]map[string]bool{} + for _, pair := range pairs { + k, v, ok := strings.Cut(pair, ":") + if !ok { + continue + } + if grouped[k] == nil { + grouped[k] = map[string]bool{} + } + grouped[k][v] = true + } + + for k, vals := range grouped { + sorted := make([]string, 0, len(vals)) + for v := range vals { + sorted = append(sorted, v) + } + sort.Strings(sorted) + variants.Variants[k] = sorted + } + return variants, nil +} + +// releaseMetadata holds hardcoded release info for known releases. +// This avoids needing a releases table — we derive release names from prow_jobs +// and fill in metadata from this map. +var releaseMetadata = map[string]struct { + previousRelease string + gaOffsetDays int // 0 = no GA date (in development) +}{ + "4.17": {previousRelease: "4.16", gaOffsetDays: -540}, + "4.18": {previousRelease: "4.17", gaOffsetDays: -395}, + "4.19": {previousRelease: "4.18", gaOffsetDays: -289}, + "4.20": {previousRelease: "4.19", gaOffsetDays: -163}, + "4.21": {previousRelease: "4.20", gaOffsetDays: -58}, + "4.22": {previousRelease: "4.21"}, + "5.0": {previousRelease: "4.22"}, +} + +func (p *PostgresProvider) QueryReleases(_ context.Context) ([]v1.Release, error) { + var releaseNames []string + err := p.dbc.DB.Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`). + Pluck("release", &releaseNames).Error + if err != nil { + return nil, fmt.Errorf("querying releases: %w", err) + } + + caps := map[v1.ReleaseCapability]bool{ + v1.ComponentReadinessCap: true, + v1.FeatureGatesCap: true, + v1.MetricsCap: true, + v1.PayloadTagsCap: true, + v1.SippyClassicCap: true, + } + + now := time.Now().UTC() + var releases []v1.Release + for _, name := range releaseNames { + rel := v1.Release{ + Release: name, + Capabilities: caps, + } + if meta, ok := releaseMetadata[name]; ok { + rel.PreviousRelease = meta.previousRelease + if meta.gaOffsetDays != 0 { + ga := now.AddDate(0, 0, meta.gaOffsetDays) + rel.GADate = &ga + } + } + releases = append(releases, rel) + } + return releases, nil +} + +func (p *PostgresProvider) QueryReleaseDates(_ context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { + // Derive time ranges from actual data in the DB rather than hardcoded GA dates. + // This ensures fallback queries find data where it actually exists. + type releaseRange struct { + Release string + Start time.Time + End time.Time + } + var ranges []releaseRange + err := p.dbc.DB.Raw(` + SELECT pj.release, + MIN(pjr.timestamp) AS start, + MAX(pjr.timestamp) AS end + FROM prow_job_runs pjr + JOIN prow_jobs pj ON pj.id = pjr.prow_job_id + WHERE pj.deleted_at IS NULL AND pjr.deleted_at IS NULL + GROUP BY pj.release + ORDER BY pj.release DESC + `).Scan(&ranges).Error + if err != nil { + return nil, []error{fmt.Errorf("querying release dates: %w", err)} + } + + var dates []crtest.ReleaseTimeRange + for _, r := range ranges { + start := r.Start + end := r.End + dates = append(dates, crtest.ReleaseTimeRange{ + Release: r.Release, + Start: &start, + End: &end, + }) + } + return dates, nil +} + +func (p *PostgresProvider) QueryUniqueVariantValues(_ context.Context, field string, nested bool) ([]string, error) { + if nested { + // Return all variant key names + var pairs []string + err := p.dbc.DB.Raw(` + SELECT DISTINCT unnest(variants) AS pair FROM prow_jobs + WHERE deleted_at IS NULL + `).Pluck("pair", &pairs).Error + if err != nil { + return nil, err + } + keys := map[string]bool{} + for _, pair := range pairs { + if k, _, ok := strings.Cut(pair, ":"); ok { + keys[k] = true + } + } + result := make([]string, 0, len(keys)) + for k := range keys { + result = append(result, k) + } + sort.Strings(result) + return result, nil + } + + // Map BQ column names to variant key names + fieldMap := map[string]string{ + "platform": "Platform", + "network": "Network", + "arch": "Architecture", + "upgrade": "Upgrade", + } + variantKey, ok := fieldMap[field] + if !ok { + return []string{}, nil + } + + var pairs []string + err := p.dbc.DB.Raw(` + SELECT DISTINCT unnest(variants) AS pair FROM prow_jobs + WHERE deleted_at IS NULL + `).Pluck("pair", &pairs).Error + if err != nil { + return nil, err + } + + vals := map[string]bool{} + for _, pair := range pairs { + if k, v, ok := strings.Cut(pair, ":"); ok && k == variantKey { + vals[v] = true + } + } + result := make([]string, 0, len(vals)) + for v := range vals { + result = append(result, v) + } + sort.Strings(result) + return result, nil +} + +// --- TestStatusQuerier --- + +// testStatusRow is the result of the aggregation query. +type testStatusRow struct { + TestID string `gorm:"column:test_id"` + TestName string `gorm:"column:test_name"` + TestSuite string `gorm:"column:test_suite"` + Component string `gorm:"column:component"` + Capabilities pq.StringArray `gorm:"column:capabilities;type:text[]"` + ProwJobID uint `gorm:"column:prow_job_id"` + TotalCount int `gorm:"column:total_count"` + SuccessCount int `gorm:"column:success_count"` + FlakeCount int `gorm:"column:flake_count"` + LastFailure *time.Time `gorm:"column:last_failure"` +} + +const testStatusQuery = ` +WITH deduped AS ( + SELECT DISTINCT ON (pjrt.prow_job_run_id, pjrt.test_id, pjrt.suite_id) + pjrt.test_id, pjrt.suite_id, pjrt.status, + pjr.timestamp, pj.id AS prow_job_id + FROM prow_job_run_tests pjrt + JOIN prow_job_runs pjr ON pjr.id = pjrt.prow_job_run_id + JOIN prow_jobs pj ON pj.id = pjr.prow_job_id + WHERE pj.release = ? + AND pjr.timestamp >= ? AND pjr.timestamp < ? + AND pjrt.deleted_at IS NULL AND pjr.deleted_at IS NULL AND pj.deleted_at IS NULL + AND (pjr.labels IS NULL OR NOT pjr.labels @> ARRAY['InfraFailure']) + ORDER BY pjrt.prow_job_run_id, pjrt.test_id, pjrt.suite_id, + CASE WHEN pjrt.status = 13 THEN 0 WHEN pjrt.status = 1 THEN 1 ELSE 2 END +) +SELECT + tow.unique_id AS test_id, + t.name AS test_name, + COALESCE(s.name, '') AS test_suite, + tow.component, + tow.capabilities, + d.prow_job_id, + COUNT(*) AS total_count, + SUM(CASE WHEN d.status IN (1, 13) THEN 1 ELSE 0 END) AS success_count, + SUM(CASE WHEN d.status = 13 THEN 1 ELSE 0 END) AS flake_count, + MAX(CASE WHEN d.status NOT IN (1, 13) THEN d.timestamp ELSE NULL END) AS last_failure +FROM deduped d +JOIN tests t ON t.id = d.test_id +JOIN test_ownerships tow ON tow.test_id = d.test_id + AND (tow.suite_id = d.suite_id OR (tow.suite_id IS NULL AND d.suite_id IS NULL)) +LEFT JOIN suites s ON s.id = d.suite_id +WHERE tow.staff_approved_obsolete = false +GROUP BY tow.unique_id, t.name, s.name, tow.component, tow.capabilities, d.prow_job_id +` + +func (p *PostgresProvider) queryTestStatus(ctx context.Context, release string, start, end time.Time, + _ crtest.JobVariants, includeVariants map[string][]string, + dbGroupBy map[string]bool) (map[string]crstatus.TestStatus, []error) { + + var rows []testStatusRow + if err := p.dbc.DB.WithContext(ctx).Raw(testStatusQuery, release, start, end).Scan(&rows).Error; err != nil { + return nil, []error{fmt.Errorf("querying test status: %w", err)} + } + + // Batch-fetch all ProwJob variants we need + jobVariantMap := p.fetchJobVariants(rows) + + result := map[string]crstatus.TestStatus{} + for _, row := range rows { + variants, ok := jobVariantMap[row.ProwJobID] + if !ok { + continue + } + + if !matchesIncludeVariants(variants, includeVariants) { + continue + } + + filtered := filterByDBGroupBy(variants, dbGroupBy) + key := crtest.KeyWithVariants{ + TestID: row.TestID, + Variants: filtered, + } + keyStr := key.KeyOrDie() + + existing, exists := result[keyStr] + if exists { + // Merge counts for same test+variant combo from different job runs + existing.Count.TotalCount += row.TotalCount + existing.Count.SuccessCount += row.SuccessCount + existing.Count.FlakeCount += row.FlakeCount + if row.LastFailure != nil && (existing.LastFailure.IsZero() || row.LastFailure.After(existing.LastFailure)) { + existing.LastFailure = *row.LastFailure + } + result[keyStr] = existing + } else { + ts := crstatus.TestStatus{ + TestName: row.TestName, + TestSuite: row.TestSuite, + Component: row.Component, + Capabilities: row.Capabilities, + Variants: variantMapToSlice(filtered), + Count: crtest.Count{ + TotalCount: row.TotalCount, + SuccessCount: row.SuccessCount, + FlakeCount: row.FlakeCount, + }, + } + if row.LastFailure != nil { + ts.LastFailure = *row.LastFailure + } + result[keyStr] = ts + } + } + + return result, nil +} + +// fetchJobVariants loads and caches ProwJob variant maps for the given rows. +func (p *PostgresProvider) fetchJobVariants(rows []testStatusRow) map[uint]map[string]string { + jobIDs := map[uint]bool{} + for _, r := range rows { + jobIDs[r.ProwJobID] = true + } + + ids := make([]uint, 0, len(jobIDs)) + for id := range jobIDs { + ids = append(ids, id) + } + + type jobRow struct { + ID uint `gorm:"column:id"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + + var jobRows []jobRow + if err := p.dbc.DB.Raw(`SELECT id, variants FROM prow_jobs WHERE id IN (?)`, ids).Scan(&jobRows).Error; err != nil { + log.WithError(err).Error("error fetching job variants") + return map[uint]map[string]string{} + } + + result := make(map[uint]map[string]string, len(jobRows)) + for _, jr := range jobRows { + result[jr.ID] = parseVariants(jr.Variants) + } + return result +} + +func (p *PostgresProvider) QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string]crstatus.TestStatus, []error) { + + dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) + for _, k := range reqOptions.VariantOption.DBGroupBy.List() { + dbGroupBy[k] = true + } + + includeVariants := reqOptions.VariantOption.IncludeVariants + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + return p.queryTestStatus( + ctx, + reqOptions.BaseRelease.Name, + reqOptions.BaseRelease.Start, + reqOptions.BaseRelease.End, + allJobVariants, + includeVariants, + dbGroupBy, + ) +} + +func (p *PostgresProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time) (map[string]crstatus.TestStatus, []error) { + + dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) + for _, k := range reqOptions.VariantOption.DBGroupBy.List() { + dbGroupBy[k] = true + } + + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + return p.queryTestStatus( + ctx, + reqOptions.SampleRelease.Name, + start, end, + allJobVariants, + includeVariants, + dbGroupBy, + ) +} + +// --- TestDetailsQuerier --- + +type testDetailRow struct { + TestID string `gorm:"column:test_id"` + TestName string `gorm:"column:test_name"` + ProwJobName string `gorm:"column:prowjob_name"` + ProwJobRunID string `gorm:"column:prowjob_run_id"` + ProwJobURL string `gorm:"column:prowjob_url"` + ProwJobStart time.Time `gorm:"column:prowjob_start"` + ProwJobID uint `gorm:"column:prow_job_id"` + Status int `gorm:"column:status"` + JiraComponent string `gorm:"column:jira_component"` + JiraComponentID *uint `gorm:"column:jira_component_id"` + Capabilities pq.StringArray `gorm:"column:capabilities;type:text[]"` +} + +const testDetailQuery = ` +SELECT + tow.unique_id AS test_id, + t.name AS test_name, + pj.name AS prowjob_name, + CAST(pjr.id AS TEXT) AS prowjob_run_id, + COALESCE(pjr.url, '') AS prowjob_url, + pjr.timestamp AS prowjob_start, + pj.id AS prow_job_id, + pjrt.status, + COALESCE(tow.jira_component, '') AS jira_component, + tow.jira_component_id, + tow.capabilities +FROM prow_job_run_tests pjrt +JOIN prow_job_runs pjr ON pjr.id = pjrt.prow_job_run_id +JOIN prow_jobs pj ON pj.id = pjr.prow_job_id +JOIN tests t ON t.id = pjrt.test_id +JOIN test_ownerships tow ON tow.test_id = pjrt.test_id + AND (tow.suite_id = pjrt.suite_id OR (tow.suite_id IS NULL AND pjrt.suite_id IS NULL)) +WHERE pj.release = ? + AND pjr.timestamp >= ? AND pjr.timestamp < ? + AND pjrt.deleted_at IS NULL AND pjr.deleted_at IS NULL AND pj.deleted_at IS NULL + AND tow.staff_approved_obsolete = false + AND (pjr.labels IS NULL OR NOT pjr.labels @> ARRAY['InfraFailure']) +ORDER BY pjr.timestamp +` + +func (p *PostgresProvider) queryTestDetails(release string, start, end time.Time, + reqOptions reqopts.RequestOptions, _ crtest.JobVariants, + includeVariants map[string][]string) (map[string][]crstatus.TestJobRunRows, []error) { + + var rows []testDetailRow + if err := p.dbc.DB.Raw(testDetailQuery, release, start, end).Scan(&rows).Error; err != nil { + return nil, []error{fmt.Errorf("querying test details: %w", err)} + } + + dbGroupBy := make(map[string]bool, reqOptions.VariantOption.DBGroupBy.Len()) + for _, k := range reqOptions.VariantOption.DBGroupBy.List() { + dbGroupBy[k] = true + } + + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + // Batch-fetch job variants + jobIDs := map[uint]bool{} + for _, r := range rows { + jobIDs[r.ProwJobID] = true + } + ids := make([]uint, 0, len(jobIDs)) + for id := range jobIDs { + ids = append(ids, id) + } + type jobRow struct { + ID uint `gorm:"column:id"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + var jobRows []jobRow + if len(ids) > 0 { + if err := p.dbc.DB.Raw(`SELECT id, variants FROM prow_jobs WHERE id IN (?)`, ids).Scan(&jobRows).Error; err != nil { + return nil, []error{fmt.Errorf("fetching job variants: %w", err)} + } + } + jobVariantMap := make(map[uint]map[string]string, len(jobRows)) + for _, jr := range jobRows { + jobVariantMap[jr.ID] = parseVariants(jr.Variants) + } + + // Filter test IDs if specified + // Build test ID filter and per-test requested variant filters + testIDFilter := map[string]bool{} + requestedVariantsByTestID := map[string]map[string]string{} + for _, tid := range reqOptions.TestIDOptions { + testIDFilter[tid.TestID] = true + if len(tid.RequestedVariants) > 0 { + requestedVariantsByTestID[tid.TestID] = tid.RequestedVariants + } + } + + result := map[string][]crstatus.TestJobRunRows{} + for _, row := range rows { + if len(testIDFilter) > 0 && !testIDFilter[row.TestID] { + continue + } + + variants, ok := jobVariantMap[row.ProwJobID] + if !ok { + continue + } + if !matchesIncludeVariants(variants, includeVariants) { + continue + } + + // Filter by requested variants (exact match for specific test+variant combo) + if rv, ok := requestedVariantsByTestID[row.TestID]; ok { + match := true + for k, v := range rv { + if variants[k] != v { + match = false + break + } + } + if !match { + continue + } + } + + filtered := filterByDBGroupBy(variants, dbGroupBy) + key := crtest.KeyWithVariants{ + TestID: row.TestID, + Variants: filtered, + } + + successCount := 0 + flakeCount := 0 + if row.Status == 1 || row.Status == 13 { + successCount = 1 + } + if row.Status == 13 { + flakeCount = 1 + } + + var jiraComponentID *big.Rat + if row.JiraComponentID != nil { + jiraComponentID = new(big.Rat).SetUint64(uint64(*row.JiraComponentID)) + } + + entry := crstatus.TestJobRunRows{ + TestKey: key, + TestKeyStr: key.KeyOrDie(), + TestName: row.TestName, + ProwJob: utils.NormalizeProwJobName(row.ProwJobName), + ProwJobRunID: row.ProwJobRunID, + ProwJobURL: row.ProwJobURL, + StartTime: row.ProwJobStart, + Count: crtest.Count{TotalCount: 1, SuccessCount: successCount, FlakeCount: flakeCount}, + JiraComponent: row.JiraComponent, + JiraComponentID: jiraComponentID, + } + + normalizedName := utils.NormalizeProwJobName(row.ProwJobName) + result[normalizedName] = append(result[normalizedName], entry) + } + + return result, nil +} + +func (p *PostgresProvider) QueryBaseJobRunTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants) (map[string][]crstatus.TestJobRunRows, []error) { + + return p.queryTestDetails( + reqOptions.BaseRelease.Name, + reqOptions.BaseRelease.Start, reqOptions.BaseRelease.End, + reqOptions, allJobVariants, reqOptions.VariantOption.IncludeVariants, + ) +} + +func (p *PostgresProvider) QuerySampleJobRunTestStatus(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + includeVariants map[string][]string, + start, end time.Time) (map[string][]crstatus.TestJobRunRows, []error) { + + return p.queryTestDetails( + reqOptions.SampleRelease.Name, + start, end, + reqOptions, allJobVariants, includeVariants, + ) +} + +// --- JobQuerier --- + +func (p *PostgresProvider) QueryJobRuns(_ context.Context, reqOptions reqopts.RequestOptions, + allJobVariants crtest.JobVariants, + release string, start, end time.Time) (map[string]dataprovider.JobRunStats, error) { + + type jobRunRow struct { + JobName string `gorm:"column:job_name"` + TotalRuns int `gorm:"column:total_runs"` + Successful int `gorm:"column:successful_runs"` + } + + var rows []jobRunRow + err := p.dbc.DB.Raw(` + SELECT + pj.name AS job_name, + COUNT(DISTINCT pjr.id) AS total_runs, + COUNT(DISTINCT CASE WHEN pjr.succeeded THEN pjr.id END) AS successful_runs + FROM prow_jobs pj + JOIN prow_job_runs pjr ON pjr.prow_job_id = pj.id + WHERE pj.release = ? + AND pjr.timestamp >= ? AND pjr.timestamp < ? + AND pj.deleted_at IS NULL AND pjr.deleted_at IS NULL + AND (pj.name LIKE 'periodic-%%' OR pj.name LIKE 'release-%%' OR pj.name LIKE 'aggregator-%%') + GROUP BY pj.name + ORDER BY pj.name + `, release, start, end).Scan(&rows).Error + if err != nil { + return nil, fmt.Errorf("querying job runs: %w", err) + } + + // Apply variant filtering in Go + includeVariants := reqOptions.VariantOption.IncludeVariants + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + // Fetch variants for all jobs + jobNames := make([]string, 0, len(rows)) + for _, r := range rows { + jobNames = append(jobNames, r.JobName) + } + jobVariantMap := map[string]map[string]string{} + if len(jobNames) > 0 { + type jvRow struct { + Name string `gorm:"column:name"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + var jvRows []jvRow + if err := p.dbc.DB.Raw(`SELECT name, variants FROM prow_jobs WHERE name IN (?) AND deleted_at IS NULL`, jobNames).Scan(&jvRows).Error; err != nil { + return nil, fmt.Errorf("fetching job variants: %w", err) + } + for _, jr := range jvRows { + jobVariantMap[jr.Name] = parseVariants(jr.Variants) + } + } + + results := map[string]dataprovider.JobRunStats{} + for _, row := range rows { + if variants, ok := jobVariantMap[row.JobName]; ok { + if !matchesIncludeVariants(variants, includeVariants) { + continue + } + } + passRate := 0.0 + if row.TotalRuns > 0 { + passRate = float64(row.Successful) / float64(row.TotalRuns) * 100 + } + results[row.JobName] = dataprovider.JobRunStats{ + JobName: row.JobName, + TotalRuns: row.TotalRuns, + SuccessfulRuns: row.Successful, + PassRate: passRate, + } + } + + return results, nil +} + +func (p *PostgresProvider) QueryJobVariantValues(_ context.Context, jobNames []string, + variantKeys []string) (map[string]map[string]string, error) { + + if len(jobNames) == 0 { + return map[string]map[string]string{}, nil + } + + type jvRow struct { + Name string `gorm:"column:name"` + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + + var rows []jvRow + if err := p.dbc.DB.Raw(`SELECT name, variants FROM prow_jobs WHERE name IN (?) AND deleted_at IS NULL`, jobNames).Scan(&rows).Error; err != nil { + return nil, fmt.Errorf("querying job variant values: %w", err) + } + + keyFilter := map[string]bool{} + for _, k := range variantKeys { + keyFilter[k] = true + } + + results := map[string]map[string]string{} + for _, row := range rows { + parsed := parseVariants(row.Variants) + if len(keyFilter) > 0 { + filtered := map[string]string{} + for k, v := range parsed { + if keyFilter[k] { + filtered[k] = v + } + } + results[row.Name] = filtered + } else { + results[row.Name] = parsed + } + } + return results, nil +} + +func (p *PostgresProvider) LookupJobVariants(_ context.Context, jobName string) (map[string]string, error) { + type jvRow struct { + Variants pq.StringArray `gorm:"column:variants;type:text[]"` + } + + var row jvRow + err := p.dbc.DB.Raw(`SELECT variants FROM prow_jobs WHERE name = ? AND deleted_at IS NULL LIMIT 1`, jobName).Scan(&row).Error + if err != nil { + return nil, fmt.Errorf("looking up job variants: %w", err) + } + return parseVariants(row.Variants), nil +} diff --git a/pkg/apis/api/componentreport/crtest/count_test.go b/pkg/apis/api/componentreport/crtest/count_test.go new file mode 100644 index 0000000000..def694331a --- /dev/null +++ b/pkg/apis/api/componentreport/crtest/count_test.go @@ -0,0 +1,308 @@ +package crtest + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCountFailures(t *testing.T) { + tests := []struct { + name string + count Count + expected int + }{ + { + name: "normal case", + count: Count{TotalCount: 10, SuccessCount: 7, FlakeCount: 1}, + expected: 2, + }, + { + name: "all success", + count: Count{TotalCount: 5, SuccessCount: 5, FlakeCount: 0}, + expected: 0, + }, + { + name: "all flakes", + count: Count{TotalCount: 5, SuccessCount: 0, FlakeCount: 5}, + expected: 0, + }, + { + name: "all failures", + count: Count{TotalCount: 5, SuccessCount: 0, FlakeCount: 0}, + expected: 5, + }, + { + name: "zero total", + count: Count{TotalCount: 0, SuccessCount: 0, FlakeCount: 0}, + expected: 0, + }, + { + name: "data inconsistency floors at zero", + count: Count{TotalCount: 5, SuccessCount: 3, FlakeCount: 3}, + expected: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.count.Failures()) + }) + } +} + +func TestCalculatePassRate(t *testing.T) { + tests := []struct { + name string + success int + failure int + flake int + treatFlakeAsFailure bool + expected float64 + }{ + { + name: "zero total without flake flag", + success: 0, + failure: 0, + flake: 0, + treatFlakeAsFailure: false, + expected: 0.0, + }, + { + name: "zero total with flake flag", + success: 0, + failure: 0, + flake: 0, + treatFlakeAsFailure: true, + expected: 0.0, + }, + { + name: "flakes as success", + success: 8, + failure: 0, + flake: 2, + treatFlakeAsFailure: false, + expected: 1.0, + }, + { + name: "flakes as failure", + success: 8, + failure: 0, + flake: 2, + treatFlakeAsFailure: true, + expected: 0.8, + }, + { + name: "mixed results flakes as success", + success: 6, + failure: 2, + flake: 2, + treatFlakeAsFailure: false, + expected: 0.8, + }, + { + name: "mixed results flakes as failure", + success: 6, + failure: 2, + flake: 2, + treatFlakeAsFailure: true, + expected: 0.6, + }, + { + name: "all failures", + success: 0, + failure: 10, + flake: 0, + treatFlakeAsFailure: false, + expected: 0.0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := CalculatePassRate(tt.success, tt.failure, tt.flake, tt.treatFlakeAsFailure) + assert.InDelta(t, tt.expected, result, 1e-9) + }) + } +} + +func TestCountAdd(t *testing.T) { + a := Count{TotalCount: 3, SuccessCount: 2, FlakeCount: 1} + b := Count{TotalCount: 7, SuccessCount: 4, FlakeCount: 2} + result := a.Add(b) + + assert.Equal(t, 10, result.TotalCount) + assert.Equal(t, 6, result.SuccessCount) + assert.Equal(t, 3, result.FlakeCount) + + // receiver should not be mutated (value receiver) + assert.Equal(t, 3, a.TotalCount) +} + +func TestCountToTestStats(t *testing.T) { + count := Count{TotalCount: 10, SuccessCount: 6, FlakeCount: 2} + + t.Run("flake as success", func(t *testing.T) { + stats := count.ToTestStats(false) + assert.Equal(t, 6, stats.SuccessCount) + assert.Equal(t, 2, stats.FailureCount) + assert.Equal(t, 2, stats.FlakeCount) + assert.InDelta(t, 0.8, stats.SuccessRate, 1e-9) // (6+2)/10 + }) + + t.Run("flake as failure", func(t *testing.T) { + stats := count.ToTestStats(true) + assert.Equal(t, 6, stats.SuccessCount) + assert.Equal(t, 2, stats.FailureCount) + assert.Equal(t, 2, stats.FlakeCount) + assert.InDelta(t, 0.6, stats.SuccessRate, 1e-9) // 6/10 + }) +} + +func TestStatsAdd(t *testing.T) { + a := NewTestStats(5, 3, 2, false) + b := NewTestStats(3, 1, 1, false) + + t.Run("recalculates rate on add", func(t *testing.T) { + result := a.Add(b, false) + assert.Equal(t, 8, result.SuccessCount) + assert.Equal(t, 4, result.FailureCount) + assert.Equal(t, 3, result.FlakeCount) + // (8+3)/15 = 11/15 + assert.InDelta(t, 11.0/15.0, result.SuccessRate, 1e-9) + }) + + t.Run("recalculates with flakes as failure", func(t *testing.T) { + result := a.Add(b, true) + assert.Equal(t, 8, result.SuccessCount) + assert.Equal(t, 4, result.FailureCount) + assert.Equal(t, 3, result.FlakeCount) + // 8/15 + assert.InDelta(t, 8.0/15.0, result.SuccessRate, 1e-9) + }) +} + +func TestStatsFailPassWithFlakes(t *testing.T) { + stats := NewTestStats(5, 3, 2, false) + + t.Run("flakes as failure", func(t *testing.T) { + fail, pass := stats.FailPassWithFlakes(true) + assert.Equal(t, 5, fail) // 3 failures + 2 flakes + assert.Equal(t, 5, pass) // 5 success only + }) + + t.Run("flakes as success", func(t *testing.T) { + fail, pass := stats.FailPassWithFlakes(false) + assert.Equal(t, 3, fail) // 3 failures only + assert.Equal(t, 7, pass) // 5 success + 2 flakes + }) +} + +func TestStatsAddTestCount(t *testing.T) { + stats := NewTestStats(5, 2, 1, false) + count := Count{TotalCount: 10, SuccessCount: 6, FlakeCount: 2} + // count.Failures() = 10 - 6 - 2 = 2 + + t.Run("flakes as success", func(t *testing.T) { + result := stats.AddTestCount(count, false) + assert.Equal(t, 11, result.SuccessCount) // 5 + 6 + assert.Equal(t, 4, result.FailureCount) // 2 + 2 + assert.Equal(t, 3, result.FlakeCount) // 1 + 2 + // (11+3)/18 + assert.InDelta(t, 14.0/18.0, result.SuccessRate, 1e-9) + }) + + t.Run("flakes as failure", func(t *testing.T) { + result := stats.AddTestCount(count, true) + assert.Equal(t, 11, result.SuccessCount) + assert.Equal(t, 4, result.FailureCount) + assert.Equal(t, 3, result.FlakeCount) + // 11/18 + assert.InDelta(t, 11.0/18.0, result.SuccessRate, 1e-9) + }) + + t.Run("with inconsistent count data", func(t *testing.T) { + bad := Count{TotalCount: 5, SuccessCount: 3, FlakeCount: 3} + // bad.Failures() should be 0, not negative + result := stats.AddTestCount(bad, false) + assert.Equal(t, 8, result.SuccessCount) // 5 + 3 + assert.Equal(t, 2, result.FailureCount) // 2 + 0 + assert.Equal(t, 4, result.FlakeCount) // 1 + 3 + }) +} + +func TestStatsPasses(t *testing.T) { + stats := NewTestStats(5, 3, 2, false) + + t.Run("without flakes as failure", func(t *testing.T) { + assert.Equal(t, 7, stats.Passes(false)) // 5 + 2 + }) + + t.Run("with flakes as failure", func(t *testing.T) { + assert.Equal(t, 5, stats.Passes(true)) // 5 only + }) +} + +func TestStatsTotal(t *testing.T) { + stats := NewTestStats(5, 3, 2, false) + assert.Equal(t, 10, stats.Total()) + + empty := NewTestStats(0, 0, 0, false) + assert.Equal(t, 0, empty.Total()) +} + +func TestStringForStatus(t *testing.T) { + tests := []struct { + name string + status Status + expected string + }{ + {"ExtremeRegression", ExtremeRegression, "Extreme"}, + {"SignificantRegression", SignificantRegression, "Significant"}, + {"ExtremeTriagedRegression", ExtremeTriagedRegression, "ExtremeTriaged"}, + {"SignificantTriagedRegression", SignificantTriagedRegression, "SignificantTriaged"}, + {"MissingSample", MissingSample, "MissingSample"}, + {"FixedRegression", FixedRegression, "Fixed"}, + {"FailedFixedRegression", FailedFixedRegression, "FailedFixed"}, + {"NotSignificant falls through to Unknown", NotSignificant, "Unknown"}, + {"MissingBasis falls through to Unknown", MissingBasis, "Unknown"}, + {"MissingBasisAndSample falls through to Unknown", MissingBasisAndSample, "Unknown"}, + {"SignificantImprovement falls through to Unknown", SignificantImprovement, "Unknown"}, + {"undefined status", Status(9999), "Unknown"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, StringForStatus(tt.status)) + }) + } +} + +func TestKeyWithVariantsKeyOrDie(t *testing.T) { + t.Run("stable serialization regardless of insertion order", func(t *testing.T) { + k1 := KeyWithVariants{ + TestID: "test-1", + Variants: map[string]string{"b": "2", "a": "1"}, + } + k2 := KeyWithVariants{ + TestID: "test-1", + Variants: map[string]string{"a": "1", "b": "2"}, + } + assert.Equal(t, k1.KeyOrDie(), k2.KeyOrDie()) + }) + + t.Run("different test IDs produce different keys", func(t *testing.T) { + k1 := KeyWithVariants{TestID: "test-1", Variants: map[string]string{"a": "1"}} + k2 := KeyWithVariants{TestID: "test-2", Variants: map[string]string{"a": "1"}} + assert.NotEqual(t, k1.KeyOrDie(), k2.KeyOrDie()) + }) + + t.Run("different variants produce different keys", func(t *testing.T) { + k1 := KeyWithVariants{TestID: "test-1", Variants: map[string]string{"a": "1"}} + k2 := KeyWithVariants{TestID: "test-1", Variants: map[string]string{"a": "2"}} + assert.NotEqual(t, k1.KeyOrDie(), k2.KeyOrDie()) + }) + + t.Run("nil variants", func(t *testing.T) { + k := KeyWithVariants{TestID: "test-1"} + result := k.KeyOrDie() + assert.Contains(t, result, "test-1") + }) +} diff --git a/pkg/apis/api/componentreport/reqopts/types_test.go b/pkg/apis/api/componentreport/reqopts/types_test.go new file mode 100644 index 0000000000..d8dccf17d4 --- /dev/null +++ b/pkg/apis/api/componentreport/reqopts/types_test.go @@ -0,0 +1,83 @@ +package reqopts + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAnyAreBaseOverrides(t *testing.T) { + tests := []struct { + name string + opts []TestIdentification + expected bool + }{ + { + name: "nil slice returns false", + opts: nil, + expected: false, + }, + { + name: "empty slice returns false", + opts: []TestIdentification{}, + expected: false, + }, + { + name: "single item with empty BaseOverrideRelease returns false", + opts: []TestIdentification{ + {TestID: "test-1"}, + }, + expected: false, + }, + { + name: "single item with BaseOverrideRelease set returns true", + opts: []TestIdentification{ + {TestID: "test-1", BaseOverrideRelease: "4.20"}, + }, + expected: true, + }, + { + name: "multiple items none have override returns false", + opts: []TestIdentification{ + {TestID: "test-1"}, + {TestID: "test-2"}, + {TestID: "test-3"}, + }, + expected: false, + }, + { + name: "multiple items only last has override returns true", + opts: []TestIdentification{ + {TestID: "test-1"}, + {TestID: "test-2"}, + {TestID: "test-3", BaseOverrideRelease: "4.20"}, + }, + expected: true, + }, + { + name: "multiple items first has override returns true", + opts: []TestIdentification{ + {TestID: "test-1", BaseOverrideRelease: "4.20"}, + {TestID: "test-2"}, + {TestID: "test-3"}, + }, + expected: true, + }, + { + name: "multiple items all have override returns true", + opts: []TestIdentification{ + {TestID: "test-1", BaseOverrideRelease: "4.19"}, + {TestID: "test-2", BaseOverrideRelease: "4.20"}, + {TestID: "test-3", BaseOverrideRelease: "4.21"}, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := AnyAreBaseOverrides(tt.opts) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/bigquery/bqlabel/labels_test.go b/pkg/bigquery/bqlabel/labels_test.go new file mode 100644 index 0000000000..4ab1e09940 --- /dev/null +++ b/pkg/bigquery/bqlabel/labels_test.go @@ -0,0 +1,84 @@ +package bqlabel + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeLabelValue(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "lowercase and replace spaces", + input: "Hello World", + expected: "hello_world", + }, + { + name: "already valid chars unchanged", + input: "test-value_123", + expected: "test-value_123", + }, + { + name: "empty string stays empty", + input: "", + expected: "", + }, + { + name: "truncate 64 chars to 63", + input: strings.Repeat("a", 64), + expected: strings.Repeat("a", 63), + }, + { + name: "exactly 63 chars unchanged", + input: strings.Repeat("b", 63), + expected: strings.Repeat("b", 63), + }, + { + name: "uppercase folded to lowercase", + input: "UPPER", + expected: "upper", + }, + { + name: "special chars replaced", + input: "foo@bar.com", + expected: "foo_bar_com", + }, + { + name: "non-latin chars replaced", + input: "日本語", + expected: "___", + }, + { + name: "slashes in URI paths replaced", + input: "/api/v1/test", + expected: "_api_v1_test", + }, + { + name: "dots in IP addresses replaced", + input: "192.168.1.1", + expected: "192_168_1_1", + }, + { + name: "spaces replaced with underscores", + input: "with spaces", + expected: "with_spaces", + }, + { + name: "mixed case with valid special chars", + input: "MiXeD-CaSe_123", + expected: "mixed-case_123", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := sanitizeLabelValue(tc.input) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/pkg/componentreadiness/resolvedissues/types_test.go b/pkg/componentreadiness/resolvedissues/types_test.go new file mode 100644 index 0000000000..f64180aad2 --- /dev/null +++ b/pkg/componentreadiness/resolvedissues/types_test.go @@ -0,0 +1,153 @@ +package resolvedissues + +import ( + "encoding/json" + "testing" + + "github.com/openshift/sippy/pkg/variantregistry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTriagedIssueKey_MarshalUnmarshalRoundTrip(t *testing.T) { + tests := []struct { + name string + key TriagedIssueKey + }{ + { + name: "basic key", + key: TriagedIssueKey{TestID: "openshift-tests.sig-auth", Variants: "aws-amd64-ovn-ha"}, + }, + { + name: "special chars in TestID", + key: TriagedIssueKey{TestID: "test:with-special_chars.v2", Variants: "aws-amd64"}, + }, + { + name: "empty Variants", + key: TriagedIssueKey{TestID: "some-test-id", Variants: ""}, + }, + { + name: "empty TestID", + key: TriagedIssueKey{TestID: "", Variants: "aws-amd64-ovn"}, + }, + { + name: "both fields empty", + key: TriagedIssueKey{TestID: "", Variants: ""}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + marshaled, err := tc.key.MarshalText() + require.NoError(t, err) + + var got TriagedIssueKey + err = got.UnmarshalText(marshaled) + require.NoError(t, err) + + assert.Equal(t, tc.key, got) + }) + } +} + +func TestTriagedIssueKey_DifferentKeysProdduceDifferentSerializations(t *testing.T) { + t.Run("differ only in Variants", func(t *testing.T) { + key1 := TriagedIssueKey{TestID: "same-test", Variants: "aws-amd64"} + key2 := TriagedIssueKey{TestID: "same-test", Variants: "gcp-arm64"} + + m1, err := key1.MarshalText() + require.NoError(t, err) + m2, err := key2.MarshalText() + require.NoError(t, err) + + assert.NotEqual(t, string(m1), string(m2)) + }) + + t.Run("differ only in TestID", func(t *testing.T) { + key1 := TriagedIssueKey{TestID: "test-alpha", Variants: "same-variant"} + key2 := TriagedIssueKey{TestID: "test-beta", Variants: "same-variant"} + + m1, err := key1.MarshalText() + require.NoError(t, err) + m2, err := key2.MarshalText() + require.NoError(t, err) + + assert.NotEqual(t, string(m1), string(m2)) + }) +} + +func TestTriagedIssueKey_JSONMapKey(t *testing.T) { + key := TriagedIssueKey{TestID: "test:with-special_chars.v2", Variants: "aws-amd64-ovn"} + + original := map[TriagedIssueKey]int{key: 42} + data, err := json.Marshal(original) + require.NoError(t, err) + + var restored map[TriagedIssueKey]int + err = json.Unmarshal(data, &restored) + require.NoError(t, err) + + val, ok := restored[key] + require.True(t, ok, "key should be present in restored map") + assert.Equal(t, 42, val) +} + +func TestTriagedIssueKey_JSONMapKeyMultipleEntries(t *testing.T) { + key1 := TriagedIssueKey{TestID: "test-a", Variants: "aws"} + key2 := TriagedIssueKey{TestID: "test-b", Variants: "gcp"} + + original := map[TriagedIssueKey]int{key1: 1, key2: 2} + data, err := json.Marshal(original) + require.NoError(t, err) + + var restored map[TriagedIssueKey]int + err = json.Unmarshal(data, &restored) + require.NoError(t, err) + + assert.Len(t, restored, 2) + assert.Equal(t, 1, restored[key1]) + assert.Equal(t, 2, restored[key2]) +} + +func TestBuildTriageMatchVariants(t *testing.T) { + t.Run("empty slice returns nil", func(t *testing.T) { + result := buildTriageMatchVariants([]string{}) + assert.Nil(t, result) + }) + + t.Run("single item", func(t *testing.T) { + result := buildTriageMatchVariants([]string{"Platform"}) + require.NotNil(t, result) + assert.Equal(t, 1, result.Len()) + assert.True(t, result.Has("Platform")) + }) + + t.Run("multiple items", func(t *testing.T) { + input := []string{"Platform", "Architecture", "Network"} + result := buildTriageMatchVariants(input) + require.NotNil(t, result) + assert.Equal(t, 3, result.Len()) + for _, v := range input { + assert.True(t, result.Has(v), "set should contain %s", v) + } + }) +} + +func TestTriageMatchVariants_ContainsExpectedVariants(t *testing.T) { + expected := []string{ + variantregistry.VariantPlatform, + variantregistry.VariantArch, + variantregistry.VariantNetwork, + variantregistry.VariantTopology, + variantregistry.VariantFeatureSet, + variantregistry.VariantUpgrade, + variantregistry.VariantSuite, + variantregistry.VariantInstaller, + } + + assert.Equal(t, 8, TriageMatchVariants.Len(), "TriageMatchVariants should contain exactly 8 variant names") + + for _, v := range expected { + assert.True(t, TriageMatchVariants.Has(v), "TriageMatchVariants should contain %s", v) + } +} diff --git a/pkg/util/param/param_test.go b/pkg/util/param/param_test.go new file mode 100644 index 0000000000..d028dc3c4d --- /dev/null +++ b/pkg/util/param/param_test.go @@ -0,0 +1,440 @@ +package param + +import ( + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCleanse(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "SQL injection characters stripped", + input: "'; DROP TABLE--", + expected: " DROP TABLE--", + }, + { + name: "spaces are allowed", + input: "hello world", + expected: "hello world", + }, + { + name: "all allowed characters pass through", + input: "test:value-1_2", + expected: "test:value-1_2", + }, + { + name: "unicode stripped entirely", + input: "名前", + expected: "", + }, + { + name: "empty stays empty", + input: "", + expected: "", + }, + { + name: "mixed allowed and disallowed", + input: "abc!@#123", + expected: "abc123", + }, + { + name: "only alphanumeric", + input: "SimpleTest123", + expected: "SimpleTest123", + }, + { + name: "tabs and newlines stripped", + input: "line1\tline2\nline3", + expected: "line1line2line3", + }, + { + name: "parentheses and brackets stripped", + input: "func(arg)[0]", + expected: "funcarg0", + }, + { + name: "percent and equals stripped", + input: "key=value%20encoded", + expected: "keyvalue20encoded", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := Cleanse(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestSafeRead(t *testing.T) { + tests := []struct { + name string + paramName string + value string + expected string + }{ + // release param: ^[\w.-]+$ + { + name: "release valid dotted version", + paramName: "release", + value: "4.16", + expected: "4.16", + }, + { + name: "release rejects SQL injection", + paramName: "release", + value: "4.16; DROP", + expected: "", + }, + { + name: "release valid word", + paramName: "release", + value: "Presubmit", + expected: "Presubmit", + }, + + // baseRelease param: ^\d+\.\d+$ + { + name: "baseRelease valid", + paramName: "baseRelease", + value: "4.16", + expected: "4.16", + }, + { + name: "baseRelease rejects triple dot", + paramName: "baseRelease", + value: "4.16.1", + expected: "", + }, + { + name: "baseRelease rejects v prefix", + paramName: "baseRelease", + value: "v4.16", + expected: "", + }, + + // prow_job_run_id param: ^\d+$ + { + name: "prow_job_run_id valid digits", + paramName: "prow_job_run_id", + value: "12345", + expected: "12345", + }, + { + name: "prow_job_run_id rejects alphanumeric", + paramName: "prow_job_run_id", + value: "123abc", + expected: "", + }, + + // test_id param: ^[\w:. -]+$ + { + name: "test_id valid with colon and hex", + paramName: "test_id", + value: "openshift-tests-upgrade:af8a62c5", + expected: "openshift-tests-upgrade:af8a62c5", + }, + { + name: "test_id valid with space", + paramName: "test_id", + value: "cluster install:0cb1bb27", + expected: "cluster install:0cb1bb27", + }, + + // view param: ^[-.\w]+$ + { + name: "view valid with dash", + paramName: "view", + value: "4.16-main", + expected: "4.16-main", + }, + { + name: "view rejects space", + paramName: "view", + value: "view name", + expected: "", + }, + + // include_success param: ^(true|false)$ + { + name: "include_success true", + paramName: "include_success", + value: "true", + expected: "true", + }, + { + name: "include_success false", + paramName: "include_success", + value: "false", + expected: "false", + }, + { + name: "include_success rejects uppercase True", + paramName: "include_success", + value: "True", + expected: "", + }, + + // empty value always returns "" + { + name: "empty value returns empty string", + paramName: "release", + value: "", + expected: "", + }, + + // prow_job_run_ids param: ^\d+(,\d+)*$ + { + name: "prow_job_run_ids single value", + paramName: "prow_job_run_ids", + value: "12345", + expected: "12345", + }, + { + name: "prow_job_run_ids comma separated", + paramName: "prow_job_run_ids", + value: "123,456,789", + expected: "123,456,789", + }, + { + name: "prow_job_run_ids rejects trailing comma", + paramName: "prow_job_run_ids", + value: "123,", + expected: "", + }, + + // start_date param: ^\d{4}-\d{2}-\d{2}$ + { + name: "start_date valid format", + paramName: "start_date", + value: "2024-01-15", + expected: "2024-01-15", + }, + { + name: "start_date rejects wrong format", + paramName: "start_date", + value: "01-15-2024", + expected: "", + }, + + // test param: ^.+$ (anything non-empty) + { + name: "test allows anything", + paramName: "test", + value: "[sig-cli] oc explain should work", + expected: "[sig-cli] oc explain should work", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := "http://example.com/api" + if tt.value != "" { + u += "?" + tt.paramName + "=" + url.QueryEscape(tt.value) + } + req := httptest.NewRequest("GET", u, nil) + result := SafeRead(req, tt.paramName) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestReadUint(t *testing.T) { + tests := []struct { + name string + paramValue string + limit int + expectedVal int + expectErr bool + }{ + { + name: "missing param returns zero and nil", + paramValue: "", + limit: 100, + expectedVal: 0, + expectErr: false, + }, + { + name: "valid value within limit", + paramValue: "42", + limit: 100, + expectedVal: 42, + expectErr: false, + }, + { + name: "value exceeds limit", + paramValue: "42", + limit: 10, + expectedVal: 0, + expectErr: true, + }, + { + name: "limit zero means no limit", + paramValue: "42", + limit: 0, + expectedVal: 42, + expectErr: false, + }, + { + name: "negative value rejected by regex", + paramValue: "-5", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + { + name: "non-numeric rejected", + paramValue: "abc", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + { + name: "value exactly at limit", + paramValue: "100", + limit: 100, + expectedVal: 100, + expectErr: false, + }, + { + name: "value one over limit", + paramValue: "101", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + { + name: "zero value", + paramValue: "0", + limit: 100, + expectedVal: 0, + expectErr: false, + }, + { + name: "decimal rejected by regex", + paramValue: "3.14", + limit: 100, + expectedVal: 0, + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := "http://example.com/api" + if tt.paramValue != "" { + u += "?count=" + url.QueryEscape(tt.paramValue) + } + req := httptest.NewRequest("GET", u, nil) + val, err := ReadUint(req, "count", tt.limit) + if tt.expectErr { + require.Error(t, err) + assert.Equal(t, 0, val) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedVal, val) + } + }) + } +} + +func TestReadBool(t *testing.T) { + tests := []struct { + name string + paramValue string + defaultValue bool + expectedVal bool + expectErr bool + }{ + { + name: "empty with default true", + paramValue: "", + defaultValue: true, + expectedVal: true, + expectErr: false, + }, + { + name: "empty with default false", + paramValue: "", + defaultValue: false, + expectedVal: false, + expectErr: false, + }, + { + name: "true value", + paramValue: "true", + defaultValue: false, + expectedVal: true, + expectErr: false, + }, + { + name: "false value", + paramValue: "false", + defaultValue: true, + expectedVal: false, + expectErr: false, + }, + { + name: "uppercase TRUE rejected", + paramValue: "TRUE", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "numeric 1 rejected", + paramValue: "1", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "yes rejected", + paramValue: "yes", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "mixed case True rejected", + paramValue: "True", + defaultValue: false, + expectedVal: false, + expectErr: true, + }, + { + name: "numeric 0 rejected", + paramValue: "0", + defaultValue: true, + expectedVal: false, + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := "http://example.com/api" + if tt.paramValue != "" { + u += "?flag=" + url.QueryEscape(tt.paramValue) + } + req := httptest.NewRequest("GET", u, nil) + val, err := ReadBool(req, "flag", tt.defaultValue) + if tt.expectErr { + require.Error(t, err) + assert.Equal(t, false, val) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectedVal, val) + } + }) + } +} diff --git a/pkg/util/sets/string_test.go b/pkg/util/sets/string_test.go new file mode 100644 index 0000000000..4522485314 --- /dev/null +++ b/pkg/util/sets/string_test.go @@ -0,0 +1,729 @@ +package sets + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewString(t *testing.T) { + tests := []struct { + name string + items []string + wantLen int + wantList []string + }{ + { + name: "empty", + items: nil, + wantLen: 0, + wantList: []string{}, + }, + { + name: "single item", + items: []string{"a"}, + wantLen: 1, + wantList: []string{"a"}, + }, + { + name: "multiple items", + items: []string{"c", "a", "b"}, + wantLen: 3, + wantList: []string{"a", "b", "c"}, + }, + { + name: "duplicates are collapsed", + items: []string{"a", "b", "a", "b", "a"}, + wantLen: 2, + wantList: []string{"a", "b"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.items...) + assert.Equal(t, tt.wantLen, s.Len()) + assert.Equal(t, tt.wantList, s.List()) + }) + } +} + +func TestInsert(t *testing.T) { + tests := []struct { + name string + initial []string + insert []string + wantLen int + wantItems []string + }{ + { + name: "insert into empty set", + initial: nil, + insert: []string{"x"}, + wantLen: 1, + wantItems: []string{"x"}, + }, + { + name: "insert new item", + initial: []string{"a"}, + insert: []string{"b"}, + wantLen: 2, + wantItems: []string{"a", "b"}, + }, + { + name: "insert duplicate does not increase length", + initial: []string{"a", "b"}, + insert: []string{"a"}, + wantLen: 2, + wantItems: []string{"a", "b"}, + }, + { + name: "insert multiple with overlap", + initial: []string{"a"}, + insert: []string{"a", "b", "c"}, + wantLen: 3, + wantItems: []string{"a", "b", "c"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + s.Insert(tt.insert...) + assert.Equal(t, tt.wantLen, s.Len()) + assert.Equal(t, tt.wantItems, s.List()) + }) + } +} + +func TestDelete(t *testing.T) { + tests := []struct { + name string + initial []string + delete []string + wantLen int + wantItems []string + }{ + { + name: "delete existing item", + initial: []string{"a", "b", "c"}, + delete: []string{"b"}, + wantLen: 2, + wantItems: []string{"a", "c"}, + }, + { + name: "delete non-existent item does not panic", + initial: []string{"a"}, + delete: []string{"z"}, + wantLen: 1, + wantItems: []string{"a"}, + }, + { + name: "delete from empty set does not panic", + initial: nil, + delete: []string{"x"}, + wantLen: 0, + wantItems: []string{}, + }, + { + name: "delete all items", + initial: []string{"a", "b"}, + delete: []string{"a", "b"}, + wantLen: 0, + wantItems: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + s.Delete(tt.delete...) + assert.Equal(t, tt.wantLen, s.Len()) + assert.Equal(t, tt.wantItems, s.List()) + }) + } +} + +func TestHas(t *testing.T) { + tests := []struct { + name string + initial []string + check string + want bool + }{ + { + name: "item present", + initial: []string{"a", "b"}, + check: "a", + want: true, + }, + { + name: "item absent", + initial: []string{"a", "b"}, + check: "c", + want: false, + }, + { + name: "empty set", + initial: nil, + check: "a", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + assert.Equal(t, tt.want, s.Has(tt.check)) + }) + } +} + +func TestHasAll(t *testing.T) { + tests := []struct { + name string + initial []string + check []string + want bool + }{ + { + name: "all present", + initial: []string{"a", "b", "c"}, + check: []string{"a", "c"}, + want: true, + }, + { + name: "some missing", + initial: []string{"a", "b"}, + check: []string{"a", "z"}, + want: false, + }, + { + name: "empty args returns true (vacuous truth)", + initial: []string{"a"}, + check: nil, + want: true, + }, + { + name: "empty set with empty args returns true", + initial: nil, + check: nil, + want: true, + }, + { + name: "empty set with non-empty args returns false", + initial: nil, + check: []string{"a"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + assert.Equal(t, tt.want, s.HasAll(tt.check...)) + }) + } +} + +func TestHasAny(t *testing.T) { + tests := []struct { + name string + initial []string + check []string + want bool + }{ + { + name: "one match", + initial: []string{"a", "b", "c"}, + check: []string{"z", "b"}, + want: true, + }, + { + name: "no match", + initial: []string{"a", "b"}, + check: []string{"x", "y"}, + want: false, + }, + { + name: "empty args returns false", + initial: []string{"a"}, + check: nil, + want: false, + }, + { + name: "empty set returns false", + initial: nil, + check: []string{"a"}, + want: false, + }, + { + name: "empty set with empty args returns false", + initial: nil, + check: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.initial...) + assert.Equal(t, tt.want, s.HasAny(tt.check...)) + }) + } +} + +func TestDifference(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want []string + }{ + { + name: "standard asymmetric: s1 minus s2", + s1: []string{"a", "b", "c"}, + s2: []string{"b", "c", "d"}, + want: []string{"a"}, + }, + { + name: "reverse direction: s2 minus s1", + s1: []string{"b", "c", "d"}, + s2: []string{"a", "b", "c"}, + want: []string{"d"}, + }, + { + name: "disjoint sets", + s1: []string{"a", "b"}, + s2: []string{"c", "d"}, + want: []string{"a", "b"}, + }, + { + name: "identical sets", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: []string{}, + }, + { + name: "empty s1", + s1: nil, + s2: []string{"a"}, + want: []string{}, + }, + { + name: "empty s2", + s1: []string{"a", "b"}, + s2: nil, + want: []string{"a", "b"}, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + result := s1.Difference(s2) + assert.Equal(t, tt.want, result.List()) + }) + } +} + +func TestDifferenceIsAsymmetric(t *testing.T) { + s1 := NewString("a", "b", "c") + s2 := NewString("b", "c", "d", "e") + + d1 := s1.Difference(s2) + d2 := s2.Difference(s1) + + assert.Equal(t, []string{"a"}, d1.List()) + assert.Equal(t, []string{"d", "e"}, d2.List()) + assert.False(t, d1.Equal(d2), "Difference must be asymmetric") +} + +func TestUnion(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want []string + }{ + { + name: "disjoint", + s1: []string{"a", "b"}, + s2: []string{"c", "d"}, + want: []string{"a", "b", "c", "d"}, + }, + { + name: "overlapping", + s1: []string{"a", "b", "c"}, + s2: []string{"b", "c", "d"}, + want: []string{"a", "b", "c", "d"}, + }, + { + name: "identical", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: []string{"a", "b"}, + }, + { + name: "empty s1", + s1: nil, + s2: []string{"a"}, + want: []string{"a"}, + }, + { + name: "empty s2", + s1: []string{"a"}, + s2: nil, + want: []string{"a"}, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + result := s1.Union(s2) + assert.Equal(t, tt.want, result.List()) + }) + } +} + +func TestUnionIsCommutative(t *testing.T) { + s1 := NewString("a", "b") + s2 := NewString("c", "d") + assert.True(t, s1.Union(s2).Equal(s2.Union(s1))) +} + +func TestIntersection(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want []string + }{ + { + name: "standard overlap", + s1: []string{"a", "b", "c"}, + s2: []string{"b", "c", "d"}, + want: []string{"b", "c"}, + }, + { + name: "disjoint", + s1: []string{"a", "b"}, + s2: []string{"c", "d"}, + want: []string{}, + }, + { + name: "identical", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: []string{"a", "b"}, + }, + { + name: "empty s1", + s1: nil, + s2: []string{"a"}, + want: []string{}, + }, + { + name: "empty s2", + s1: []string{"a"}, + s2: nil, + want: []string{}, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + result := s1.Intersection(s2) + assert.Equal(t, tt.want, result.List()) + }) + } +} + +func TestIntersectionOptimizationPath(t *testing.T) { + // The implementation walks the smaller set. Verify both code paths + // produce the same result regardless of which set is smaller. + small := NewString("b", "c") // len=2 + large := NewString("a", "b", "c", "d") // len=4 + + t.Run("s1 smaller than s2", func(t *testing.T) { + result := small.Intersection(large) + assert.Equal(t, []string{"b", "c"}, result.List()) + }) + + t.Run("s1 larger than s2", func(t *testing.T) { + result := large.Intersection(small) + assert.Equal(t, []string{"b", "c"}, result.List()) + }) + + t.Run("commutative", func(t *testing.T) { + assert.True(t, small.Intersection(large).Equal(large.Intersection(small))) + }) +} + +func TestIsSuperset(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want bool + }{ + { + name: "proper superset", + s1: []string{"a", "b", "c"}, + s2: []string{"a", "b"}, + want: true, + }, + { + name: "equal sets are supersets of each other", + s1: []string{"a", "b"}, + s2: []string{"a", "b"}, + want: true, + }, + { + name: "not a superset", + s1: []string{"a", "b"}, + s2: []string{"a", "c"}, + want: false, + }, + { + name: "empty set is superset of empty set", + s1: nil, + s2: nil, + want: true, + }, + { + name: "any set is superset of empty set", + s1: []string{"a"}, + s2: nil, + want: true, + }, + { + name: "empty set is not superset of non-empty set", + s1: nil, + s2: []string{"a"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + assert.Equal(t, tt.want, s1.IsSuperset(s2)) + }) + } +} + +func TestEqual(t *testing.T) { + tests := []struct { + name string + s1 []string + s2 []string + want bool + }{ + { + name: "identical contents", + s1: []string{"a", "b", "c"}, + s2: []string{"c", "b", "a"}, + want: true, + }, + { + name: "different sizes", + s1: []string{"a", "b"}, + s2: []string{"a"}, + want: false, + }, + { + name: "same length but different contents", + s1: []string{"a", "b"}, + s2: []string{"a", "c"}, + want: false, + }, + { + name: "both empty", + s1: nil, + s2: nil, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s1 := NewString(tt.s1...) + s2 := NewString(tt.s2...) + assert.Equal(t, tt.want, s1.Equal(s2)) + }) + } +} + +func TestList(t *testing.T) { + tests := []struct { + name string + items []string + want []string + }{ + { + name: "sorted output", + items: []string{"c", "a", "b"}, + want: []string{"a", "b", "c"}, + }, + { + name: "already sorted", + items: []string{"a", "b", "c"}, + want: []string{"a", "b", "c"}, + }, + { + name: "empty set returns empty slice", + items: nil, + want: []string{}, + }, + { + name: "single item", + items: []string{"x"}, + want: []string{"x"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.items...) + assert.Equal(t, tt.want, s.List()) + }) + } +} + +func TestListIsDeterministic(t *testing.T) { + s := NewString("z", "a", "m", "b", "y") + first := s.List() + for i := 0; i < 100; i++ { + assert.Equal(t, first, s.List(), "List() must return deterministic sorted order") + } +} + +func TestUnsortedList(t *testing.T) { + s := NewString("a", "b", "c") + result := s.UnsortedList() + assert.Len(t, result, 3) + assert.ElementsMatch(t, []string{"a", "b", "c"}, result) +} + +func TestPopAny(t *testing.T) { + t.Run("pop from non-empty set", func(t *testing.T) { + s := NewString("a", "b", "c") + val, ok := s.PopAny() + require.True(t, ok) + assert.NotEmpty(t, val) + assert.False(t, s.Has(val), "popped item must be removed from the set") + assert.Equal(t, 2, s.Len()) + }) + + t.Run("pop from empty set", func(t *testing.T) { + s := NewString() + val, ok := s.PopAny() + assert.False(t, ok) + assert.Equal(t, "", val) + }) + + t.Run("pop all items", func(t *testing.T) { + s := NewString("x", "y") + seen := map[string]bool{} + for s.Len() > 0 { + val, ok := s.PopAny() + require.True(t, ok) + seen[val] = true + } + assert.Equal(t, map[string]bool{"x": true, "y": true}, seen) + _, ok := s.PopAny() + assert.False(t, ok) + }) +} + +func TestLen(t *testing.T) { + tests := []struct { + name string + items []string + want int + }{ + {"empty", nil, 0}, + {"one", []string{"a"}, 1}, + {"three", []string{"a", "b", "c"}, 3}, + {"duplicates don't count", []string{"a", "a", "a"}, 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewString(tt.items...) + assert.Equal(t, tt.want, s.Len()) + }) + } +} + +func TestStringKeySet(t *testing.T) { + t.Run("from map[string]int", func(t *testing.T) { + m := map[string]int{"foo": 1, "bar": 2, "baz": 3} + s := StringKeySet(m) + assert.Equal(t, 3, s.Len()) + assert.Equal(t, []string{"bar", "baz", "foo"}, s.List()) + }) + + t.Run("from map[string]string", func(t *testing.T) { + m := map[string]string{"a": "x", "b": "y"} + s := StringKeySet(m) + assert.Equal(t, []string{"a", "b"}, s.List()) + }) + + t.Run("from empty map", func(t *testing.T) { + m := map[string]bool{} + s := StringKeySet(m) + assert.Equal(t, 0, s.Len()) + assert.Equal(t, []string{}, s.List()) + }) + + t.Run("panics with non-map argument", func(t *testing.T) { + assert.Panics(t, func() { + StringKeySet("not a map") + }) + }) + + t.Run("panics with slice argument", func(t *testing.T) { + assert.Panics(t, func() { + StringKeySet([]string{"a", "b"}) + }) + }) +} + +func TestInsertReturnsSelf(t *testing.T) { + s := NewString() + returned := s.Insert("a", "b") + assert.True(t, s.Equal(returned), "Insert should return the same set for chaining") +} + +func TestDeleteReturnsSelf(t *testing.T) { + s := NewString("a", "b") + returned := s.Delete("a") + assert.True(t, s.Equal(returned), "Delete should return the same set for chaining") +} diff --git a/scripts/e2e.sh b/scripts/e2e.sh index a97d141535..fb355d6d9a 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -11,14 +11,17 @@ PSQL_PORT="23433" REDIS_CONTAINER="sippy-e2e-test-redis" REDIS_PORT="23479" -if [[ -z "$GCS_SA_JSON_PATH" ]]; then - echo "Must provide path to GCS credential in GCS_SA_JSON_PATH env var" 1>&2 - exit 1 +if [ -z "$GCS_SA_JSON_PATH" ]; then + echo "WARNING: GCS_SA_JSON_PATH not set, data sync and BigQuery tests will be skipped" 1>&2 fi +E2E_EXIT_CODE=0 clean_up () { ARG=$? + if [ $ARG -ne 0 ]; then + E2E_EXIT_CODE=$ARG + fi echo "Stopping sippy API child process: $CHILD_PID" kill $CHILD_PID 2>/dev/null && wait $CHILD_PID 2>/dev/null # Generate coverage report from the server's coverage data @@ -27,10 +30,13 @@ clean_up () { go tool covdata percent -i="$COVDIR" go tool covdata textfmt -i="$COVDIR" -o=e2e-coverage.out # Merge test binary coverage (from -coverprofile) into server binary coverage - if [ -f e2e-test-coverage.out ]; then - echo "Merging test binary coverage into server coverage..." - tail -n +2 e2e-test-coverage.out >> e2e-coverage.out - fi + for f in e2e-test-coverage.out e2e-bq-test-coverage.out unit-test-coverage.out; do + if [ -f "$f" ]; then + echo "Merging $f into server coverage..." + tail -n +2 "$f" >> e2e-coverage.out + rm -f "$f" + fi + done echo "Coverage data written to e2e-coverage.out" echo "View HTML report: go tool cover -html=e2e-coverage.out -o=e2e-coverage.html" fi @@ -40,7 +46,23 @@ clean_up () { echo "Tearing down container $REDIS_CONTAINER" $DOCKER stop -i $REDIS_CONTAINER $DOCKER rm -i $REDIS_CONTAINER - exit $ARG + exit $E2E_EXIT_CODE +} + +wait_for_sippy() { + echo "Waiting for sippy API to start on port $SIPPY_API_PORT..." + TIMEOUT=600 + ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + if curl -s "http://localhost:$SIPPY_API_PORT/api/health" > /dev/null 2>&1; then + echo "Sippy API is ready after ${ELAPSED}s" + return 0 + fi + sleep 2 + ELAPSED=$((ELAPSED + 2)) + done + echo "Timeout waiting for sippy API to start after ${TIMEOUT}s" + return 1 } trap clean_up EXIT @@ -65,6 +87,7 @@ sleep 5 export SIPPY_E2E_DSN="postgresql://postgres:password@localhost:$PSQL_PORT/postgres" export REDIS_URL="redis://localhost:$REDIS_PORT" +export SIPPY_E2E_REPO_ROOT="$(pwd)" # Build with coverage instrumentation COVDIR="$(pwd)/e2e-coverage" @@ -76,8 +99,7 @@ go build -cover -coverpkg=./cmd/...,./pkg/... -mod vendor -o ./sippy ./cmd/sippy echo "Loading database..." GOCOVERDIR="$COVDIR" ./sippy seed-data \ --init-database \ - --database-dsn="$SIPPY_E2E_DSN" \ - --release="4.20" + --database-dsn="$SIPPY_E2E_DSN" # Spawn sippy server off into a separate process: export SIPPY_API_PORT="18080" @@ -91,29 +113,69 @@ GOCOVERDIR="$COVDIR" ./sippy serve \ --log-level debug \ --views config/e2e-views.yaml \ --google-service-account-credential-file $GCS_SA_JSON_PATH \ - --redis-url="$REDIS_URL" > e2e.log 2>&1 & + --redis-url="$REDIS_URL" \ + --data-provider postgres > e2e.log 2>&1 & CHILD_PID=$! -# Give it time to start up, and fill the redis cache -echo "Waiting for sippy API to start on port $SIPPY_API_PORT, see e2e.log for output..." -TIMEOUT=600 -ELAPSED=0 -while [ $ELAPSED -lt $TIMEOUT ]; do - if curl -s "http://localhost:$SIPPY_API_PORT/api/health" > /dev/null 2>&1; then - echo "Sippy API is ready after ${ELAPSED}s" - break - fi - sleep 2 - ELAPSED=$((ELAPSED + 2)) -done +wait_for_sippy || exit 1 -if [ $ELAPSED -ge $TIMEOUT ]; then - echo "Timeout waiting for sippy API to start after ${TIMEOUT}s" - exit 1 +# Prime the component readiness cache so triage tests can find cached reports +echo "Priming component readiness cache..." +VIEWS=$(curl -sf "http://localhost:$SIPPY_API_PORT/api/component_readiness/views") || { echo "Failed to fetch views"; exit 1; } +for VIEW in $(echo "$VIEWS" | jq -r '.[].name'); do + echo " Priming cache for view: $VIEW" + curl -sf "http://localhost:$SIPPY_API_PORT/api/component_readiness?view=$VIEW" > /dev/null || { echo "Failed to prime cache for view: $VIEW"; exit 1; } +done +echo "Cache priming complete" + +# Phase 1: Run postgres-backed tests +echo "=== Phase 1: Running postgres-backed e2e tests ===" +gotestsum \ + ./test/e2e/componentreadiness/postgres/... \ + ./test/e2e/componentreadiness/bugs/... \ + ./test/e2e/datasync/... \ + ./test/e2e/ \ + -count 1 -p 1 -coverprofile=e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... +POSTGRES_EXIT=$? +if [ $POSTGRES_EXIT -ne 0 ]; then + E2E_EXIT_CODE=$POSTGRES_EXIT fi +# Phase 2: Run BigQuery-backed tests (if credentials are available) +if [ -n "$GCS_SA_JSON_PATH" ]; then + echo "=== Phase 2: Running BigQuery-backed e2e tests ===" + echo "Stopping postgres-backed server..." + kill $CHILD_PID 2>/dev/null && wait $CHILD_PID 2>/dev/null -# Run our tests that request against the API, args ensure serially and fresh test code compile: -gotestsum ./test/e2e/... -count 1 -p 1 -coverprofile=e2e-test-coverage.out -coverpkg=./pkg/...,./cmd/... + GOCOVERDIR="$COVDIR" ./sippy serve \ + --listen ":$SIPPY_API_PORT" \ + --listen-metrics ":12112" \ + --database-dsn="$SIPPY_E2E_DSN" \ + --log-level debug \ + --views config/e2e-views.yaml \ + --google-service-account-credential-file $GCS_SA_JSON_PATH \ + --redis-url="$REDIS_URL" \ + --data-provider bigquery > e2e-bq.log 2>&1 & + CHILD_PID=$! + + wait_for_sippy || exit 1 + + gotestsum \ + ./test/e2e/componentreadiness/bigquery/... \ + -count 1 -p 1 -coverprofile=e2e-bq-test-coverage.out -coverpkg=./pkg/...,./cmd/... + BQ_EXIT=$? + if [ $BQ_EXIT -ne 0 ]; then + E2E_EXIT_CODE=$BQ_EXIT + fi +else + echo "=== Phase 2: Skipping BigQuery tests (GCS_SA_JSON_PATH not set) ===" +fi -# WARNING: do not place more commands here without addressing return code from go test not being overridden by the cleanup func +echo "=== Running unit tests for coverage ===" +gotestsum \ + ./pkg/... \ + -count 1 -coverprofile=unit-test-coverage.out -coverpkg=./pkg/...,./cmd/... +UNIT_EXIT=$? +if [ $UNIT_EXIT -ne 0 ]; then + E2E_EXIT_CODE=$UNIT_EXIT +fi diff --git a/test/e2e/componentreadiness/componentreadiness_test.go b/test/e2e/componentreadiness/bigquery/componentreadiness_test.go similarity index 87% rename from test/e2e/componentreadiness/componentreadiness_test.go rename to test/e2e/componentreadiness/bigquery/componentreadiness_test.go index 5deb044923..6bad2a5fec 100644 --- a/test/e2e/componentreadiness/componentreadiness_test.go +++ b/test/e2e/componentreadiness/bigquery/componentreadiness_test.go @@ -1,7 +1,8 @@ -package componentreadiness +package bigquery import ( "fmt" + "os" "testing" "github.com/openshift/sippy/pkg/apis/api/componentreport" @@ -12,6 +13,10 @@ import ( ) func TestComponentReadinessViews(t *testing.T) { + if os.Getenv("GCS_SA_JSON_PATH") == "" { + t.Skip("GCS_SA_JSON_PATH not set, skipping BigQuery tests") + } + var views []crview.View err := util.SippyGet("/api/component_readiness/views", &views) require.NoError(t, err, "error making http request") @@ -22,6 +27,5 @@ func TestComponentReadinessViews(t *testing.T) { var report componentreport.ComponentReport err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", views[0].Name), &report) require.NoError(t, err, "error making http request") - // We expect over 50 components at time of writing, asserting 25 should be safe assert.Greater(t, len(report.Rows), 25, "component report does not have rows we would expect") } diff --git a/test/e2e/componentreadiness/postgres/componentreadiness_test.go b/test/e2e/componentreadiness/postgres/componentreadiness_test.go new file mode 100644 index 0000000000..5ba45278e7 --- /dev/null +++ b/test/e2e/componentreadiness/postgres/componentreadiness_test.go @@ -0,0 +1,460 @@ +package postgres + +import ( + "fmt" + "net/url" + "testing" + "time" + + "github.com/openshift/sippy/pkg/apis/api/componentreport" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" + "github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails" + "github.com/openshift/sippy/pkg/db/models" + "github.com/openshift/sippy/test/e2e/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestComponentReadinessViews(t *testing.T) { + var views []crview.View + err := util.SippyGet("/api/component_readiness/views", &views) + require.NoError(t, err, "error making http request") + t.Logf("found %d views", len(views)) + require.Greater(t, len(views), 0, "no views returned, check server cli params") + + // Make a basic request for the first view and ensure we get some data back + var report componentreport.ComponentReport + err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", views[0].Name), &report) + require.NoError(t, err, "error making http request") + assert.Greater(t, len(report.Rows), 10, "component report does not have rows we would expect") +} + +func TestTestDetails(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // First get a report to find a regressed test + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &report) + require.NoError(t, err, "error getting component report") + require.NotEmpty(t, report.Rows, "report should have rows") + + // Find a regressed test to query details for + var testID, component, capability string + variants := map[string]string{} + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + testID = rt.TestID + component = rt.Component + capability = rt.Capability + variants = rt.ColumnIdentification.Variants + found = true + break + } + if found { + break + } + } + if found { + break + } + } + require.True(t, found, "should find at least one regressed test in the report") + + // Build query params for test_details + params := url.Values{} + params.Set("view", viewName) + params.Set("testId", testID) + params.Set("component", component) + params.Set("capability", capability) + for k, v := range variants { + params.Set(k, v) + } + + var details testdetails.Report + err = util.SippyGet(fmt.Sprintf("/api/component_readiness/test_details?%s", params.Encode()), &details) + require.NoError(t, err, "error getting test details") + + assert.Equal(t, testID, details.TestID, "response should match requested test ID") + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + for _, analysis := range details.Analyses { + total := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, total, 0, "sample stats should have run data") + } +} + +func TestTestDetailsWithFallback(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // Request test details for test-fallback-improves with testBasisRelease=4.20. + // This exercises the releasefallback middleware's QueryTestDetails path which + // queries base job run status for the override release. + params := url.Values{} + params.Set("view", viewName) + params.Set("testId", "test-fallback-improves") + params.Set("component", "comp-FallbackImproves") + params.Set("capability", "cap1") + params.Set("testBasisRelease", "4.20") + // All DBGroupBy variants must be specified for test details + params.Set("Architecture", "amd64") + params.Set("Platform", "aws") + params.Set("Network", "ovn") + params.Set("Topology", "ha") + params.Set("Installer", "ipi") + params.Set("FeatureSet", "default") + params.Set("Suite", "parallel") + params.Set("Upgrade", "none") + params.Set("LayeredProduct", "none") + + var details testdetails.Report + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/test_details?%s", params.Encode()), &details) + require.NoError(t, err, "error getting fallback test details") + + assert.Equal(t, "test-fallback-improves", details.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // The fallback path should produce analyses with sample data + for _, analysis := range details.Analyses { + sampleTotal := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, sampleTotal, 0, "sample stats should have run data") + } +} + +func TestTestDetailsForNewTestAPI(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // Request test details for test-new-test-pass-rate-fail which has no base data. + // This exercises the GenerateDetailsReportForTest path where BaseStats is nil. + params := url.Values{} + params.Set("view", viewName) + params.Set("testId", "test-new-test-pass-rate-fail") + params.Set("component", "comp-NewTestPassRate") + params.Set("capability", "cap1") + params.Set("Architecture", "amd64") + params.Set("Platform", "aws") + params.Set("Network", "ovn") + params.Set("Topology", "ha") + params.Set("Installer", "ipi") + params.Set("FeatureSet", "default") + params.Set("Suite", "parallel") + params.Set("Upgrade", "none") + params.Set("LayeredProduct", "none") + + var details testdetails.Report + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/test_details?%s", params.Encode()), &details) + require.NoError(t, err, "error getting new test details") + + assert.Equal(t, "test-new-test-pass-rate-fail", details.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + for _, analysis := range details.Analyses { + sampleTotal := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, sampleTotal, 0, "sample stats should have run data") + } +} + +func TestReportWithIncludeVariantsAPI(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&includeVariant=Platform:aws", viewName), &report) + require.NoError(t, err, "error getting filtered report") + require.NotEmpty(t, report.Rows, "filtered report should have rows") + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Equal(t, "aws", col.ColumnIdentification.Variants["Platform"], + "all columns should be aws when filtering by Platform:aws") + } + } +} + +func TestVariants(t *testing.T) { + // /api/component_readiness/variants returns CacheVariants (legacy BQ column names) + var variants map[string][]string + err := util.SippyGet("/api/component_readiness/variants", &variants) + require.NoError(t, err, "error getting variants") + assert.Contains(t, variants, "platform", "should have platform variant") + assert.Contains(t, variants, "network", "should have network variant") + assert.Contains(t, variants, "arch", "should have arch variant") + + // /api/job_variants returns JobVariants with all variant groups + var jobVariants crtest.JobVariants + err = util.SippyGet("/api/job_variants", &jobVariants) + require.NoError(t, err, "error getting job variants") + require.NotEmpty(t, jobVariants.Variants, "should have variants") + + expectedVariants := []string{"Platform", "Architecture", "Network", "Topology", "FeatureSet"} + for _, v := range expectedVariants { + assert.Contains(t, jobVariants.Variants, v, "should have %s variant", v) + assert.NotEmpty(t, jobVariants.Variants[v], "%s variant should have values", v) + } +} + +func TestRegressionByID(t *testing.T) { + // List regressions to find one we can query by ID + var regressions []models.TestRegression + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions?release=%s", util.Release), ®ressions) + require.NoError(t, err, "error listing regressions") + require.NotEmpty(t, regressions, "should have regressions from seed data") + + // Fetch one by ID + regression := regressions[0] + var fetched models.TestRegression + err = util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions/%d", regression.ID), &fetched) + require.NoError(t, err, "error getting regression by ID") + + assert.Equal(t, regression.ID, fetched.ID) + assert.Equal(t, regression.TestID, fetched.TestID) + assert.Equal(t, regression.TestName, fetched.TestName) + assert.Equal(t, regression.Release, fetched.Release) + + // HATEOAS links + assert.NotNil(t, fetched.Links, "regression should have HATEOAS links") + assert.Contains(t, fetched.Links, "test_details", "should have test_details link") +} + +func TestRegressionByIDNotFound(t *testing.T) { + var fetched models.TestRegression + err := util.SippyGet("/api/component_readiness/regressions/999999", &fetched) + require.Error(t, err, "should return error for non-existent regression") +} + +func TestColumnGroupByAndDBGroupBy(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + t.Run("default grouping from view", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + // Default view has columnGroupBy: Network, Platform, Topology + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Contains(t, col.ColumnIdentification.Variants, "Topology") + } + } + }) + + t.Run("override columnGroupBy to Platform only", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&columnGroupBy=Platform", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform", + "columns should still have Platform") + assert.Equal(t, 1, len(col.ColumnIdentification.Variants), + "columns should only have Platform when columnGroupBy is overridden to Platform") + } + } + }) + + t.Run("override columnGroupBy to Platform,Network", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&columnGroupBy=Platform,Network", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Equal(t, 2, len(col.ColumnIdentification.Variants), + "columns should have exactly Platform and Network") + } + } + }) + + t.Run("override dbGroupBy reduces aggregation", func(t *testing.T) { + // With fewer dbGroupBy dimensions, we should get fewer rows because tests + // are aggregated at a coarser level + var defaultReport componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &defaultReport) + require.NoError(t, err) + + // Use a minimal dbGroupBy — just Platform, Network, Topology (must include columnGroupBy) + var reducedReport componentreport.ComponentReport + err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&dbGroupBy=Platform,Network,Topology", viewName), &reducedReport) + require.NoError(t, err) + require.NotEmpty(t, reducedReport.Rows) + + // The reduced dbGroupBy should produce a valid report. We can't strictly assert + // fewer rows since it depends on the data, but it should still work without error. + t.Logf("default report: %d rows, reduced dbGroupBy: %d rows", len(defaultReport.Rows), len(reducedReport.Rows)) + }) + + t.Run("override both columnGroupBy and dbGroupBy together", func(t *testing.T) { + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s&columnGroupBy=Platform&dbGroupBy=Platform,Architecture,Network,Topology", viewName), &report) + require.NoError(t, err) + require.NotEmpty(t, report.Rows) + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Equal(t, 1, len(col.ColumnIdentification.Variants), + "columns should only have Platform when columnGroupBy is Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + } + } + }) +} + +func TestComponentReadinessWithExplicitParams(t *testing.T) { + // Test using explicit date/release params instead of a view. + // This exercises parseDateRange and parseAdvancedOptions. + now := time.Now().UTC().Truncate(time.Hour) + params := url.Values{} + params.Set("baseRelease", "4.21") + params.Set("sampleRelease", "4.22") + params.Set("baseStartTime", now.Add(-60*24*time.Hour).Format(time.RFC3339)) + params.Set("baseEndTime", now.Add(-30*24*time.Hour).Format(time.RFC3339)) + params.Set("sampleStartTime", now.Add(-3*24*time.Hour).Format(time.RFC3339)) + params.Set("sampleEndTime", now.Format(time.RFC3339)) + params.Set("confidence", "95") + params.Set("minFail", "3") + params.Set("pity", "5") + params.Set("passRateNewTests", "90") + params.Set("columnGroupBy", "Network,Platform,Topology") + params.Set("dbGroupBy", "Architecture,FeatureSet,Installer,Network,Platform,Suite,Topology,Upgrade,LayeredProduct") + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?%s", params.Encode()), &report) + require.NoError(t, err, "explicit params request should succeed") + assert.NotEmpty(t, report.Rows, "report with explicit params should have rows") + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Contains(t, col.ColumnIdentification.Variants, "Topology") + } + } +} + +func TestVariantCrossCompare(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + t.Run("cross-compare Platform with specific value", func(t *testing.T) { + // The default view includes Platform:aws,gcp. Cross-comparing Platform + // with a specific value means the sample uses only that Platform value. + params := url.Values{} + params.Set("view", viewName) + params.Add("variantCrossCompare", "Platform") + params.Add("compareVariant", "Platform:gcp") + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?%s", params.Encode()), &report) + require.NoError(t, err, "cross-compare request should succeed") + t.Logf("cross-compare Platform:gcp report: %d rows", len(report.Rows)) + + // Columns should still respect the columnGroupBy from the view + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + } + } + }) + + t.Run("cross-compare without compareVariant values", func(t *testing.T) { + // Cross-compare a group without specifying compareVariant values means + // no restriction on that variant in the sample + params := url.Values{} + params.Set("view", viewName) + params.Add("variantCrossCompare", "Platform") + + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?%s", params.Encode()), &report) + require.NoError(t, err, "cross-compare without compareVariant should succeed") + t.Logf("cross-compare (no compareVariant) report: %d rows", len(report.Rows)) + }) +} + +func TestTriageAffectsReportStatus(t *testing.T) { + viewName := fmt.Sprintf("%s-main", util.Release) + + // Get a report and find a regressed test + var report componentreport.ComponentReport + err := util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &report) + require.NoError(t, err) + + var regressionID uint + var originalStatus crtest.Status + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.ReportStatus == crtest.SignificantRegression || rt.ReportStatus == crtest.ExtremeRegression { + if rt.Regression != nil { + regressionID = rt.Regression.ID + originalStatus = rt.ReportStatus + found = true + } + } + if found { + break + } + } + if found { + break + } + } + if found { + break + } + } + require.True(t, found, "should find a regressed test with a regression ID") + t.Logf("found regression ID %d with status %d", regressionID, originalStatus) + + // Create a triage for this regression + triage := models.Triage{ + URL: "https://issues.redhat.com/browse/OCPBUGS-99999", + Type: models.TriageTypeProduct, + Regressions: []models.TestRegression{ + {ID: regressionID}, + }, + } + var triageResponse models.Triage + err = util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) + require.NoError(t, err, "creating triage should succeed") + t.Logf("created triage ID %d", triageResponse.ID) + + // Clean up triage when done + defer func() { + _ = util.SippyDelete(fmt.Sprintf("/api/component_readiness/triages/%d", triageResponse.ID)) + }() + + // Re-fetch the report — PostAnalysis runs outside the cache, so triaged status should be reflected + var updatedReport componentreport.ComponentReport + err = util.SippyGet(fmt.Sprintf("/api/component_readiness?view=%s", viewName), &updatedReport) + require.NoError(t, err) + + // Find the same regression and verify its status changed to triaged + foundTriaged := false + for _, row := range updatedReport.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.Regression == nil || rt.Regression.ID != regressionID { + continue + } + expectedStatus := crtest.SignificantTriagedRegression + if originalStatus == crtest.ExtremeRegression { + expectedStatus = crtest.ExtremeTriagedRegression + } + assert.Equal(t, expectedStatus, rt.ReportStatus, + "regression should have triaged status after triage creation") + assert.NotEmpty(t, rt.Explanations, "triaged test should have explanations") + foundTriaged = true + } + } + } + assert.True(t, foundTriaged, "should find the triaged regression in the updated report") +} diff --git a/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go b/test/e2e/componentreadiness/postgres/regressiontracker/regressiontracker_test.go similarity index 78% rename from test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go rename to test/e2e/componentreadiness/postgres/regressiontracker/regressiontracker_test.go index 809e226227..640018e876 100644 --- a/test/e2e/componentreadiness/regressiontracker/regressiontracker_test.go +++ b/test/e2e/componentreadiness/postgres/regressiontracker/regressiontracker_test.go @@ -21,11 +21,19 @@ import ( "github.com/stretchr/testify/require" ) -func cleanupAllRegressions(dbc *db.DB) { - // Delete all test regressions in the e2e postgres db. - res := dbc.DB.Where("1 = 1").Delete(&models.TestRegression{}) - if res.Error != nil { - log.Errorf("error deleting test regressions: %v", res.Error) +// cleanupRegressions deletes only the specified test regressions and their +// associated triage links from the e2e postgres db. Tests should clean up +// only what they create to avoid destroying seed data regressions. +func cleanupRegressions(dbc *db.DB, regressions ...*models.TestRegression) { + for _, r := range regressions { + if r == nil { + continue + } + dbc.DB.Exec("DELETE FROM triage_regressions WHERE test_regression_id = ?", r.ID) + res := dbc.DB.Delete(r) + if res.Error != nil { + log.Errorf("error deleting test regression %d: %v", r.ID, res.Error) + } } } @@ -36,7 +44,7 @@ func Test_RegressionTracker(t *testing.T) { newRegression := componentreport.ReportTestSummary{ TestComparison: testdetails.TestComparison{ BaseStats: &testdetails.ReleaseStats{ - Release: "4.18", + Release: "4.20", }, }, Identification: crtest.Identification{ @@ -56,27 +64,27 @@ func Test_RegressionTracker(t *testing.T) { }, } view := crview.View{ - Name: "4.19-main", + Name: "4.22-main", SampleRelease: reqopts.RelativeRelease{ Release: reqopts.Release{ - Name: "4.19", + Name: "4.22", }, }, } t.Run("open a new regression", func(t *testing.T) { - defer cleanupAllRegressions(dbc) tr, err := tracker.OpenRegression(view, newRegression) + defer cleanupRegressions(dbc, tr) require.NoError(t, err) - assert.Equal(t, "4.19", tr.Release) - assert.Equal(t, "4.18", tr.BaseRelease, "BaseRelease should be set from BaseStats.Release") + assert.Equal(t, "4.22", tr.Release) + assert.Equal(t, "4.20", tr.BaseRelease, "BaseRelease should be set from BaseStats.Release") assert.ElementsMatch(t, pq.StringArray([]string{"a:b", "c:d"}), tr.Variants) assert.True(t, tr.ID > 0) }) t.Run("close and reopen a regression", func(t *testing.T) { - defer cleanupAllRegressions(dbc) tr, err := tracker.OpenRegression(view, newRegression) + defer cleanupRegressions(dbc, tr) require.NoError(t, err) // look it up just to be sure: @@ -105,63 +113,75 @@ func Test_RegressionTracker(t *testing.T) { }) t.Run("list current regressions for release", func(t *testing.T) { - defer cleanupAllRegressions(dbc) var err error - open419, err := rawCreateRegression(dbc, "4.19", + open422, err := rawCreateRegression(dbc, "4.22", "test1ID", "test 1", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Time{}) require.NoError(t, err) - recentlyClosed419, err := rawCreateRegression(dbc, "4.19", + recentlyClosed422, err := rawCreateRegression(dbc, "4.22", "test2ID", "test 2", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Now().Add(-2*24*time.Hour)) require.NoError(t, err) - _, err = rawCreateRegression(dbc, "4.19", + oldClosed422, err := rawCreateRegression(dbc, "4.22", "test3ID", "test 3", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Now().Add(-70*24*time.Hour)) require.NoError(t, err) - _, err = rawCreateRegression(dbc, "4.18", + other421, err := rawCreateRegression(dbc, "4.21", "test1ID", "test 1", []string{"a:b", "c:d"}, time.Now().Add(-77*24*time.Hour), time.Time{}) require.NoError(t, err) + defer cleanupRegressions(dbc, open422, recentlyClosed422, oldClosed422, other421) - // List all regressions for 4.19, should exclude 4.18, include recently closed regressions, + // List all regressions for 4.22, should exclude 4.21, include recently closed regressions, // and exclude older closed regressions. - relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.19") + relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.22") require.NoError(t, err) - assert.Equal(t, 2, len(relRegressions)) + + // Verify our expected regressions are present (open + recently closed) + foundIDs := make(map[uint]bool) for _, rel := range relRegressions { - assert.True(t, rel.ID == open419.ID || rel.ID == recentlyClosed419.ID, - "unexpected regression was returned: %+v", *rel) + foundIDs[rel.ID] = true } + assert.True(t, foundIDs[open422.ID], "open regression should be in list") + assert.True(t, foundIDs[recentlyClosed422.ID], "recently closed regression should be in list") + assert.False(t, foundIDs[oldClosed422.ID], "old closed regression should not be in list") + assert.False(t, foundIDs[other421.ID], "4.21 regression should not be in list") }) t.Run("list returns regressions with BaseRelease set", func(t *testing.T) { - defer cleanupAllRegressions(dbc) - tr, err := rawCreateRegressionWithBase(dbc, "4.19", "4.18", "baseTestID", "base test", + tr, err := rawCreateRegressionWithBase(dbc, "4.22", "4.21", "baseTestID", "base test", []string{"a:b"}, time.Now().Add(-1*24*time.Hour), time.Time{}) require.NoError(t, err) - relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.19") + defer cleanupRegressions(dbc, tr) + relRegressions, err := tracker.ListCurrentRegressionsForRelease("4.22") require.NoError(t, err) - require.Len(t, relRegressions, 1) - assert.Equal(t, tr.ID, relRegressions[0].ID) - assert.Equal(t, "4.18", relRegressions[0].BaseRelease, "ListCurrentRegressionsForRelease should return BaseRelease") + + // Find our regression in the list and verify BaseRelease is set + var found *models.TestRegression + for _, rel := range relRegressions { + if rel.ID == tr.ID { + found = rel + break + } + } + require.NotNil(t, found, "created regression should be in list") + assert.Equal(t, "4.21", found.BaseRelease, "ListCurrentRegressionsForRelease should return BaseRelease") }) t.Run("closing a regression should resolve associated triages that have no other active regressions", func(t *testing.T) { - defer cleanupAllRegressions(dbc) - regressionToClose, err := tracker.OpenRegression(view, newRegression) require.NoError(t, err) + defer cleanupRegressions(dbc, regressionToClose) // Create a second regression that will remain open secondRegression := componentreport.ReportTestSummary{ TestComparison: testdetails.TestComparison{ BaseStats: &testdetails.ReleaseStats{ - Release: "4.18", + Release: "4.20", }, }, Identification: crtest.Identification{ @@ -182,6 +202,7 @@ func Test_RegressionTracker(t *testing.T) { } regressionToRemainOpened, err := tracker.OpenRegression(view, secondRegression) require.NoError(t, err) + defer cleanupRegressions(dbc, regressionToRemainOpened) // Create first triage associated only with the first regression triage := models.Triage{ @@ -195,6 +216,10 @@ func Test_RegressionTracker(t *testing.T) { dbWithContext := dbc.DB.WithContext(context.WithValue(context.Background(), models.CurrentUserKey, "e2e-test")) res := dbWithContext.Create(&triage) require.NoError(t, res.Error) + defer func() { + dbc.DB.Exec("DELETE FROM triage_regressions WHERE triage_id = ?", triage.ID) + dbc.DB.Delete(&triage) + }() // Create second triage associated with both regressions triage2 := models.Triage{ @@ -208,6 +233,10 @@ func Test_RegressionTracker(t *testing.T) { } res = dbWithContext.Create(&triage2) require.NoError(t, res.Error) + defer func() { + dbc.DB.Exec("DELETE FROM triage_regressions WHERE triage_id = ?", triage2.ID) + dbc.DB.Delete(&triage2) + }() // Close the regression with a time of NOW, this should not result in resolved triage regressionToClose.Closed = sql.NullTime{Valid: true, Time: time.Now()} diff --git a/test/e2e/componentreadiness/postgres/report/report_test.go b/test/e2e/componentreadiness/postgres/report/report_test.go new file mode 100644 index 0000000000..1eaf8e43bd --- /dev/null +++ b/test/e2e/componentreadiness/postgres/report/report_test.go @@ -0,0 +1,452 @@ +package report + +import ( + "context" + "strings" + "testing" + "time" + + componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" + pgprovider "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" + "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" + "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" + "github.com/openshift/sippy/pkg/apis/cache" + "github.com/openshift/sippy/pkg/util/sets" + "github.com/openshift/sippy/test/e2e/util" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupProvider(t *testing.T) (*pgprovider.PostgresProvider, reqopts.RequestOptions) { + dbc := util.CreateE2EPostgresConnection(t) + provider := pgprovider.NewPostgresProvider(dbc, nil) + + now := time.Now().UTC().Truncate(time.Hour) + + reqOptions := reqopts.RequestOptions{ + BaseRelease: reqopts.Release{ + Name: "4.21", + Start: now.Add(-60 * 24 * time.Hour), + End: now.Add(-30 * 24 * time.Hour), + }, + SampleRelease: reqopts.Release{ + Name: "4.22", + Start: now.Add(-3 * 24 * time.Hour), + End: now, + }, + VariantOption: reqopts.Variants{ + ColumnGroupBy: sets.NewString("Network", "Platform", "Topology"), + DBGroupBy: sets.NewString("Architecture", "FeatureSet", "Installer", "Network", "Platform", "Suite", "Topology", "Upgrade", "LayeredProduct"), + }, + AdvancedOption: reqopts.Advanced{ + Confidence: 95, + PityFactor: 5, + MinimumFailure: 3, + PassRateRequiredNewTests: 90, + IncludeMultiReleaseAnalysis: true, + }, + CacheOption: cache.RequestOptions{}, + } + + return provider, reqOptions +} + +func TestReportStatuses(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs, "GetComponentReport returned errors: %v", errs) + require.NotEmpty(t, report.Rows, "report should have rows") + + // Collect regressed test statuses keyed by testID+column platform. + type cellKey struct { + testID string + platform string + } + regressedStatuses := map[cellKey]crtest.Status{} + + for _, row := range report.Rows { + for _, col := range row.Columns { + if col.Status <= crtest.SignificantRegression { + for _, rt := range col.RegressedTests { + key := cellKey{testID: rt.TestID, platform: col.ColumnIdentification.Variants["Platform"]} + if existing, ok := regressedStatuses[key]; !ok || rt.ReportStatus < existing { + regressedStatuses[key] = rt.ReportStatus + } + } + } + } + } + + // Verify structural integrity + for _, row := range report.Rows { + assert.NotEmpty(t, row.RowIdentification.Component, "every row should have a component") + for _, col := range row.Columns { + assert.NotEmpty(t, col.ColumnIdentification.Variants, "column should have variants") + assert.Contains(t, col.ColumnIdentification.Variants, "Platform") + assert.Contains(t, col.ColumnIdentification.Variants, "Network") + assert.Contains(t, col.ColumnIdentification.Variants, "Topology") + } + } + + // Collect all cell statuses across the grid + statusCounts := map[crtest.Status]int{} + for _, row := range report.Rows { + for _, col := range row.Columns { + statusCounts[col.Status]++ + } + } + + t.Logf("Status distribution: %v", statusCounts) + + assert.Contains(t, statusCounts, crtest.NotSignificant, "should have NotSignificant cells") + assert.Contains(t, statusCounts, crtest.MissingSample, "should have MissingSample cells") + + hasRegression := statusCounts[crtest.SignificantRegression] > 0 || statusCounts[crtest.ExtremeRegression] > 0 + assert.True(t, hasRegression, "should have at least one regression cell") + + // Verify specific regressed test statuses per platform + assert.Equal(t, crtest.ExtremeRegression, regressedStatuses[cellKey{"test-extreme-regression", "aws"}], + "extreme regression on aws should have ExtremeRegression status") + assert.Equal(t, crtest.SignificantRegression, regressedStatuses[cellKey{"test-significant-regression", "aws"}], + "significant regression on aws should have SignificantRegression status") +} + +func TestTestDetails(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + releases, err := provider.QueryReleases(ctx) + require.NoError(t, err) + + // Generate report to find a regressed test + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + var testID reqopts.TestIdentification + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + testID = reqopts.TestIdentification{ + Component: rt.RowIdentification.Component, + Capability: rt.RowIdentification.Capability, + TestID: rt.RowIdentification.TestID, + RequestedVariants: rt.ColumnIdentification.Variants, + } + found = true + break + } + if found { + break + } + } + if found { + break + } + } + require.True(t, found, "should find at least one regressed test") + + detailReqOpts := reqOptions + detailReqOpts.TestIDOptions = []reqopts.TestIdentification{testID} + + details, detailErrs := componentreadiness.GetTestDetails(ctx, provider, nil, detailReqOpts, releases, "") + require.Empty(t, detailErrs, "GetTestDetails returned errors: %v", detailErrs) + + assert.Equal(t, testID.TestID, details.Identification.RowIdentification.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // With postgres provider, test details should have actual run data + for _, analysis := range details.Analyses { + total := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, total, 0, "sample stats should have run data") + } +} + +func TestFallback(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs, "GetComponentReport returned errors: %v", errs) + + type regressedInfo struct { + status crtest.Status + explanations []string + } + regressedByID := map[string]regressedInfo{} + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + regressedByID[rt.TestID] = regressedInfo{ + status: rt.ReportStatus, + explanations: rt.Explanations, + } + } + } + } + + if info, ok := regressedByID["test-fallback-improves"]; ok { + t.Logf("test-fallback-improves: status=%d, explanations=%v", info.status, info.explanations) + hasOverride := false + for _, exp := range info.explanations { + if strings.Contains(exp, "Overrode base stats") && strings.Contains(exp, "4.20") { + hasOverride = true + } + } + assert.True(t, hasOverride, "fallback-improves should mention override to 4.20 in explanations") + } else { + t.Error("test-fallback-improves should be in regressed tests") + } + + if info, ok := regressedByID["test-fallback-double"]; ok { + t.Logf("test-fallback-double: status=%d, explanations=%v", info.status, info.explanations) + hasOverride := false + for _, exp := range info.explanations { + if strings.Contains(exp, "Overrode base stats") && strings.Contains(exp, "4.19") { + hasOverride = true + } + } + assert.True(t, hasOverride, "fallback-double should mention override to 4.19 in explanations") + } else { + t.Error("test-fallback-double should be in regressed tests") + } +} + +func TestFallbackInsufficientRuns(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-fallback-insufficient-runs" { + found = true + for _, exp := range rt.Explanations { + assert.NotContains(t, exp, "Overrode base stats", + "insufficient-runs test should NOT have fallback override explanation") + } + } + } + } + } + assert.True(t, found, "test-fallback-insufficient-runs should be in the report as a regression") +} + +func TestMissingBasis(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + hasMissingBasis := false + hasMissingSample := false + for _, row := range report.Rows { + for _, col := range row.Columns { + if col.Status == crtest.MissingBasis { + hasMissingBasis = true + } + if col.Status == crtest.MissingSample { + hasMissingSample = true + } + } + } + + assert.True(t, hasMissingBasis, "report should have at least one MissingBasis cell (new test with good pass rate)") + assert.True(t, hasMissingSample, "report should have at least one MissingSample cell") +} + +func TestNewTestPassRateRegression(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + // The new flaky test (70% pass rate) should be flagged as a regression + // via buildPassRateTestStats since it's below the 90% PassRateRequiredNewTests threshold. + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-new-test-pass-rate-fail" { + found = true + assert.Equal(t, crtest.ExtremeRegression, rt.ReportStatus, + "new test with 70%% pass rate should be an extreme regression (below 80%% extreme threshold)") + assert.Equal(t, crtest.PassRate, rt.Comparison, + "new test regression should use pass rate comparison, not fisher exact") + } + } + } + } + assert.True(t, found, "test-new-test-pass-rate-fail should appear as a regressed test") +} + +func TestTestDetailsForFallbackTest(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + releases, err := provider.QueryReleases(ctx) + require.NoError(t, err) + + // test-fallback-improves has data in 4.21 (worse) and 4.20 (better), + // so the report should override the base to 4.20. + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + var testID reqopts.TestIdentification + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-fallback-improves" { + testID = reqopts.TestIdentification{ + Component: rt.RowIdentification.Component, + Capability: rt.RowIdentification.Capability, + TestID: rt.RowIdentification.TestID, + RequestedVariants: rt.ColumnIdentification.Variants, + BaseOverrideRelease: "4.20", + } + found = true + } + } + } + } + require.True(t, found, "test-fallback-improves should be in the report") + + detailReqOpts := reqOptions + detailReqOpts.TestIDOptions = []reqopts.TestIdentification{testID} + + details, detailErrs := componentreadiness.GetTestDetails(ctx, provider, nil, detailReqOpts, releases, "") + require.Empty(t, detailErrs, "GetTestDetails returned errors: %v", detailErrs) + + assert.Equal(t, "test-fallback-improves", details.Identification.RowIdentification.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // The fallback path should produce an analysis with base override data + require.GreaterOrEqual(t, len(details.Analyses), 1) + for _, analysis := range details.Analyses { + total := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, total, 0, "sample stats should have run data") + } +} + +func TestTestDetailsForNewTest(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + releases, err := provider.QueryReleases(ctx) + require.NoError(t, err) + + // test-new-test-pass-rate-fail only exists in 4.22 sample, no base data. + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + var testID reqopts.TestIdentification + found := false + for _, row := range report.Rows { + for _, col := range row.Columns { + for _, rt := range col.RegressedTests { + if rt.TestID == "test-new-test-pass-rate-fail" { + testID = reqopts.TestIdentification{ + Component: rt.RowIdentification.Component, + Capability: rt.RowIdentification.Capability, + TestID: rt.RowIdentification.TestID, + RequestedVariants: rt.ColumnIdentification.Variants, + } + found = true + } + } + } + } + require.True(t, found, "test-new-test-pass-rate-fail should be in the report") + + detailReqOpts := reqOptions + detailReqOpts.TestIDOptions = []reqopts.TestIdentification{testID} + + details, detailErrs := componentreadiness.GetTestDetails(ctx, provider, nil, detailReqOpts, releases, "") + require.Empty(t, detailErrs, "GetTestDetails returned errors: %v", detailErrs) + + assert.Equal(t, "test-new-test-pass-rate-fail", details.Identification.RowIdentification.TestID) + assert.NotEmpty(t, details.Analyses, "details should have analyses") + + // New test should have sample data but no base data + for _, analysis := range details.Analyses { + sampleTotal := analysis.SampleStats.SuccessCount + analysis.SampleStats.FailureCount + analysis.SampleStats.FlakeCount + assert.Greater(t, sampleTotal, 0, "sample stats should have run data") + if analysis.BaseStats != nil { + baseTotal := analysis.BaseStats.SuccessCount + analysis.BaseStats.FailureCount + analysis.BaseStats.FlakeCount + assert.Equal(t, 0, baseTotal, "base stats should have no run data for a new test") + } + } +} + +func TestReportDevMode(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + t.Setenv("DEV_MODE", "1") + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs, "GetComponentReport in DEV_MODE returned errors: %v", errs) + assert.NotEmpty(t, report.Rows, "DEV_MODE report should still have rows") +} + +func TestReportWithIncludeVariants(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + reqOptions.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + } + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs, "GetComponentReport with IncludeVariants returned errors: %v", errs) + require.NotEmpty(t, report.Rows, "filtered report should have rows") + + for _, row := range report.Rows { + for _, col := range row.Columns { + assert.Equal(t, "aws", col.ColumnIdentification.Variants["Platform"], + "all columns should be aws when filtering by Platform:aws") + } + } +} + +func TestSignificantImprovement(t *testing.T) { + provider, reqOptions := setupProvider(t) + ctx := context.Background() + + report, errs := componentreadiness.GetComponentReport(ctx, provider, nil, reqOptions, "") + require.Empty(t, errs) + + hasImprovement := false + for _, row := range report.Rows { + for _, col := range row.Columns { + if col.Status == crtest.SignificantImprovement { + hasImprovement = true + } + } + } + assert.True(t, hasImprovement, "report should have at least one SignificantImprovement cell") +} + +func TestJobVariants(t *testing.T) { + provider, _ := setupProvider(t) + ctx := context.Background() + + variants, errs := componentreadiness.GetJobVariants(ctx, provider) + require.Empty(t, errs) + require.NotEmpty(t, variants.Variants) + + assert.Contains(t, variants.Variants, "Platform") + assert.Contains(t, variants.Variants, "Architecture") + assert.Contains(t, variants.Variants, "Network") + assert.Contains(t, variants.Variants, "Topology") + assert.Contains(t, variants.Variants, "FeatureSet") +} diff --git a/test/e2e/componentreadiness/triage/triageapi_test.go b/test/e2e/componentreadiness/postgres/triage/triageapi_test.go similarity index 79% rename from test/e2e/componentreadiness/triage/triageapi_test.go rename to test/e2e/componentreadiness/postgres/triage/triageapi_test.go index f854055178..5c626214f6 100644 --- a/test/e2e/componentreadiness/triage/triageapi_test.go +++ b/test/e2e/componentreadiness/postgres/triage/triageapi_test.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "os" + "sort" + "strings" "testing" "time" @@ -40,12 +42,19 @@ var view = crview.View{ }, } -func cleanupAllTriages(dbc *db.DB) { - // Delete all triage and test regressions in the e2e postgres db. - dbc.DB.Exec("DELETE FROM triage_regressions WHERE 1=1") - res := dbc.DB.Where("1 = 1").Delete(&models.Triage{}) - if res.Error != nil { - log.Errorf("error deleting triage records: %v", res.Error) +// cleanupTriages deletes only the specified triages and their associated +// regression links. Tests should clean up only what they create to avoid +// destroying seed data. +func cleanupTriages(dbc *db.DB, triages ...*models.Triage) { + for _, tr := range triages { + if tr == nil || tr.ID == 0 { + continue + } + dbc.DB.Exec("DELETE FROM triage_regressions WHERE triage_id = ?", tr.ID) + res := dbc.DB.Delete(tr) + if res.Error != nil { + log.Errorf("error deleting triage %d: %v", tr.ID, res.Error) + } } } @@ -63,7 +72,6 @@ func Test_TriageAPI(t *testing.T) { defer dbc.DB.Delete(testRegression2) t.Run("create requires a valid triage type", func(t *testing.T) { - defer cleanupAllTriages(dbc) triage1 := models.Triage{ URL: jiraBug.URL, Regressions: []models.TestRegression{ @@ -82,8 +90,22 @@ func Test_TriageAPI(t *testing.T) { require.Error(t, err) }) + t.Run("create fails with non-existent regression ID", func(t *testing.T) { + triage := models.Triage{ + URL: jiraBug.URL, + Type: models.TriageTypeProduct, + Regressions: []models.TestRegression{ + {ID: testRegression1.ID}, + {ID: 999999}, // non-existent + }, + } + + var triageResponse models.Triage + err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) + require.Error(t, err, "should fail when a regression ID does not exist") + }) + t.Run("create generates audit_log record", func(t *testing.T) { - defer cleanupAllTriages(dbc) triage1 := models.Triage{ URL: jiraBug.URL, Regressions: []models.TestRegression{ @@ -97,6 +119,7 @@ func Test_TriageAPI(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage1, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) var auditLog models.AuditLog res := dbc.DB. @@ -117,8 +140,8 @@ func Test_TriageAPI(t *testing.T) { }) t.Run("get", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // ensure hateoas links are present require.NotEmpty(t, triageResponse.Links["self"]) @@ -132,44 +155,29 @@ func Test_TriageAPI(t *testing.T) { triageResponse.Links["audit_logs"]) }) t.Run("get with expanded regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) - - r := createTestRegressionWithDetails(t, tracker, view, "expanded-test-1", "component-expand", "capability-expand", "TestExpanded1", nil, crtest.ExtremeRegression) - defer dbc.DB.Delete(r.Regression) - - r2 := createTestRegressionWithDetails(t, tracker, view, "expanded-test-2", "component-expand", "capability-expand", "TestExpanded2", nil, crtest.SignificantRegression) - defer dbc.DB.Delete(r2.Regression) - - // TODO(sgoeddel): If we ever have a need for another view available within e2e tests we could verify that we could get regressed_tests - // for multiple views at once here, but it isn't worth the overhead now. - - // Create a triage with the test regressions + // Use real regressions from seed data instead of injecting fake data into cache. + // We filter to seed data regressions (test IDs starting with "test-") because + // other subtests in this function create synthetic regressions that won't appear + // in the component report. + realRegressions := getSeedDataRegressions(t) + require.GreaterOrEqual(t, len(realRegressions), 2, "seed data should produce at least 2 regressions") + + // Create a triage with real regressions triage := models.Triage{ URL: jiraBug.URL, Type: models.TriageTypeProduct, Regressions: []models.TestRegression{ - {ID: r.Regression.ID}, - {ID: r2.Regression.ID}, + {ID: realRegressions[0].ID}, + {ID: realRegressions[1].ID}, }, } var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) require.Equal(t, 2, len(triageResponse.Regressions)) - // Add the test regressions to the component report cache so they can be found by the expanded endpoint - cache, err := util.NewE2ECacheManipulator(util.Release) - if err != nil { - t.Fatalf("Failed to create component report cache: %v", err) - } - defer cache.Close() - - err = cache.AddTestRegressionsToReport([]componentreport.ReportTestSummary{r, r2}) - if err != nil { - t.Fatalf("Failed to add test regressions to component report: %v", err) - } - // Validate that the expanded regressions are present var expandedTriage sippyserver.ExpandedTriage err = util.SippyGet(fmt.Sprintf("/api/component_readiness/triages/%d?view=%s-main&expand=regressions", triageResponse.ID, util.Release), &expandedTriage) @@ -183,20 +191,20 @@ func Test_TriageAPI(t *testing.T) { regressedTestsForView := expandedTriage.RegressedTests[expectedViewKey] assert.Len(t, regressedTestsForView, 2, "ExpandedTriage should contain 2 regressed tests for view %q", expectedViewKey) - // Verify status values are marked as their respective triaged values in the expanded response - statusMap := make(map[uint]crtest.Status) + // Verify status values are marked as triaged (the triage causes status transformation) for _, regressedTest := range regressedTestsForView { - if regressedTest != nil && regressedTest.Regression != nil { - statusMap[regressedTest.Regression.ID] = regressedTest.TestComparison.ReportStatus - } + require.NotNil(t, regressedTest, "regressed test should not be nil") + require.NotNil(t, regressedTest.Regression, "regressed test should have regression data") + assert.True(t, + regressedTest.TestComparison.ReportStatus == crtest.ExtremeTriagedRegression || + regressedTest.TestComparison.ReportStatus == crtest.SignificantTriagedRegression, + "regressed test %s should have a triaged status, got %d", + regressedTest.Regression.TestID, regressedTest.TestComparison.ReportStatus) } - - assert.Equal(t, crtest.ExtremeTriagedRegression, statusMap[r.Regression.ID], "First regressed test should have ExtremeTriagedRegression status") - assert.Equal(t, crtest.SignificantTriagedRegression, statusMap[r2.Regression.ID], "Second regressed test should have SignificantTriagedRegression status") }) t.Run("list", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var allTriages []models.Triage err := util.SippyGet("/api/component_readiness/triages", &allTriages) @@ -226,8 +234,8 @@ func Test_TriageAPI(t *testing.T) { } }) t.Run("update to add regression", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // Update with a new regression: var triageResponse2 models.Triage @@ -250,8 +258,6 @@ func Test_TriageAPI(t *testing.T) { triageResponse2.Links["audit_logs"]) }) t.Run("update to remove a regression", func(t *testing.T) { - defer cleanupAllTriages(dbc) - triage := models.Triage{ URL: jiraBug.URL, Type: models.TriageTypeProduct, @@ -264,6 +270,7 @@ func Test_TriageAPI(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) assert.Equal(t, 2, len(triageResponse.Regressions)) // Update to remove one regression - keep only testRegression1 @@ -277,8 +284,8 @@ func Test_TriageAPI(t *testing.T) { assert.NotEqual(t, triageResponse.UpdatedAt, triageResponse2.UpdatedAt) }) t.Run("update to remove all regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage triageResponse.Regressions = []models.TestRegression{} @@ -287,8 +294,8 @@ func Test_TriageAPI(t *testing.T) { assert.Equal(t, 0, len(triageResponse2.Regressions)) }) t.Run("update to resolve triage sets resolution reason to user", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // Resolve the triage by setting the Resolved timestamp resolvedTime := time.Now() @@ -304,8 +311,8 @@ func Test_TriageAPI(t *testing.T) { assert.Equal(t, models.User, updateResponse.ResolutionReason, "Resolution reason should be set to 'user'") }) t.Run("update fails if resource has no ID", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage triageResponse.ID = 0 @@ -313,8 +320,8 @@ func Test_TriageAPI(t *testing.T) { require.Error(t, err) }) t.Run("update fails if URL has no ID", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage // No ID specified in URL should not work for an update @@ -322,16 +329,16 @@ func Test_TriageAPI(t *testing.T) { require.Error(t, err) }) t.Run("update fails if URL ID and resource ID do not match", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) var triageResponse2 models.Triage err := util.SippyPut("/api/component_readiness/triages/128736182736128736", &triageResponse, &triageResponse2) require.Error(t, err) }) t.Run("update generates audit_log record", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) originalTriage := deepCopyTriage(t, triageResponse) // Update with a new regression, and a changed description: @@ -364,8 +371,8 @@ func Test_TriageAPI(t *testing.T) { assertTriageDataMatches(t, originalTriage, oldTriageData, "OldData") }) t.Run("delete generates audit_log record", func(t *testing.T) { - defer cleanupAllTriages(dbc) triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) originalTriage := deepCopyTriage(t, triageResponse) // Delete the triage record @@ -391,8 +398,6 @@ func Test_TriageAPI(t *testing.T) { }) t.Run("audit endpoint returns full lifecycle operations", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a triage triage := models.Triage{ URL: "https://issues.redhat.com/browse/OCPBUGS-8888", @@ -406,6 +411,7 @@ func Test_TriageAPI(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) require.True(t, triageResponse.ID > 0) // Small delay to ensure different timestamps @@ -548,8 +554,8 @@ func Test_RegressionAPI(t *testing.T) { release := view.SampleRelease.Release.Name t.Run("list regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) - _ = createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + triageResponse := createAndValidateTriageRecord(t, jiraBug.URL, testRegression1) + defer cleanupTriages(dbc, &triageResponse) // Test listing regressions by release (release is required) var allRegressions []models.TestRegression @@ -578,8 +584,6 @@ func Test_RegressionAPI(t *testing.T) { assert.Contains(t, testDetailsHREF, "testId=", "test_details link should contain testId parameter") }) t.Run("error when both view and release are specified", func(t *testing.T) { - defer cleanupAllTriages(dbc) - var regressions []models.TestRegression err := util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions?view=%s-main&release=%s", util.Release, util.Release), ®ressions) require.Error(t, err, "Expected error when both view and release are provided") @@ -616,8 +620,6 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { defer dbc.DB.Delete(noMatchRegression.Regression) t.Run("find potential matching triages", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create triages with the matching regressions triage1 := models.Triage{ URL: jiraBug.URL, @@ -629,6 +631,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponse1 models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage1, &triageResponse1) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse1) triage2 := models.Triage{ URL: jiraBug.URL, @@ -640,6 +643,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponse2 models.Triage err = util.SippyPost("/api/component_readiness/triages", &triage2, &triageResponse2) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse2) // Create a triage with the no-match regression (should not appear in results) triageNoMatch := models.Triage{ @@ -652,6 +656,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponseNoMatch models.Triage err = util.SippyPost("/api/component_readiness/triages", &triageNoMatch, &triageResponseNoMatch) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponseNoMatch) // Query for potential matches for the target regression var potentialMatches []componentreadiness.PotentialMatchingTriage @@ -692,8 +697,6 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { }) t.Run("no potential matches found", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a triage with the no-match regression (different name and time) triage := models.Triage{ URL: jiraBug.URL, @@ -705,6 +708,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) // Query for potential matches for the target regression var potentialMatches []componentreadiness.PotentialMatchingTriage @@ -717,8 +721,6 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { }) t.Run("resolved triage confidence level capped at 5", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a regression with the exact same test name (edit distance 0, would normally give confidence 6) exactMatchRegression := createTestRegressionWithDetails(t, tracker, view, "exact-match", "component-e", "capability-v", "TestTargetFunction", &differentFailureTime, crtest.ExtremeRegression) defer dbc.DB.Delete(exactMatchRegression.Regression) @@ -734,6 +736,7 @@ func Test_RegressionPotentialMatchingTriages(t *testing.T) { var triageResponseExactMatch models.Triage err := util.SippyPost("/api/component_readiness/triages", &triageExactMatch, &triageResponseExactMatch) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponseExactMatch) // Resolve the triage resolvedTime := time.Now() @@ -818,8 +821,6 @@ func Test_TriageRawDB(t *testing.T) { defer dbc.DB.Delete(testRegression) t.Run("test Triage model in postgres", func(t *testing.T) { - defer cleanupAllTriages(dbc) - triage1 := models.Triage{ URL: "http://myjira", Regressions: []models.TestRegression{ @@ -828,6 +829,7 @@ func Test_TriageRawDB(t *testing.T) { } res := dbWithContext.Create(&triage1) require.NoError(t, res.Error) + defer cleanupTriages(dbc, &triage1) testRegression.Triages = append(testRegression.Triages, triage1) res = dbWithContext.Save(&testRegression) require.NoError(t, res.Error) @@ -865,6 +867,7 @@ func Test_TriageRawDB(t *testing.T) { } res = dbWithContext.Create(&triage2) require.NoError(t, res.Error) + defer cleanupTriages(dbc, &triage2) testRegression.Triages = append(testRegression.Triages, triage2) res = dbWithContext.Save(&testRegression) require.NoError(t, res.Error) @@ -887,8 +890,6 @@ func Test_TriageRawDB(t *testing.T) { }) t.Run("test Triage model Bug relationship", func(t *testing.T) { - defer cleanupAllTriages(dbc) - jiraBug := createBug(t, dbWithContext) defer dbWithContext.Delete(jiraBug) @@ -898,6 +899,7 @@ func Test_TriageRawDB(t *testing.T) { } res := dbWithContext.Create(&triage1) require.NoError(t, res.Error) + defer cleanupTriages(dbc, &triage1) // Lookup the Triage again to ensure we persisted what we expect: res = dbWithContext.First(&triage1, triage1.ID) @@ -948,6 +950,26 @@ func deepCopyTriage(t *testing.T, original models.Triage) models.Triage { return triageCopy } +// getSeedDataRegressions fetches regressions from the API and filters to only those +// from the seed data (test IDs starting with "test-"), excluding any synthetic regressions +// created by other test functions in this package. +func getSeedDataRegressions(t *testing.T) []models.TestRegression { + var allRegressions []models.TestRegression + err := util.SippyGet(fmt.Sprintf("/api/component_readiness/regressions?release=%s", util.Release), &allRegressions) + require.NoError(t, err) + + var seedRegressions []models.TestRegression + for _, r := range allRegressions { + if strings.HasPrefix(r.TestID, "test-") { + seedRegressions = append(seedRegressions, r) + } + } + sort.Slice(seedRegressions, func(i, j int) bool { + return seedRegressions[i].TestID < seedRegressions[j].TestID + }) + return seedRegressions +} + func assertTriageDataMatches(t *testing.T, expectedTriage, actualTriage models.Triage, field string) { assert.Equal(t, expectedTriage.ID, actualTriage.ID, "%s ID should match the expected triage ID", field) assert.Equal(t, expectedTriage.URL, actualTriage.URL, "%s URL should match the expected triage URL", field) @@ -1012,47 +1034,38 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { testRegressions[9] = createTestRegressionWithDetails(t, tracker, view, "match-similar-2", "component-j", "capability-q", "TestAnotheOne", &differentFailureTime, crtest.SignificantImprovement) // missing 'r' from "TestAnotherOne" defer dbc.DB.Delete(testRegressions[9].Regression) - // Add all test regressions to the component report so they can be found by GetTriagePotentialMatches - cache, err := util.NewE2ECacheManipulator(util.Release) - if err != nil { - t.Fatalf("Failed to create component report cache: %v", err) - } - defer cache.Close() - - err = cache.AddTestRegressionsToReport(testRegressions) - if err != nil { - t.Fatalf("Failed to add test regressions to component report: %v", err) - } - t.Run("find potential matching regressions", func(t *testing.T) { - defer cleanupAllTriages(dbc) + // Use real regressions from seed data that appear in the component report. + // Synthetic regressions with fake test IDs won't appear in the report, so + // GetTriagePotentialMatches would skip them entirely. + realRegressions := getSeedDataRegressions(t) + require.GreaterOrEqual(t, len(realRegressions), 3, "seed data should produce at least 3 regressions") - // Create a triage with two linked regressions + // Create a triage linked to the first real regression triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-1234", Type: models.TriageTypeProduct, Regressions: []models.TestRegression{ - {ID: testRegressions[0].Regression.ID}, // TestSomething with commonFailureTime - {ID: testRegressions[1].Regression.ID}, // TestAnother with unique failure time + {ID: realRegressions[0].ID}, }, } var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) - require.Equal(t, 2, len(triageResponse.Regressions)) + defer cleanupTriages(dbc, &triageResponse) + require.Equal(t, 1, len(triageResponse.Regressions)) - // Query for potential matches + // Query for potential matches — the other real regressions should appear var potentialMatches []componentreadiness.PotentialMatchingRegression - endpoint := fmt.Sprintf("/api/component_readiness/triages/%d/matches?baseRelease=%s&sampleRelease=%s", triageResponse.ID, view.BaseRelease.Release.Name, view.SampleRelease.Release.Name) err = util.SippyGet(endpoint, &potentialMatches) require.NoError(t, err) - // Verify the results - assert.True(t, len(potentialMatches) > 0, "Should find some potential matches") + // Should find some potential matches from the other real regressions + assert.True(t, len(potentialMatches) > 0, "Should find potential matches from real seed data regressions") - // Verify HATEOAS links are present in potential match responses + // Verify HATEOAS links are present on matches baseURL := fmt.Sprintf("http://%s:%s", os.Getenv("SIPPY_ENDPOINT"), os.Getenv("SIPPY_API_PORT")) for _, match := range potentialMatches { assert.Equal(t, fmt.Sprintf("%s/api/component_readiness/triages/%d/matches", baseURL, triageResponse.ID), @@ -1061,124 +1074,27 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { match.Links["triage"], "Potential match should have triage link") } - // Verify status values are correctly returned for the potential matches - statusMap := make(map[uint]crtest.Status) + // Verify that the linked regression is NOT in the potential matches + foundRegressionIDs := make(map[uint]bool) for _, match := range potentialMatches { if match.RegressedTest.Regression != nil { - statusMap[match.RegressedTest.Regression.ID] = match.RegressedTest.TestComparison.ReportStatus + foundRegressionIDs[match.RegressedTest.Regression.ID] = true } } + assert.False(t, foundRegressionIDs[realRegressions[0].ID], "Linked regression should not appear in potential matches") - // Build maps for easier verification - foundRegressionIDs := make(map[uint]bool) - matchesBySimilarName := make(map[uint][]componentreadiness.SimilarlyNamedTest) - matchesBySameFailure := make(map[uint][]models.TestRegression) - confidenceLevels := make(map[uint]int) - + // Verify each match has valid regression data and a confidence level for _, match := range potentialMatches { - if match.RegressedTest.Regression == nil { - continue // Skip if no regression data - } - regressionID := match.RegressedTest.Regression.ID - foundRegressionIDs[regressionID] = true - confidenceLevels[regressionID] = match.ConfidenceLevel - if len(match.SimilarlyNamedTests) > 0 { - matchesBySimilarName[regressionID] = match.SimilarlyNamedTests - } - if len(match.SameLastFailures) > 0 { - matchesBySameFailure[regressionID] = match.SameLastFailures - } - } - - // Verify that linked regressions are NOT in the potential matches - assert.False(t, foundRegressionIDs[testRegressions[0].Regression.ID], "Linked regression 0 should not appear in potential matches") - assert.False(t, foundRegressionIDs[testRegressions[1].Regression.ID], "Linked regression 1 should not appear in potential matches") - - // Verify expected matches are found - assert.True(t, foundRegressionIDs[testRegressions[2].Regression.ID], "Should find regression 2 (similar name to TestSomething)") - assert.True(t, foundRegressionIDs[testRegressions[3].Regression.ID], "Should find regression 3 (same failure time)") - assert.True(t, foundRegressionIDs[testRegressions[4].Regression.ID], "Should find regression 4 (both similar name and same failure)") - assert.True(t, foundRegressionIDs[testRegressions[5].Regression.ID], "Should find regression 5 (similar name)") - assert.True(t, foundRegressionIDs[testRegressions[8].Regression.ID], "Should find regression 8 (same failure time)") - assert.True(t, foundRegressionIDs[testRegressions[9].Regression.ID], "Should find regression 9 (similar name to TestAnother)") - - // Verify the status values are correctly returned - assert.Equal(t, crtest.ExtremeTriagedRegression, statusMap[testRegressions[2].Regression.ID], "Regression 2 should have ExtremeTriagedRegression status") - assert.Equal(t, crtest.SignificantTriagedRegression, statusMap[testRegressions[3].Regression.ID], "Regression 3 should have SignificantTriagedRegression status") - assert.Equal(t, crtest.FixedRegression, statusMap[testRegressions[4].Regression.ID], "Regression 4 should have FixedRegression status") - assert.Equal(t, crtest.MissingSample, statusMap[testRegressions[5].Regression.ID], "Regression 5 should have MissingSample status") - assert.Equal(t, crtest.MissingBasisAndSample, statusMap[testRegressions[8].Regression.ID], "Regression 8 should have MissingBasisAndSample status") - assert.Equal(t, crtest.SignificantImprovement, statusMap[testRegressions[9].Regression.ID], "Regression 9 should have SignificantImprovement status") - - // Verify non-matches are not found - assert.False(t, foundRegressionIDs[testRegressions[6].Regression.ID], "Should not find regression 6 (no match)") - assert.False(t, foundRegressionIDs[testRegressions[7].Regression.ID], "Should not find regression 7 (name too different)") - - // Verify match reasons are correct - - // Regression 2: Should match by similar name to "TestSomething" - if assert.Contains(t, matchesBySimilarName, testRegressions[2].Regression.ID) { - matches := matchesBySimilarName[testRegressions[2].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].Regression.ID, "Should match against TestSomething regression") - // TestSomthng vs TestSomething = edit distance 2, so score = 6-2 = 4 - assert.Equal(t, 4, confidenceLevels[testRegressions[2].Regression.ID], "Confidence should be 4 (edit distance 2: 6-2)") - } - - // Regression 3: Should match by same failure time - if assert.Contains(t, matchesBySameFailure, testRegressions[3].Regression.ID) { - matches := matchesBySameFailure[testRegressions[3].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one same failure time") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].ID, "Should match against commonFailureTime regression") - assert.Equal(t, 1, confidenceLevels[testRegressions[3].Regression.ID], "Confidence should be 1 (1 failure match * 1)") - } - - // Regression 4: Should match both similar name AND same failure time - if assert.Contains(t, matchesBySimilarName, testRegressions[4].Regression.ID) { - nameMatches := matchesBySimilarName[testRegressions[4].Regression.ID] - assert.Equal(t, 1, len(nameMatches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[1].Regression.ID, nameMatches[0].Regression.ID, "Should match against TestAnotherOne regression") - } - if assert.Contains(t, matchesBySameFailure, testRegressions[4].Regression.ID) { - failureMatches := matchesBySameFailure[testRegressions[4].Regression.ID] - assert.Equal(t, 1, len(failureMatches), "Should match exactly one same failure time") - assert.Equal(t, testRegressions[0].Regression.ID, failureMatches[0].ID, "Should match against commonFailureTime regression") - // TestAnoterOne vs TestAnotherOne = edit distance 1, so name score = 6-1 = 5, failure = 1, total = 6 - assert.Equal(t, 6, confidenceLevels[testRegressions[4].Regression.ID], "Confidence should be 6 (name edit distance 1: 6-1=5, plus 1 failure match)") - } - - // Regression 5: Should match by similar name only - if assert.Contains(t, matchesBySimilarName, testRegressions[5].Regression.ID) { - matches := matchesBySimilarName[testRegressions[5].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].Regression.ID, "Should match against TestSomething regression") - // TestSomthing vs TestSomething = edit distance 1, so score = 6-1 = 5 - assert.Equal(t, 5, confidenceLevels[testRegressions[5].Regression.ID], "Confidence should be 5 (edit distance 1: 6-1)") - } - assert.NotContains(t, matchesBySameFailure, testRegressions[5].Regression.ID, "Should not match by failure time") - - // Regression 8: Should match by same failure time only - if assert.Contains(t, matchesBySameFailure, testRegressions[8].Regression.ID) { - matches := matchesBySameFailure[testRegressions[8].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one same failure time") - assert.Equal(t, testRegressions[0].Regression.ID, matches[0].ID, "Should match against commonFailureTime regression") - assert.Equal(t, 1, confidenceLevels[testRegressions[8].Regression.ID], "Confidence should be 1 (1 failure match * 1)") - } - assert.NotContains(t, matchesBySimilarName, testRegressions[8].Regression.ID, "Should not match by similar name") - - // Regression 9: Should match by similar name to "TestAnotherOne" - if assert.Contains(t, matchesBySimilarName, testRegressions[9].Regression.ID) { - matches := matchesBySimilarName[testRegressions[9].Regression.ID] - assert.Equal(t, 1, len(matches), "Should match exactly one similar name") - assert.Equal(t, testRegressions[1].Regression.ID, matches[0].Regression.ID, "Should match against TestAnotherOne regression") - // TestAnotheOne vs TestAnotherOne = edit distance 1, so score = 6-1 = 5 - assert.Equal(t, 5, confidenceLevels[testRegressions[9].Regression.ID], "Confidence should be 5 (edit distance 1: 6-1)") + require.NotNil(t, match.RegressedTest.Regression, "matched regression should not be nil") + assert.Greater(t, match.ConfidenceLevel, 0, "confidence level should be positive") + // Each match should have at least one reason (similar name or same failure time) + assert.True(t, + len(match.SimilarlyNamedTests) > 0 || len(match.SameLastFailures) > 0, + "match for regression %d should have at least one match reason", match.RegressedTest.Regression.ID) } }) t.Run("empty potential matches when no regressions exist", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a triage with one linked regression triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-1234", @@ -1191,6 +1107,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) // Query for potential matches var potentialMatches []componentreadiness.PotentialMatchingRegression @@ -1212,8 +1129,6 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { }) t.Run("empty potential matches when release pair does not match any view", func(t *testing.T) { - defer cleanupAllTriages(dbc) - triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-9999", Type: models.TriageTypeProduct, @@ -1225,6 +1140,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) + defer cleanupTriages(dbc, &triageResponse) var potentialMatches []componentreadiness.PotentialMatchingRegression endpoint := fmt.Sprintf("/api/component_readiness/triages/%d/matches?baseRelease=no-such-base&sampleRelease=no-such-sample", triageResponse.ID) @@ -1234,7 +1150,7 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { }) t.Run("error when triage not found", func(t *testing.T) { - var potentialMatches []interface{} + var potentialMatches []any endpoint := "/api/component_readiness/triages/999999/matches" err := util.SippyGet(endpoint, &potentialMatches) @@ -1242,37 +1158,44 @@ func Test_TriagePotentialMatchingRegressions(t *testing.T) { }) t.Run("verify status values in triage responses", func(t *testing.T) { - defer cleanupAllTriages(dbc) - // Create a triage with regressions that have different status values + // Use real regressions that appear in the component report so we can verify + // status transformation via the expand endpoint + realRegressions := getSeedDataRegressions(t) + require.GreaterOrEqual(t, len(realRegressions), 2, "seed data should produce at least 2 regressions") + triage := models.Triage{ URL: "https://issues.redhat.com/OCPBUGS-5678", Type: models.TriageTypeProduct, Regressions: []models.TestRegression{ - {ID: testRegressions[0].Regression.ID}, // ExtremeRegression - {ID: testRegressions[1].Regression.ID}, // SignificantRegression - {ID: testRegressions[4].Regression.ID}, // FixedRegression + {ID: realRegressions[0].ID}, + {ID: realRegressions[1].ID}, }, } var triageResponse models.Triage err := util.SippyPost("/api/component_readiness/triages", &triage, &triageResponse) require.NoError(t, err) - require.Equal(t, 3, len(triageResponse.Regressions)) - - // Note: TestComparison (including status) is not available on the basic TestRegression model - // returned by the triage API. Status is only available in the potential matches endpoint - // where regressions are represented as ReportTestSummary with full component report data. - // - // However, we can verify that our test setup correctly created regressions with different IDs - regressionIDs := make(map[uint]bool) - for _, regression := range triageResponse.Regressions { - regressionIDs[regression.ID] = true - } + defer cleanupTriages(dbc, &triageResponse) + require.Equal(t, 2, len(triageResponse.Regressions)) - assert.True(t, regressionIDs[testRegressions[0].Regression.ID], "Should find first regression") - assert.True(t, regressionIDs[testRegressions[1].Regression.ID], "Should find second regression") - assert.True(t, regressionIDs[testRegressions[4].Regression.ID], "Should find third regression") + // Use the expand endpoint to get status values from the component report + var expandedTriage sippyserver.ExpandedTriage + err = util.SippyGet(fmt.Sprintf("/api/component_readiness/triages/%d?view=%s-main&expand=regressions", triageResponse.ID, util.Release), &expandedTriage) + require.NoError(t, err) + + regressedTests := expandedTriage.RegressedTests[view.Name] + require.NotEmpty(t, regressedTests, "expanded triage should contain regressed tests") + + // Verify each regressed test has a triaged status (the triage transforms the status) + for _, rt := range regressedTests { + require.NotNil(t, rt, "regressed test should not be nil") + require.NotNil(t, rt.Regression, "regressed test should have regression data") + status := rt.TestComparison.ReportStatus + assert.True(t, + status == crtest.ExtremeTriagedRegression || status == crtest.SignificantTriagedRegression, + "regression %s status should be triaged, got %d", rt.Regression.TestID, status) + } }) } diff --git a/test/e2e/datasync/datasync_test.go b/test/e2e/datasync/datasync_test.go new file mode 100644 index 0000000000..f3ff3facef --- /dev/null +++ b/test/e2e/datasync/datasync_test.go @@ -0,0 +1,117 @@ +package datasync + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/openshift/sippy/test/e2e/util" + "github.com/stretchr/testify/require" +) + +func TestDataSync(t *testing.T) { + sippyImage := os.Getenv("SIPPY_E2E_SIPPY_IMAGE") + if sippyImage == "" { + t.Skip("SIPPY_E2E_SIPPY_IMAGE not set, skipping data sync test") + } + + dbc := util.CreateE2EPostgresConnection(t) + + var countBefore int64 + require.NoError(t, dbc.DB.Table("prow_job_runs").Count(&countBefore).Error) + t.Logf("prow_job_runs before sync: %d", countBefore) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + kubectl := os.Getenv("KUBECTL_CMD") + if kubectl == "" { + kubectl = "oc" + } + + // Create a Job that runs sippy load as a pod on the cluster, with GCS + // credentials and coverage instrumentation. + jobManifest := fmt.Sprintf(`apiVersion: batch/v1 +kind: Job +metadata: + name: sippy-datasync-job + namespace: sippy-e2e +spec: + template: + spec: + containers: + - name: sippy + image: %s + resources: + limits: + memory: 8Gi + command: ["/bin/sippy-cover"] + args: + - load + - --loader + - prow + - --release + - "%s" + - --prow-load-since + - 2h + - --config + - config/e2e-openshift.yaml + - --google-service-account-credential-file + - /tmp/secrets/gcs-cred + - --database-dsn + - postgresql://postgres:password@postgres.sippy-e2e.svc.cluster.local:5432/postgres + - --skip-matview-refresh + - --log-level + - debug + env: + - name: GOCOVERDIR + value: /tmp/coverage + volumeMounts: + - mountPath: /tmp/secrets + name: gcs-cred + readOnly: true + - mountPath: /tmp/coverage + name: coverage + imagePullSecrets: + - name: regcred + volumes: + - name: gcs-cred + secret: + secretName: gcs-cred + - name: coverage + persistentVolumeClaim: + claimName: sippy-coverage + restartPolicy: Never + backoffLimit: 0`, sippyImage, util.Release) + + // Apply the job manifest + applyCmd := exec.CommandContext(ctx, kubectl, "apply", "-f", "-") // #nosec G204 + applyCmd.Stdin = strings.NewReader(jobManifest) + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + require.NoError(t, applyCmd.Run(), "failed to create datasync job") + + // Wait for the job to complete + waitCmd := exec.CommandContext(ctx, kubectl, "-n", "sippy-e2e", "wait", // #nosec G204 + "--for=condition=complete", "job/sippy-datasync-job", "--timeout=600s") + waitCmd.Stdout = os.Stdout + waitCmd.Stderr = os.Stderr + waitErr := waitCmd.Run() + + // Collect logs regardless of success/failure + logCmd := exec.CommandContext(ctx, kubectl, "-n", "sippy-e2e", "logs", // #nosec G204 + "--selector=job-name=sippy-datasync-job") + logCmd.Stdout = os.Stdout + logCmd.Stderr = os.Stderr + _ = logCmd.Run() + + require.NoError(t, waitErr, "sippy load job should complete successfully") + + var countAfter int64 + require.NoError(t, dbc.DB.Table("prow_job_runs").Count(&countAfter).Error) + t.Logf("prow_job_runs after sync: %d (loaded %d new)", countAfter, countAfter-countBefore) +} diff --git a/test/e2e/util/cache_manipulator.go b/test/e2e/util/cache_manipulator.go deleted file mode 100644 index 416095fdf0..0000000000 --- a/test/e2e/util/cache_manipulator.go +++ /dev/null @@ -1,238 +0,0 @@ -package util - -import ( - "encoding/json" - "fmt" - "os" - "strings" - "time" - - "gopkg.in/redis.v5" - - "github.com/openshift/sippy/pkg/api/componentreadiness" - "github.com/openshift/sippy/pkg/apis/api/componentreport" - "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" - log "github.com/sirupsen/logrus" -) - -// E2ECacheManipulator provides utilities for manipulating values in the Redis cache -type E2ECacheManipulator struct { - release string - client *redis.Client -} - -func NewE2ECacheManipulator(release string) (*E2ECacheManipulator, error) { - client, err := connectToRedis() - if err != nil { - return nil, err - } - return &E2ECacheManipulator{ - release: release, - client: client, - }, nil -} - -func (c *E2ECacheManipulator) Close() { - if c.client != nil { - c.client.Close() - } -} - -// connectToRedis creates a Redis client connection -func connectToRedis() (*redis.Client, error) { - redisURL := os.Getenv("REDIS_URL") - if redisURL == "" { - redisURL = "redis://localhost:23479" // Default for e2e tests - } - - opts, err := redis.ParseURL(redisURL) - if err != nil { - return nil, fmt.Errorf("failed to parse Redis URL: %w", err) - } - - client := redis.NewClient(opts) - - // Test Redis connection - if err := client.Ping().Err(); err != nil { - client.Close() - return nil, fmt.Errorf("failed to connect to Redis: %w", err) - } - - return client, nil -} - -// AddTestRegressionsToReport adds test regressions to the component report in cache -// so they can be found by GetTriagePotentialMatches function -func (c *E2ECacheManipulator) AddTestRegressionsToReport(testRegressions []componentreport.ReportTestSummary) error { - // Get the current component report directly from the cache - report, cacheKey, err := c.GetReport() - if err != nil { - return fmt.Errorf("failed to get component report from cache: %w", err) - } - - // Add each test regression to the appropriate place in the report - for _, testRegression := range testRegressions { - if testRegression.Regression == nil { - continue // Skip if no regression data - } - - // Find or create the appropriate row for this component - var targetRowIndex = -1 - for i, row := range report.Rows { - if row.Component == testRegression.Component { - targetRowIndex = i - break - } - } - - // If no row exists for this component, create one - if targetRowIndex == -1 { - newRow := componentreport.ReportRow{ - RowIdentification: crtest.RowIdentification{ - Component: testRegression.Component, - Capability: testRegression.Capability, - }, - Columns: []componentreport.ReportColumn{ - { - ColumnIdentification: crtest.ColumnIdentification{ - Variants: testRegression.Variants, - }, - Status: crtest.SignificantRegression, - RegressedTests: []componentreport.ReportTestSummary{testRegression}, - }, - }, - } - report.Rows = append(report.Rows, newRow) - } else { - // Add to existing row - find or create a matching column - row := &report.Rows[targetRowIndex] - var targetColumnIndex = -1 - - // Look for a column with matching variants - for i, col := range row.Columns { - if variantsMatch(col.Variants, testRegression.Variants) { - targetColumnIndex = i - break - } - } - - if targetColumnIndex == -1 { - // Create new column - newColumn := componentreport.ReportColumn{ - ColumnIdentification: crtest.ColumnIdentification{ - Variants: testRegression.Variants, - }, - Status: crtest.SignificantRegression, - RegressedTests: []componentreport.ReportTestSummary{testRegression}, - } - row.Columns = append(row.Columns, newColumn) - } else { - // Add to existing column - row.Columns[targetColumnIndex].RegressedTests = append(row.Columns[targetColumnIndex].RegressedTests, testRegression) - } - } - } - - // Update the cached component report so GetTriagePotentialMatches can find the test regressions - err = c.updateReportWithKey(report, cacheKey) - if err != nil { - log.WithError(err).Warn("Failed to update cached component report, test may not work as expected") - } - - return nil -} - -// GetReport retrieves the component report directly from Redis cache -func (c *E2ECacheManipulator) GetReport() (componentreport.ComponentReport, string, error) { - // Find the ComponentReport cache key by scanning for keys that start with "ComponentReport~" - keyPattern := "_SIPPY_*ComponentReport~*" - keys, err := c.client.Keys(keyPattern).Result() - if err != nil { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to scan for ComponentReport keys: %w", err) - } - - var cacheKey string - for _, key := range keys { - // Strip the prefixes to get the JSON part - // Key format: "_SIPPY_cc:ComponentReport~{JSON}" or "_SIPPY_ComponentReport~{JSON}" - jsonPart := key - - // Remove "_SIPPY_" prefix if present - jsonPart = strings.TrimPrefix(jsonPart, "_SIPPY_") - - // Remove "cc:" prefix if present (compressed cache prefix) - jsonPart = strings.TrimPrefix(jsonPart, "cc:") - - // Remove "ComponentReport~" prefix - if !strings.HasPrefix(jsonPart, "ComponentReport~") { - log.Warnf("Unexpected cache key format, missing ComponentReport~ prefix: %s", key) - continue - } - jsonPart = jsonPart[len("ComponentReport~"):] - - gk := &componentreadiness.GeneratorCacheKey{} - if err := json.Unmarshal([]byte(jsonPart), gk); err != nil { - log.Warnf("Failed to unmarshal ComponentReport key JSON '%s' from key '%s': %v", jsonPart, key, err) - continue - } - if gk.SampleRelease.Name == Release { - cacheKey = key - break - } - } - if cacheKey == "" { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to find proper ComponentReport key") - } - log.Debugf("Found ComponentReport cache key: %s", cacheKey) - - // Get the cached data - cachedData, err := c.client.Get(cacheKey).Bytes() - if err != nil { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to get cached data for key %s: %w", cacheKey, err) - } - - // Unmarshal the component report - var report componentreport.ComponentReport - err = json.Unmarshal(cachedData, &report) - if err != nil { - return componentreport.ComponentReport{}, "", fmt.Errorf("failed to unmarshal component report: %w", err) - } - - log.Debugf("Retrieved cached component report with %d rows", len(report.Rows)) - return report, cacheKey, nil -} - -// updateReportWithKey updates the Redis cache with the modified component report using the exact cache key -func (c *E2ECacheManipulator) updateReportWithKey(report componentreport.ComponentReport, cacheKey string) error { - if cacheKey == "" { - return fmt.Errorf("cache key is empty, cannot update cache") - } - - // Marshal the modified report to JSON - reportJSON, err := json.Marshal(report) - if err != nil { - return fmt.Errorf("failed to marshal component report: %w", err) - } - - // Store plain JSON in Redis cache with a reasonable expiration (1 hour) - err = c.client.Set(cacheKey, reportJSON, time.Hour).Err() - if err != nil { - return fmt.Errorf("failed to store in Redis cache: %w", err) - } - - log.Debugf("Updated cached component report with key %s, %d rows", cacheKey, len(report.Rows)) - return nil -} - -// variantsMatch checks if two variant maps are equivalent -func variantsMatch(a, b map[string]string) bool { - if len(a) != len(b) { - return false - } - for k, v := range a { - if b[k] != v { - return false - } - } - return true -} diff --git a/test/e2e/util/e2erequest.go b/test/e2e/util/e2erequest.go index aca9bb91cb..98ce6638bb 100644 --- a/test/e2e/util/e2erequest.go +++ b/test/e2e/util/e2erequest.go @@ -13,8 +13,8 @@ import ( const ( // Needs to match what we import in the e2e.sh script - Release = "4.20" - BaseRelease = "4.19" + Release = "4.22" + BaseRelease = "4.21" // APIPort is the port e2e.sh launches the sippy API on. These values must be kept in sync. APIPort = 18080