diff --git a/pkg/api/job_runs.go b/pkg/api/job_runs.go index c6cebebe5f..a36581fdd4 100644 --- a/pkg/api/job_runs.go +++ b/pkg/api/job_runs.go @@ -106,12 +106,41 @@ func JobsRunsReportFromDB(dbc *db.DB, filterOpts *filter.FilterOptions, release } res := q.Scan(&jobsResult) + if res.Error != nil { + return nil, res.Error + } + + // Fetch annotations separately to avoid bloating the materialized view. + if len(jobsResult) > 0 { + ids := make([]int, len(jobsResult)) + for i, jr := range jobsResult { + ids[i] = jr.ID + } + var annotations []models.ProwJobRunAnnotation + if err := dbc.DB.Where("prow_job_run_id IN ?", ids).Find(&annotations).Error; err != nil { + return nil, err + } + annotationsByRun := make(map[string]apitype.AnnotationMap) + for _, a := range annotations { + annotationID := strconv.FormatUint(uint64(a.ProwJobRunID), 10) + if annotationsByRun[annotationID] == nil { + annotationsByRun[annotationID] = make(apitype.AnnotationMap) + } + annotationsByRun[annotationID][a.Key] = a.Value + } + for i := range jobsResult { + if am, ok := annotationsByRun[strconv.Itoa(jobsResult[i].ID)]; ok { + jobsResult[i].Annotations = am + } + } + } + return &apitype.PaginationResult{ Rows: jobsResult, TotalRows: rowCount, PageSize: pagination.PerPage, Page: pagination.Page, - }, res.Error + }, nil } // FetchJobRun returns a single job run loaded from postgres and populated with the ProwJob and test results. diff --git a/pkg/apis/api/types.go b/pkg/apis/api/types.go index 6ddea3c28d..4649e50fe6 100644 --- a/pkg/apis/api/types.go +++ b/pkg/apis/api/types.go @@ -356,8 +356,12 @@ type JobRun struct { PullRequestSHA string `json:"pull_request_sha"` PullRequestAuthor string `json:"pull_request_author"` Labels pq.StringArray `json:"labels" gorm:"type:text[]"` + Annotations AnnotationMap `json:"annotations,omitempty"` } +// AnnotationMap is a map[string]string for job run annotations. +type AnnotationMap map[string]string + func (run JobRun) GetFieldType(param string) ColumnType { switch param { case "name": diff --git a/pkg/apis/prow/prow.go b/pkg/apis/prow/prow.go index 90afd3c166..72cb452fbe 100644 --- a/pkg/apis/prow/prow.go +++ b/pkg/apis/prow/prow.go @@ -92,6 +92,7 @@ type ProwJobStatus struct { } type ProwJob struct { - Spec ProwJobSpec `json:"spec,omitempty"` - Status ProwJobStatus `json:"status,omitempty"` + Spec ProwJobSpec `json:"spec,omitempty"` + Status ProwJobStatus `json:"status,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` } diff --git a/pkg/dataloader/prowloader/bigqueryjobs.go b/pkg/dataloader/prowloader/bigqueryjobs.go index bce35375a2..0ffa7b3440 100644 --- a/pkg/dataloader/prowloader/bigqueryjobs.go +++ b/pkg/dataloader/prowloader/bigqueryjobs.go @@ -2,6 +2,7 @@ package prowloader import ( "strconv" + "strings" "time" "cloud.google.com/go/bigquery" @@ -43,6 +44,7 @@ func (pl *ProwLoader) fetchProwJobsFromOpenShiftBigQuery() ([]prow.ProwJob, []er prowjob_type, prowjob_cluster, prowjob_url, + prowjob_annotations, pr_sha, pr_author, pr_number, @@ -103,6 +105,21 @@ func (pl *ProwLoader) fetchProwJobsFromOpenShiftBigQuery() ([]prow.ProwJob, []er // Do not return an error as that will cause the job to fail. continue } + // Filter out annotations with excluded prefixes + filteredAnnotations := make(map[string]string) + for _, a := range bqjr.Annotations { + parts := strings.SplitN(a, "=", 2) + if len(parts) != 2 { + continue + } + key := parts[0] + value := parts[1] + if strings.HasPrefix(key, "prow.k8s.io") || strings.HasPrefix(key, "ci.openshift.io") { + continue + } + filteredAnnotations[key] = value + } + prowJobs[bqjr.BuildID] = prow.ProwJob{ Spec: prow.ProwJobSpec{ Type: bqjr.Type, @@ -122,6 +139,7 @@ func (pl *ProwLoader) fetchProwJobsFromOpenShiftBigQuery() ([]prow.ProwJob, []er URL: bqjr.URL, BuildID: bqjr.BuildID, }, + Annotations: filteredAnnotations, } count++ } @@ -152,4 +170,5 @@ type bigqueryProwJobRun struct { PROrg bigquery.NullString `bigquery:"org"` PRRepo bigquery.NullString `bigquery:"repo"` GCSBucket bigquery.NullString `bigquery:"gcs_bucket"` + Annotations []string `bigquery:"prowjob_annotations"` } diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 21be20d4d5..1031b780cd 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -921,6 +921,14 @@ func (pl *ProwLoader) processGCSBucketJobRun(ctx context.Context, pj *prow.ProwJ return err } + var annotations []models.ProwJobRunAnnotation + for k, v := range pj.Annotations { + annotations = append(annotations, models.ProwJobRunAnnotation{ + Key: k, + Value: v, + }) + } + var duration time.Duration if pj.Status.CompletionTime != nil { duration = pj.Status.CompletionTime.Sub(pj.Status.StartTime) @@ -942,6 +950,7 @@ func (pl *ProwLoader) processGCSBucketJobRun(ctx context.Context, pj *prow.ProwJ TestFailures: failures, Succeeded: overallResult == sippyprocessingv1.JobSucceeded, Labels: labels, + Annotations: annotations, }).Error if err != nil { return err diff --git a/pkg/db/db.go b/pkg/db/db.go index 6ecf1b8cab..849185b527 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -74,6 +74,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { &models.ReleaseJobRun{}, &models.ProwJob{}, &models.ProwJobRun{}, + &models.ProwJobRunAnnotation{}, &models.Test{}, &models.Suite{}, &models.ProwJobRunTest{}, diff --git a/pkg/db/models/prow.go b/pkg/db/models/prow.go index 3df1991582..18fd49e037 100644 --- a/pkg/db/models/prow.go +++ b/pkg/db/models/prow.go @@ -46,8 +46,9 @@ type ProwJobRun struct { GCSBucket string URL string TestFailures int - Tests []ProwJobRunTest `gorm:"constraint:OnDelete:CASCADE;"` - PullRequests []ProwPullRequest `gorm:"many2many:prow_job_run_prow_pull_requests;constraint:OnDelete:CASCADE;"` + Tests []ProwJobRunTest `gorm:"constraint:OnDelete:CASCADE;"` + PullRequests []ProwPullRequest `gorm:"many2many:prow_job_run_prow_pull_requests;constraint:OnDelete:CASCADE;"` + Annotations []ProwJobRunAnnotation `gorm:"constraint:OnDelete:CASCADE;"` Failed bool // InfrastructureFailure is true if the job run failed, for reasons which appear to be related to test/CI infra. InfrastructureFailure bool @@ -65,6 +66,14 @@ type ProwJobRun struct { ClusterData ClusterData `gorm:"-"` } +// ProwJobRunAnnotation stores a single key-value annotation for a ProwJobRun. +type ProwJobRunAnnotation struct { + gorm.Model + ProwJobRunID uint `gorm:"index;uniqueIndex:idx_prow_job_run_annotations_key"` + Key string `gorm:"uniqueIndex:idx_prow_job_run_annotations_key"` + Value string +} + type Test struct { gorm.Model Name string `gorm:"uniqueIndex"`