From f6090ad847f46019af91a43a9eeb6d7acc426aeb Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Thu, 12 Mar 2026 18:49:15 +0000 Subject: [PATCH 1/6] Initial plan From dcfa5655d268d9aa7f5bcb932edf5b33925b3673 Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Thu, 12 Mar 2026 18:55:20 +0000 Subject: [PATCH 2/6] Add gh devlake query command with pipelines subcommand - Created `query` parent command with three subcommands - Implemented `query pipelines` with full JSON/table output - Added `ListPipelines` method to devlake client - Created placeholder `query dora` and `query copilot` commands - Updated README command reference table - Added docs/query.md documentation Note: DORA and Copilot subcommands are placeholders that explain the architectural constraint (DevLake does not expose metrics APIs). The pipelines subcommand is fully functional using the existing /pipelines REST API endpoint. Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- README.md | 3 + cmd/query.go | 25 +++++++ cmd/query_copilot.go | 68 +++++++++++++++++ cmd/query_dora.go | 62 +++++++++++++++ cmd/query_pipelines.go | 138 ++++++++++++++++++++++++++++++++++ docs/query.md | 149 +++++++++++++++++++++++++++++++++++++ internal/devlake/client.go | 32 ++++++++ internal/devlake/types.go | 7 ++ 8 files changed, 484 insertions(+) create mode 100644 cmd/query.go create mode 100644 cmd/query_copilot.go create mode 100644 cmd/query_dora.go create mode 100644 cmd/query_pipelines.go create mode 100644 docs/query.md diff --git a/README.md b/README.md index 35d4d2b..7698fb1 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,9 @@ See [Token Handling](docs/token-handling.md) for env key names and multi-plugin | `gh devlake configure project list` | List all projects | [configure-project.md](docs/configure-project.md) | | `gh devlake configure project delete` | Delete a project | [configure-project.md](docs/configure-project.md) | | `gh devlake configure full` | Connections + scopes + project in one step | [configure-full.md](docs/configure-full.md) | +| `gh devlake query pipelines` | Query recent pipeline runs | [query.md](docs/query.md) | +| `gh devlake query dora` | Query DORA metrics (placeholder — requires API) | [query.md](docs/query.md) | +| `gh devlake query copilot` | Query Copilot metrics (placeholder — requires API) | [query.md](docs/query.md) | | `gh devlake start` | Start stopped or exited DevLake services | [start.md](docs/start.md) | | `gh devlake stop` | Stop running services (preserves containers and data) | [stop.md](docs/stop.md) | | `gh devlake cleanup` | Tear down local or Azure resources | [cleanup.md](docs/cleanup.md) | diff --git a/cmd/query.go b/cmd/query.go new file mode 100644 index 0000000..da0d34b --- /dev/null +++ b/cmd/query.go @@ -0,0 +1,25 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +var queryCmd = &cobra.Command{ + Use: "query", + Short: "Query DevLake data and metrics", + Long: `Query DevLake's aggregated data and metrics. + +Retrieve DORA metrics, Copilot usage data, pipeline status, and other +metrics in a structured format (JSON by default, --format table for +human-readable output). + +Examples: + gh devlake query pipelines --project my-team + gh devlake query pipelines --limit 20 + gh devlake query pipelines --status TASK_COMPLETED`, +} + +func init() { + queryCmd.GroupID = "operate" + rootCmd.AddCommand(queryCmd) +} diff --git a/cmd/query_copilot.go b/cmd/query_copilot.go new file mode 100644 index 0000000..399f9bf --- /dev/null +++ b/cmd/query_copilot.go @@ -0,0 +1,68 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var ( + queryCopilotProject string + queryCopilotTimeframe string +) + +var queryCopilotCmd = &cobra.Command{ + Use: "copilot", + Short: "Query Copilot usage metrics (requires DevLake metrics API)", + Long: `Query GitHub Copilot usage metrics for a project. + +NOTE: This command requires DevLake to expose a metrics API endpoint. +Currently, Copilot metrics are stored in the gh-copilot plugin tables +and visualized in Grafana dashboards, but not available via the REST API. + +To view Copilot metrics today, use the Grafana dashboards at your DevLake +Grafana endpoint (shown in 'gh devlake status'). + +Planned output format: +{ + "project": "my-team", + "timeframe": "30d", + "metrics": { + "totalSeats": 45, + "activeUsers": 38, + "acceptanceRate": 0.34, + "topLanguages": [ + { "language": "TypeScript", "acceptances": 1200, "suggestions": 3500 } + ], + "topEditors": [ + { "editor": "vscode", "users": 30 } + ] + } +}`, + RunE: runQueryCopilot, +} + +func init() { + queryCopilotCmd.Flags().StringVar(&queryCopilotProject, "project", "", "Project name (required)") + queryCopilotCmd.Flags().StringVar(&queryCopilotTimeframe, "timeframe", "30d", "Time window for metrics (e.g., 7d, 30d, 90d)") + queryCopilotCmd.MarkFlagRequired("project") + queryCmd.AddCommand(queryCopilotCmd) +} + +func runQueryCopilot(cmd *cobra.Command, args []string) error { + return fmt.Errorf(`Copilot metrics query is not yet implemented. + +DevLake does not currently expose a metrics API endpoint. Copilot metrics are +stored in the gh-copilot plugin's database tables and visualized in Grafana +dashboards, but not accessible via the REST API. + +To view Copilot metrics, visit your Grafana endpoint (shown in 'gh devlake status') +and navigate to the Copilot dashboards. + +Future implementation will require: + 1. Upstream DevLake metrics API endpoint for Copilot plugin + 2. OR direct database query support (requires DB credentials) + 3. OR Grafana API integration to fetch dashboard data + +Track progress at: https://github.com/DevExpGBB/gh-devlake/issues`) +} diff --git a/cmd/query_dora.go b/cmd/query_dora.go new file mode 100644 index 0000000..52b24e9 --- /dev/null +++ b/cmd/query_dora.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var ( + queryDoraProject string + queryDoraTimeframe string +) + +var queryDoraCmd = &cobra.Command{ + Use: "dora", + Short: "Query DORA metrics (requires DevLake metrics API)", + Long: `Query DORA (DevOps Research and Assessment) metrics for a project. + +NOTE: This command requires DevLake to expose a metrics API endpoint. +Currently, DORA metrics are calculated in Grafana dashboards but not +available via the REST API. This is a placeholder for future enhancement. + +To view DORA metrics today, use the Grafana dashboards at your DevLake +Grafana endpoint (shown in 'gh devlake status'). + +Planned output format: +{ + "project": "my-team", + "timeframe": "30d", + "metrics": { + "deploymentFrequency": { "value": 4.2, "unit": "per_week", "rating": "high" }, + "leadTimeForChanges": { "value": 2.3, "unit": "hours", "rating": "elite" }, + "changeFailureRate": { "value": 0.08, "unit": "ratio", "rating": "high" }, + "meanTimeToRestore": { "value": 1.5, "unit": "hours", "rating": "elite" } + } +}`, + RunE: runQueryDora, +} + +func init() { + queryDoraCmd.Flags().StringVar(&queryDoraProject, "project", "", "Project name (required)") + queryDoraCmd.Flags().StringVar(&queryDoraTimeframe, "timeframe", "30d", "Time window for metrics (e.g., 7d, 30d, 90d)") + queryDoraCmd.MarkFlagRequired("project") + queryCmd.AddCommand(queryDoraCmd) +} + +func runQueryDora(cmd *cobra.Command, args []string) error { + return fmt.Errorf(`DORA metrics query is not yet implemented. + +DevLake does not currently expose a metrics API endpoint. DORA metrics are +calculated in Grafana dashboards using SQL queries against the domain layer. + +To view DORA metrics, visit your Grafana endpoint (shown in 'gh devlake status') +and navigate to the DORA dashboards. + +Future implementation will require: + 1. Upstream DevLake metrics API endpoint + 2. OR direct database query support (requires DB credentials) + 3. OR Grafana API integration to fetch dashboard data + +Track progress at: https://github.com/DevExpGBB/gh-devlake/issues`) +} diff --git a/cmd/query_pipelines.go b/cmd/query_pipelines.go new file mode 100644 index 0000000..5bab93c --- /dev/null +++ b/cmd/query_pipelines.go @@ -0,0 +1,138 @@ +package cmd + +import ( + "fmt" + "strings" + + "github.com/DevExpGBB/gh-devlake/internal/devlake" + "github.com/spf13/cobra" +) + +var ( + queryPipelinesProject string + queryPipelinesStatus string + queryPipelinesLimit int + queryPipelinesFormat string +) + +var queryPipelinesCmd = &cobra.Command{ + Use: "pipelines", + Short: "Query recent pipeline runs", + Long: `Query recent pipeline runs for a project or across all projects. + +Retrieves pipeline execution history with status, timing, and task completion +information. Output is JSON by default; use --format table for human-readable display. + +Examples: + gh devlake query pipelines + gh devlake query pipelines --project my-team + gh devlake query pipelines --status TASK_COMPLETED --limit 10 + gh devlake query pipelines --format table`, + RunE: runQueryPipelines, +} + +func init() { + queryPipelinesCmd.Flags().StringVar(&queryPipelinesProject, "project", "", "Filter by project name") + queryPipelinesCmd.Flags().StringVar(&queryPipelinesStatus, "status", "", "Filter by status (TASK_CREATED, TASK_RUNNING, TASK_COMPLETED, TASK_FAILED)") + queryPipelinesCmd.Flags().IntVar(&queryPipelinesLimit, "limit", 20, "Maximum number of pipelines to return") + queryPipelinesCmd.Flags().StringVar(&queryPipelinesFormat, "format", "json", "Output format (json or table)") + queryCmd.AddCommand(queryPipelinesCmd) +} + +type pipelineQueryResult struct { + ID int `json:"id"` + Status string `json:"status"` + BlueprintID int `json:"blueprintId,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + BeganAt string `json:"beganAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` + FinishedTasks int `json:"finishedTasks"` + TotalTasks int `json:"totalTasks"` + Message string `json:"message,omitempty"` +} + +func runQueryPipelines(cmd *cobra.Command, args []string) error { + // Discover DevLake instance + var disc *devlake.DiscoveryResult + var client *devlake.Client + var err error + + if !outputJSON && queryPipelinesFormat != "table" { + client, disc, err = discoverClient(cfgURL) + if err != nil { + return fmt.Errorf("discovering DevLake: %w", err) + } + } else { + // Quiet discovery for JSON/table output + disc, err = devlake.Discover(cfgURL) + if err != nil { + return fmt.Errorf("discovering DevLake: %w", err) + } + client = devlake.NewClient(disc.URL) + } + + // If --project is specified, resolve it to a blueprint ID + var blueprintID int + if queryPipelinesProject != "" { + proj, err := client.GetProject(queryPipelinesProject) + if err != nil { + return fmt.Errorf("getting project %q: %w", queryPipelinesProject, err) + } + if proj.Blueprint != nil { + blueprintID = proj.Blueprint.ID + } else { + return fmt.Errorf("project %q has no blueprint", queryPipelinesProject) + } + } + + // Query pipelines + resp, err := client.ListPipelines(queryPipelinesStatus, blueprintID, 1, queryPipelinesLimit) + if err != nil { + return fmt.Errorf("listing pipelines: %w", err) + } + + // Transform to output format + results := make([]pipelineQueryResult, len(resp.Pipelines)) + for i, p := range resp.Pipelines { + results[i] = pipelineQueryResult{ + ID: p.ID, + Status: p.Status, + BlueprintID: p.BlueprintID, + CreatedAt: p.CreatedAt, + BeganAt: p.BeganAt, + FinishedAt: p.FinishedAt, + FinishedTasks: p.FinishedTasks, + TotalTasks: p.TotalTasks, + Message: p.Message, + } + } + + // Output + if outputJSON || queryPipelinesFormat == "json" { + return printJSON(results) + } + + // Table format + printBanner("DevLake — Pipeline Query") + if len(results) == 0 { + fmt.Println("\n No pipelines found.") + return nil + } + + fmt.Printf("\n Found %d pipeline(s)\n", len(results)) + fmt.Println(" " + strings.Repeat("─", 80)) + fmt.Printf(" %-6s %-15s %-10s %-20s\n", "ID", "STATUS", "TASKS", "FINISHED AT") + fmt.Println(" " + strings.Repeat("─", 80)) + for _, r := range results { + status := r.Status + tasks := fmt.Sprintf("%d/%d", r.FinishedTasks, r.TotalTasks) + finished := r.FinishedAt + if finished == "" { + finished = "(running)" + } + fmt.Printf(" %-6d %-15s %-10s %-20s\n", r.ID, status, tasks, finished) + } + fmt.Println() + + return nil +} diff --git a/docs/query.md b/docs/query.md new file mode 100644 index 0000000..e10a637 --- /dev/null +++ b/docs/query.md @@ -0,0 +1,149 @@ +# gh devlake query + +Query DevLake's aggregated data and metrics. + +## Usage + +```bash +gh devlake query [flags] +``` + +## Subcommands + +### pipelines + +Query recent pipeline runs. + +```bash +gh devlake query pipelines [flags] +``` + +**Flags:** +- `--project ` - Filter by project name +- `--status ` - Filter by status (`TASK_CREATED`, `TASK_RUNNING`, `TASK_COMPLETED`, `TASK_FAILED`) +- `--limit ` - Maximum number of pipelines to return (default: 20) +- `--format ` - Output format: `json` or `table` (default: `json`) + +**Examples:** + +```bash +# List recent pipelines as JSON +gh devlake query pipelines + +# List pipelines for a specific project +gh devlake query pipelines --project my-team + +# List only completed pipelines +gh devlake query pipelines --status TASK_COMPLETED --limit 10 + +# Display as table +gh devlake query pipelines --format table +``` + +**Output (JSON):** + +```json +[ + { + "id": 123, + "status": "TASK_COMPLETED", + "blueprintId": 1, + "createdAt": "2026-03-12T10:00:00Z", + "beganAt": "2026-03-12T10:00:05Z", + "finishedAt": "2026-03-12T10:15:30Z", + "finishedTasks": 12, + "totalTasks": 12 + } +] +``` + +**Output (Table):** + +``` +════════════════════════════════════════ + DevLake — Pipeline Query +════════════════════════════════════════ + + Found 3 pipeline(s) + ──────────────────────────────────────────────────────────────────────────────── + ID STATUS TASKS FINISHED AT + ──────────────────────────────────────────────────────────────────────────────── + 123 TASK_COMPLETED 12/12 2026-03-12T10:15:30Z + 122 TASK_COMPLETED 12/12 2026-03-12T09:15:30Z + 121 TASK_RUNNING 8/12 (running) +``` + +--- + +### dora + +Query DORA (DevOps Research and Assessment) metrics. + +```bash +gh devlake query dora --project [flags] +``` + +**Status:** 🚧 Not yet implemented + +**Reason:** DevLake does not currently expose a metrics API endpoint. DORA metrics are calculated in Grafana dashboards using SQL queries against the domain layer tables, but these calculations are not available via the REST API. + +**Workaround:** View DORA metrics in your Grafana dashboards: +```bash +gh devlake status # Shows Grafana URL +``` + +Then navigate to the DORA dashboards in Grafana. + +**Future implementation requires:** +1. Upstream DevLake metrics API endpoint +2. OR direct database query support (requires DB credentials in state files) +3. OR Grafana API integration to fetch dashboard data + +--- + +### copilot + +Query GitHub Copilot usage metrics. + +```bash +gh devlake query copilot --project [flags] +``` + +**Status:** 🚧 Not yet implemented + +**Reason:** DevLake does not currently expose a metrics API endpoint. Copilot metrics are stored in the `gh-copilot` plugin's database tables and visualized in Grafana dashboards, but not accessible via the REST API. + +**Workaround:** View Copilot metrics in your Grafana dashboards: +```bash +gh devlake status # Shows Grafana URL +``` + +Then navigate to the Copilot dashboards in Grafana. + +**Future implementation requires:** +1. Upstream DevLake metrics API endpoint for Copilot plugin +2. OR direct database query support (requires DB credentials in state files) +3. OR Grafana API integration to fetch dashboard data + +--- + +## Global Flags + +These flags are inherited from the root command: + +- `--url ` - DevLake API base URL (auto-discovered if omitted) +- `--json` - Output as JSON (suppresses banners and interactive prompts) + +## Architecture Notes + +The `query` command is designed to be extensible: + +- **Current:** The `pipelines` subcommand uses the existing `/pipelines` REST API endpoint +- **Future:** The `dora` and `copilot` subcommands are placeholders awaiting upstream API support + +When DevLake exposes metrics APIs, the existing command structure will remain the same — only the implementation will change from returning an error to fetching actual metrics. + +## See Also + +- `gh devlake status` - Check DevLake deployment and connection status +- `gh devlake configure project list` - List all projects diff --git a/internal/devlake/client.go b/internal/devlake/client.go index 3cb2617..61020ed 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strings" "time" ) @@ -512,3 +513,34 @@ func (c *Client) TriggerMigration() error { resp.Body.Close() return nil } + +// PipelineListResponse is the response from GET /pipelines. +type PipelineListResponse struct { + Pipelines []Pipeline `json:"pipelines"` + Count int64 `json:"count"` +} + +// ListPipelines returns pipelines with optional query parameters. +// status can be empty, "TASK_CREATED", "TASK_RUNNING", "TASK_COMPLETED", "TASK_FAILED", etc. +// blueprintID filters by blueprint (0 = no filter). +// page and pageSize control pagination (0 = use defaults). +func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { + path := "/pipelines?" + params := []string{} + if status != "" { + params = append(params, "status="+status) + } + if blueprintID > 0 { + params = append(params, fmt.Sprintf("blueprint_id=%d", blueprintID)) + } + if page > 0 { + params = append(params, fmt.Sprintf("page=%d", page)) + } + if pageSize > 0 { + params = append(params, fmt.Sprintf("pagesize=%d", pageSize)) + } + if len(params) > 0 { + path += strings.Join(params, "&") + } + return doGet[PipelineListResponse](c, path) +} diff --git a/internal/devlake/types.go b/internal/devlake/types.go index 9e60540..feaa397 100644 --- a/internal/devlake/types.go +++ b/internal/devlake/types.go @@ -271,7 +271,14 @@ type BlueprintScope struct { // Pipeline represents a DevLake pipeline (returned from trigger or GET). type Pipeline struct { ID int `json:"id"` + CreatedAt string `json:"createdAt,omitempty"` + UpdatedAt string `json:"updatedAt,omitempty"` + BeganAt string `json:"beganAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` Status string `json:"status"` + Message string `json:"message,omitempty"` FinishedTasks int `json:"finishedTasks"` TotalTasks int `json:"totalTasks"` + BlueprintID int `json:"blueprintId,omitempty"` + SkipOnFail bool `json:"skipOnFail,omitempty"` } From 811b4982f7e339cf1f84c9fbcfaa28de66e9dd73 Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:15:56 +0000 Subject: [PATCH 3/6] Refactor query commands to use internal/query abstraction - Move query logic from cmd handlers to internal/query package - Implement DORA query with available API data + clear limitations - Implement Copilot query with available API data + clear limitations - Keep pipelines query functional through the new abstraction - All validation passes: go build, go test, go vet Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/query_copilot.go | 74 ++++++++++++++++--------------- cmd/query_dora.go | 71 +++++++++++++++++------------- cmd/query_pipelines.go | 79 +++++++++++++-------------------- internal/query/copilot.go | 78 +++++++++++++++++++++++++++++++++ internal/query/dora.go | 74 +++++++++++++++++++++++++++++++ internal/query/engine.go | 51 ++++++++++++++++++++++ internal/query/pipelines.go | 87 +++++++++++++++++++++++++++++++++++++ internal/query/registry.go | 32 ++++++++++++++ internal/query/types.go | 36 +++++++++++++++ 9 files changed, 470 insertions(+), 112 deletions(-) create mode 100644 internal/query/copilot.go create mode 100644 internal/query/dora.go create mode 100644 internal/query/engine.go create mode 100644 internal/query/pipelines.go create mode 100644 internal/query/registry.go create mode 100644 internal/query/types.go diff --git a/cmd/query_copilot.go b/cmd/query_copilot.go index 399f9bf..0bd5fb0 100644 --- a/cmd/query_copilot.go +++ b/cmd/query_copilot.go @@ -3,6 +3,8 @@ package cmd import ( "fmt" + "github.com/DevExpGBB/gh-devlake/internal/devlake" + "github.com/DevExpGBB/gh-devlake/internal/query" "github.com/spf13/cobra" ) @@ -13,56 +15,60 @@ var ( var queryCopilotCmd = &cobra.Command{ Use: "copilot", - Short: "Query Copilot usage metrics (requires DevLake metrics API)", + Short: "Query Copilot usage metrics (limited by available API data)", Long: `Query GitHub Copilot usage metrics for a project. -NOTE: This command requires DevLake to expose a metrics API endpoint. -Currently, Copilot metrics are stored in the gh-copilot plugin tables -and visualized in Grafana dashboards, but not available via the REST API. +NOTE: GitHub Copilot usage metrics (total seats, active users, acceptance rates, +language breakdowns, editor usage) are stored in _tool_gh_copilot_* tables and +visualized in Grafana dashboards, but DevLake does not expose a /metrics or +/copilot API endpoint. -To view Copilot metrics today, use the Grafana dashboards at your DevLake -Grafana endpoint (shown in 'gh devlake status'). +This command returns available connection metadata and explains what additional +API endpoints would be needed to retrieve Copilot metrics via CLI. -Planned output format: -{ - "project": "my-team", - "timeframe": "30d", - "metrics": { - "totalSeats": 45, - "activeUsers": 38, - "acceptanceRate": 0.34, - "topLanguages": [ - { "language": "TypeScript", "acceptances": 1200, "suggestions": 3500 } - ], - "topEditors": [ - { "editor": "vscode", "users": 30 } - ] - } -}`, +Copilot metrics are currently available in Grafana dashboards at your DevLake +Grafana endpoint (shown in 'gh devlake status').`, RunE: runQueryCopilot, } func init() { queryCopilotCmd.Flags().StringVar(&queryCopilotProject, "project", "", "Project name (required)") queryCopilotCmd.Flags().StringVar(&queryCopilotTimeframe, "timeframe", "30d", "Time window for metrics (e.g., 7d, 30d, 90d)") - queryCopilotCmd.MarkFlagRequired("project") queryCmd.AddCommand(queryCopilotCmd) } func runQueryCopilot(cmd *cobra.Command, args []string) error { - return fmt.Errorf(`Copilot metrics query is not yet implemented. + // Validate project flag + if queryCopilotProject == "" { + return fmt.Errorf("--project flag is required") + } -DevLake does not currently expose a metrics API endpoint. Copilot metrics are -stored in the gh-copilot plugin's database tables and visualized in Grafana -dashboards, but not accessible via the REST API. + // Discover DevLake instance + disc, err := devlake.Discover(cfgURL) + if err != nil { + return fmt.Errorf("discovering DevLake: %w", err) + } + client := devlake.NewClient(disc.URL) -To view Copilot metrics, visit your Grafana endpoint (shown in 'gh devlake status') -and navigate to the Copilot dashboards. + // Get the query definition + queryDef, err := query.Get("copilot") + if err != nil { + return fmt.Errorf("getting copilot query: %w", err) + } -Future implementation will require: - 1. Upstream DevLake metrics API endpoint for Copilot plugin - 2. OR direct database query support (requires DB credentials) - 3. OR Grafana API integration to fetch dashboard data + // Build parameters + params := map[string]interface{}{ + "project": queryCopilotProject, + "timeframe": queryCopilotTimeframe, + } -Track progress at: https://github.com/DevExpGBB/gh-devlake/issues`) + // Execute the query + engine := query.NewEngine(client) + result, err := engine.Execute(queryDef, params) + if err != nil { + return fmt.Errorf("executing copilot query: %w", err) + } + + // Output result as JSON + return printJSON(result) } diff --git a/cmd/query_dora.go b/cmd/query_dora.go index 52b24e9..0e1cc1c 100644 --- a/cmd/query_dora.go +++ b/cmd/query_dora.go @@ -3,6 +3,8 @@ package cmd import ( "fmt" + "github.com/DevExpGBB/gh-devlake/internal/devlake" + "github.com/DevExpGBB/gh-devlake/internal/query" "github.com/spf13/cobra" ) @@ -13,50 +15,59 @@ var ( var queryDoraCmd = &cobra.Command{ Use: "dora", - Short: "Query DORA metrics (requires DevLake metrics API)", + Short: "Query DORA metrics (limited by available API data)", Long: `Query DORA (DevOps Research and Assessment) metrics for a project. -NOTE: This command requires DevLake to expose a metrics API endpoint. -Currently, DORA metrics are calculated in Grafana dashboards but not -available via the REST API. This is a placeholder for future enhancement. - -To view DORA metrics today, use the Grafana dashboards at your DevLake -Grafana endpoint (shown in 'gh devlake status'). - -Planned output format: -{ - "project": "my-team", - "timeframe": "30d", - "metrics": { - "deploymentFrequency": { "value": 4.2, "unit": "per_week", "rating": "high" }, - "leadTimeForChanges": { "value": 2.3, "unit": "hours", "rating": "elite" }, - "changeFailureRate": { "value": 0.08, "unit": "ratio", "rating": "high" }, - "meanTimeToRestore": { "value": 1.5, "unit": "hours", "rating": "elite" } - } -}`, +NOTE: Full DORA metric calculations (deployment frequency, lead time, change +failure rate, MTTR) require SQL queries against DevLake's domain layer tables. +DevLake does not expose database credentials or a metrics API endpoint. + +This command returns project metadata and explains what additional API +endpoints would be needed to compute DORA metrics via CLI. + +DORA metrics are currently available in Grafana dashboards at your DevLake +Grafana endpoint (shown in 'gh devlake status').`, RunE: runQueryDora, } func init() { queryDoraCmd.Flags().StringVar(&queryDoraProject, "project", "", "Project name (required)") queryDoraCmd.Flags().StringVar(&queryDoraTimeframe, "timeframe", "30d", "Time window for metrics (e.g., 7d, 30d, 90d)") - queryDoraCmd.MarkFlagRequired("project") queryCmd.AddCommand(queryDoraCmd) } func runQueryDora(cmd *cobra.Command, args []string) error { - return fmt.Errorf(`DORA metrics query is not yet implemented. + // Validate project flag + if queryDoraProject == "" { + return fmt.Errorf("--project flag is required") + } + + // Discover DevLake instance + disc, err := devlake.Discover(cfgURL) + if err != nil { + return fmt.Errorf("discovering DevLake: %w", err) + } + client := devlake.NewClient(disc.URL) -DevLake does not currently expose a metrics API endpoint. DORA metrics are -calculated in Grafana dashboards using SQL queries against the domain layer. + // Get the query definition + queryDef, err := query.Get("dora") + if err != nil { + return fmt.Errorf("getting dora query: %w", err) + } -To view DORA metrics, visit your Grafana endpoint (shown in 'gh devlake status') -and navigate to the DORA dashboards. + // Build parameters + params := map[string]interface{}{ + "project": queryDoraProject, + "timeframe": queryDoraTimeframe, + } -Future implementation will require: - 1. Upstream DevLake metrics API endpoint - 2. OR direct database query support (requires DB credentials) - 3. OR Grafana API integration to fetch dashboard data + // Execute the query + engine := query.NewEngine(client) + result, err := engine.Execute(queryDef, params) + if err != nil { + return fmt.Errorf("executing dora query: %w", err) + } -Track progress at: https://github.com/DevExpGBB/gh-devlake/issues`) + // Output result as JSON + return printJSON(result) } diff --git a/cmd/query_pipelines.go b/cmd/query_pipelines.go index 5bab93c..558f26c 100644 --- a/cmd/query_pipelines.go +++ b/cmd/query_pipelines.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/DevExpGBB/gh-devlake/internal/devlake" + "github.com/DevExpGBB/gh-devlake/internal/query" "github.com/spf13/cobra" ) @@ -39,18 +40,6 @@ func init() { queryCmd.AddCommand(queryPipelinesCmd) } -type pipelineQueryResult struct { - ID int `json:"id"` - Status string `json:"status"` - BlueprintID int `json:"blueprintId,omitempty"` - CreatedAt string `json:"createdAt,omitempty"` - BeganAt string `json:"beganAt,omitempty"` - FinishedAt string `json:"finishedAt,omitempty"` - FinishedTasks int `json:"finishedTasks"` - TotalTasks int `json:"totalTasks"` - Message string `json:"message,omitempty"` -} - func runQueryPipelines(cmd *cobra.Command, args []string) error { // Discover DevLake instance var disc *devlake.DiscoveryResult @@ -71,66 +60,60 @@ func runQueryPipelines(cmd *cobra.Command, args []string) error { client = devlake.NewClient(disc.URL) } - // If --project is specified, resolve it to a blueprint ID - var blueprintID int + // Get the query definition + queryDef, err := query.Get("pipelines") + if err != nil { + return fmt.Errorf("getting pipelines query: %w", err) + } + + // Build parameters + params := map[string]interface{}{ + "limit": queryPipelinesLimit, + } if queryPipelinesProject != "" { - proj, err := client.GetProject(queryPipelinesProject) - if err != nil { - return fmt.Errorf("getting project %q: %w", queryPipelinesProject, err) - } - if proj.Blueprint != nil { - blueprintID = proj.Blueprint.ID - } else { - return fmt.Errorf("project %q has no blueprint", queryPipelinesProject) - } + params["project"] = queryPipelinesProject + } + if queryPipelinesStatus != "" { + params["status"] = queryPipelinesStatus } - // Query pipelines - resp, err := client.ListPipelines(queryPipelinesStatus, blueprintID, 1, queryPipelinesLimit) + // Execute the query + engine := query.NewEngine(client) + result, err := engine.Execute(queryDef, params) if err != nil { - return fmt.Errorf("listing pipelines: %w", err) + return fmt.Errorf("executing pipelines query: %w", err) } - // Transform to output format - results := make([]pipelineQueryResult, len(resp.Pipelines)) - for i, p := range resp.Pipelines { - results[i] = pipelineQueryResult{ - ID: p.ID, - Status: p.Status, - BlueprintID: p.BlueprintID, - CreatedAt: p.CreatedAt, - BeganAt: p.BeganAt, - FinishedAt: p.FinishedAt, - FinishedTasks: p.FinishedTasks, - TotalTasks: p.TotalTasks, - Message: p.Message, - } + // Cast result to slice of PipelineResult + pipelines, ok := result.([]query.PipelineResult) + if !ok { + return fmt.Errorf("unexpected result type: %T", result) } // Output if outputJSON || queryPipelinesFormat == "json" { - return printJSON(results) + return printJSON(pipelines) } // Table format printBanner("DevLake — Pipeline Query") - if len(results) == 0 { + if len(pipelines) == 0 { fmt.Println("\n No pipelines found.") return nil } - fmt.Printf("\n Found %d pipeline(s)\n", len(results)) + fmt.Printf("\n Found %d pipeline(s)\n", len(pipelines)) fmt.Println(" " + strings.Repeat("─", 80)) fmt.Printf(" %-6s %-15s %-10s %-20s\n", "ID", "STATUS", "TASKS", "FINISHED AT") fmt.Println(" " + strings.Repeat("─", 80)) - for _, r := range results { - status := r.Status - tasks := fmt.Sprintf("%d/%d", r.FinishedTasks, r.TotalTasks) - finished := r.FinishedAt + for _, p := range pipelines { + status := p.Status + tasks := fmt.Sprintf("%d/%d", p.FinishedTasks, p.TotalTasks) + finished := p.FinishedAt if finished == "" { finished = "(running)" } - fmt.Printf(" %-6d %-15s %-10s %-20s\n", r.ID, status, tasks, finished) + fmt.Printf(" %-6d %-15s %-10s %-20s\n", p.ID, status, tasks, finished) } fmt.Println() diff --git a/internal/query/copilot.go b/internal/query/copilot.go new file mode 100644 index 0000000..6eb1c5b --- /dev/null +++ b/internal/query/copilot.go @@ -0,0 +1,78 @@ +package query + +import ( + "fmt" + + "github.com/DevExpGBB/gh-devlake/internal/devlake" +) + +func init() { + Register(copilotQueryDef) +} + +var copilotQueryDef = &QueryDef{ + Name: "copilot", + Description: "Query GitHub Copilot metrics (limited by available API data)", + Params: []QueryParam{ + {Name: "project", Type: "string", Required: true}, + {Name: "timeframe", Type: "string", Required: false, Default: "30d"}, + }, + Execute: executeCopilotQuery, +} + +// CopilotResult represents Copilot metrics that can be retrieved from available APIs. +// NOTE: Copilot usage metrics (acceptance rates, language breakdowns) are stored in +// _tool_gh_copilot_* tables but not exposed via REST API. +type CopilotResult struct { + Project string `json:"project"` + Timeframe string `json:"timeframe"` + AvailableData map[string]interface{} `json:"availableData"` + Limitations string `json:"limitations"` +} + +func executeCopilotQuery(client *devlake.Client, params map[string]interface{}) (interface{}, error) { + projectName, ok := params["project"].(string) + if !ok || projectName == "" { + return nil, fmt.Errorf("project parameter is required") + } + + timeframe := "30d" + if tf, ok := params["timeframe"].(string); ok && tf != "" { + timeframe = tf + } + + // Get project info + proj, err := client.GetProject(projectName) + if err != nil { + return nil, fmt.Errorf("getting project %q: %w", projectName, err) + } + + // Check if gh-copilot plugin is configured + connections, err := client.ListConnections("gh-copilot") + if err != nil { + // Plugin might not be available + connections = []devlake.Connection{} + } + + availableData := map[string]interface{}{ + "projectName": proj.Name, + "copilotConnectionsFound": len(connections), + } + + if len(connections) > 0 { + availableData["connections"] = connections + } + + result := CopilotResult{ + Project: projectName, + Timeframe: timeframe, + AvailableData: availableData, + Limitations: "GitHub Copilot usage metrics (total seats, active users, acceptance rates, language " + + "breakdowns, editor usage) are stored in _tool_gh_copilot_* tables and visualized in Grafana " + + "dashboards, but DevLake does not expose a /metrics or /copilot API endpoint. To retrieve " + + "Copilot metrics via CLI, DevLake would need to add a metrics API that returns aggregated " + + "Copilot usage data.", + } + + return result, nil +} diff --git a/internal/query/dora.go b/internal/query/dora.go new file mode 100644 index 0000000..c333409 --- /dev/null +++ b/internal/query/dora.go @@ -0,0 +1,74 @@ +package query + +import ( + "fmt" + + "github.com/DevExpGBB/gh-devlake/internal/devlake" +) + +func init() { + Register(doraQueryDef) +} + +var doraQueryDef = &QueryDef{ + Name: "dora", + Description: "Query DORA metrics (limited by available API data)", + Params: []QueryParam{ + {Name: "project", Type: "string", Required: true}, + {Name: "timeframe", Type: "string", Required: false, Default: "30d"}, + }, + Execute: executeDoraQuery, +} + +// DoraResult represents DORA metrics that can be retrieved from available APIs. +// NOTE: Full DORA calculations require direct database access (which DevLake doesn't +// expose to external clients). This returns what's available via REST API. +type DoraResult struct { + Project string `json:"project"` + Timeframe string `json:"timeframe"` + AvailableData map[string]interface{} `json:"availableData"` + Limitations string `json:"limitations"` +} + +func executeDoraQuery(client *devlake.Client, params map[string]interface{}) (interface{}, error) { + projectName, ok := params["project"].(string) + if !ok || projectName == "" { + return nil, fmt.Errorf("project parameter is required") + } + + timeframe := "30d" + if tf, ok := params["timeframe"].(string); ok && tf != "" { + timeframe = tf + } + + // Get project info (this is what's available via API) + proj, err := client.GetProject(projectName) + if err != nil { + return nil, fmt.Errorf("getting project %q: %w", projectName, err) + } + + availableData := map[string]interface{}{ + "projectName": proj.Name, + "projectDescription": proj.Description, + "enabledMetrics": proj.Metrics, + } + + if proj.Blueprint != nil { + availableData["blueprintId"] = proj.Blueprint.ID + availableData["blueprintName"] = proj.Blueprint.Name + availableData["syncSchedule"] = proj.Blueprint.CronConfig + } + + result := DoraResult{ + Project: projectName, + Timeframe: timeframe, + AvailableData: availableData, + Limitations: "DORA metric calculations (deployment frequency, lead time, change failure rate, MTTR) " + + "require SQL queries against DevLake's domain layer tables. DevLake does not expose database " + + "credentials or a metrics API endpoint. These calculations are currently only available in " + + "Grafana dashboards. To compute DORA metrics via CLI, DevLake would need to add a /metrics " + + "or /dora API endpoint that returns pre-calculated values.", + } + + return result, nil +} diff --git a/internal/query/engine.go b/internal/query/engine.go new file mode 100644 index 0000000..3f69ef4 --- /dev/null +++ b/internal/query/engine.go @@ -0,0 +1,51 @@ +package query + +import ( + "fmt" + + "github.com/DevExpGBB/gh-devlake/internal/devlake" +) + +// Engine executes queries against DevLake's REST API. +type Engine struct { + client *devlake.Client +} + +// NewEngine creates a new query engine with the given DevLake client. +func NewEngine(client *devlake.Client) *Engine { + return &Engine{ + client: client, + } +} + +// Execute runs a query with the given parameters. +func (e *Engine) Execute(queryDef *QueryDef, params map[string]interface{}) (interface{}, error) { + if queryDef == nil { + return nil, fmt.Errorf("query definition is nil") + } + if queryDef.Execute == nil { + return nil, fmt.Errorf("query %q has no execute function", queryDef.Name) + } + + // Validate required parameters + for _, param := range queryDef.Params { + if param.Required { + if _, ok := params[param.Name]; !ok { + // Check if there's a default value + if param.Default != "" { + params[param.Name] = param.Default + } else { + return nil, fmt.Errorf("required parameter %q not provided", param.Name) + } + } + } + } + + // Execute the query + return queryDef.Execute(e.client, params) +} + +// GetClient returns the underlying DevLake client. +func (e *Engine) GetClient() *devlake.Client { + return e.client +} diff --git a/internal/query/pipelines.go b/internal/query/pipelines.go new file mode 100644 index 0000000..b6663a1 --- /dev/null +++ b/internal/query/pipelines.go @@ -0,0 +1,87 @@ +package query + +import ( + "fmt" + + "github.com/DevExpGBB/gh-devlake/internal/devlake" +) + +func init() { + Register(pipelinesQueryDef) +} + +var pipelinesQueryDef = &QueryDef{ + Name: "pipelines", + Description: "Query recent pipeline runs", + Params: []QueryParam{ + {Name: "project", Type: "string", Required: false}, + {Name: "status", Type: "string", Required: false}, + {Name: "limit", Type: "int", Required: false, Default: "20"}, + }, + Execute: executePipelinesQuery, +} + +// PipelineResult represents a single pipeline query result. +type PipelineResult struct { + ID int `json:"id"` + Status string `json:"status"` + BlueprintID int `json:"blueprintId,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + BeganAt string `json:"beganAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` + FinishedTasks int `json:"finishedTasks"` + TotalTasks int `json:"totalTasks"` + Message string `json:"message,omitempty"` +} + +func executePipelinesQuery(client *devlake.Client, params map[string]interface{}) (interface{}, error) { + // Extract parameters + var blueprintID int + if projectName, ok := params["project"].(string); ok && projectName != "" { + proj, err := client.GetProject(projectName) + if err != nil { + return nil, fmt.Errorf("getting project %q: %w", projectName, err) + } + if proj.Blueprint != nil { + blueprintID = proj.Blueprint.ID + } else { + return nil, fmt.Errorf("project %q has no blueprint", projectName) + } + } + + status := "" + if s, ok := params["status"].(string); ok { + status = s + } + + limit := 20 + if l, ok := params["limit"].(int); ok { + limit = l + } else if l, ok := params["limit"].(string); ok { + fmt.Sscanf(l, "%d", &limit) + } + + // Query pipelines via API + resp, err := client.ListPipelines(status, blueprintID, 1, limit) + if err != nil { + return nil, fmt.Errorf("listing pipelines: %w", err) + } + + // Transform to output format + results := make([]PipelineResult, len(resp.Pipelines)) + for i, p := range resp.Pipelines { + results[i] = PipelineResult{ + ID: p.ID, + Status: p.Status, + BlueprintID: p.BlueprintID, + CreatedAt: p.CreatedAt, + BeganAt: p.BeganAt, + FinishedAt: p.FinishedAt, + FinishedTasks: p.FinishedTasks, + TotalTasks: p.TotalTasks, + Message: p.Message, + } + } + + return results, nil +} diff --git a/internal/query/registry.go b/internal/query/registry.go new file mode 100644 index 0000000..c899bd6 --- /dev/null +++ b/internal/query/registry.go @@ -0,0 +1,32 @@ +package query + +import "fmt" + +// registry holds all registered queries. +var registry = make(map[string]*QueryDef) + +// Register adds a query definition to the registry. +func Register(def *QueryDef) { + if def == nil || def.Name == "" { + panic("cannot register nil or unnamed query") + } + registry[def.Name] = def +} + +// Get retrieves a query definition by name. +func Get(name string) (*QueryDef, error) { + def, ok := registry[name] + if !ok { + return nil, fmt.Errorf("query %q not found", name) + } + return def, nil +} + +// List returns all registered query names. +func List() []string { + names := make([]string, 0, len(registry)) + for name := range registry { + names = append(names, name) + } + return names +} diff --git a/internal/query/types.go b/internal/query/types.go new file mode 100644 index 0000000..6756b2b --- /dev/null +++ b/internal/query/types.go @@ -0,0 +1,36 @@ +// Package query provides an extensible abstraction for querying DevLake data. +// Instead of direct SQL queries (DevLake doesn't expose DB credentials), this +// package defines queries as API endpoint patterns with client-side transformations. +package query + +import ( + "github.com/DevExpGBB/gh-devlake/internal/devlake" +) + +// QueryDef describes a reusable, parameterized query against DevLake's API. +// Unlike the original SQL-based design, this uses HTTP API endpoints since +// DevLake doesn't expose database credentials to external clients. +type QueryDef struct { + Name string // e.g. "pipelines", "dora_metrics" + Description string // human-readable description + Params []QueryParam // declared parameters with types and defaults + Execute QueryExecuteFunc // function that executes the query +} + +// QueryParam describes a parameter for a query. +type QueryParam struct { + Name string // parameter name + Type string // "string", "int", "duration" + Required bool // whether the parameter is required + Default string // default value if not provided +} + +// QueryExecuteFunc is the signature for query execution functions. +// It takes a client, parameters, and returns results or an error. +type QueryExecuteFunc func(client *devlake.Client, params map[string]interface{}) (interface{}, error) + +// QueryResult wraps the output of a query execution. +type QueryResult struct { + Data interface{} // the actual result data + Metadata map[string]string // optional metadata about the query +} From 16208129a68603c737b4ba33affef34184733b7c Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:30:35 +0000 Subject: [PATCH 4/6] Address PR feedback: fix docs, engine defaults, and limit parsing - Update docs/query.md to describe DORA/Copilot as partial implementations - Fix engine.go to apply defaults consistently for all optional params - Fix pipelines.go to return error on invalid limit parsing instead of silently ignoring - All validation passes: go build, go test, go vet Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- docs/query.md | 67 +++++++++++++++++++++++++++++-------- internal/query/engine.go | 18 +++++----- internal/query/pipelines.go | 7 +++- 3 files changed, 68 insertions(+), 24 deletions(-) diff --git a/docs/query.md b/docs/query.md index e10a637..71107e1 100644 --- a/docs/query.md +++ b/docs/query.md @@ -83,20 +83,38 @@ Query DORA (DevOps Research and Assessment) metrics. gh devlake query dora --project [flags] ``` -**Status:** 🚧 Not yet implemented +**Status:** ⚠️ Partial implementation (limited by available API data) -**Reason:** DevLake does not currently expose a metrics API endpoint. DORA metrics are calculated in Grafana dashboards using SQL queries against the domain layer tables, but these calculations are not available via the REST API. +**What's available:** +- Project metadata (name, description, blueprint info) +- Clear explanation of limitations in the response -**Workaround:** View DORA metrics in your Grafana dashboards: +**What's not available:** +Full DORA metric calculations (deployment frequency, lead time for changes, change failure rate, mean time to restore) require SQL queries against DevLake's domain layer tables. DevLake does not expose database credentials or a metrics API endpoint. + +**Current output (JSON):** + +```json +{ + "project": "my-team", + "timeframe": "30d", + "availableData": { + "project": { "name": "my-team", "blueprint": {...} } + }, + "limitations": "Full DORA metrics require SQL against domain tables..." +} +``` + +**Workaround for full metrics:** View DORA metrics in your Grafana dashboards: ```bash gh devlake status # Shows Grafana URL ``` Then navigate to the DORA dashboards in Grafana. -**Future implementation requires:** +**Full implementation requires:** 1. Upstream DevLake metrics API endpoint -2. OR direct database query support (requires DB credentials in state files) +2. OR direct database query support (requires DB credentials) 3. OR Grafana API integration to fetch dashboard data --- @@ -109,20 +127,40 @@ Query GitHub Copilot usage metrics. gh devlake query copilot --project [flags] ``` -**Status:** 🚧 Not yet implemented +**Status:** ⚠️ Partial implementation (limited by available API data) -**Reason:** DevLake does not currently expose a metrics API endpoint. Copilot metrics are stored in the `gh-copilot` plugin's database tables and visualized in Grafana dashboards, but not accessible via the REST API. +**What's available:** +- Project metadata (name, description, blueprint info) +- GitHub Copilot connection information +- Clear explanation of limitations in the response + +**What's not available:** +Copilot usage metrics (total seats, active users, acceptance rates, language breakdowns, editor usage) are stored in `_tool_gh_copilot_*` database tables and visualized in Grafana dashboards, but DevLake does not expose a metrics API endpoint. + +**Current output (JSON):** + +```json +{ + "project": "my-team", + "timeframe": "30d", + "availableData": { + "project": { "name": "my-team", "blueprint": {...} }, + "connections": [...] + }, + "limitations": "Copilot metrics in _tool_gh_copilot_* tables require metrics API..." +} +``` -**Workaround:** View Copilot metrics in your Grafana dashboards: +**Workaround for full metrics:** View Copilot metrics in your Grafana dashboards: ```bash gh devlake status # Shows Grafana URL ``` Then navigate to the Copilot dashboards in Grafana. -**Future implementation requires:** +**Full implementation requires:** 1. Upstream DevLake metrics API endpoint for Copilot plugin -2. OR direct database query support (requires DB credentials in state files) +2. OR direct database query support (requires DB credentials) 3. OR Grafana API integration to fetch dashboard data --- @@ -136,12 +174,13 @@ These flags are inherited from the root command: ## Architecture Notes -The `query` command is designed to be extensible: +The `query` command uses the `internal/query/` package for extensible API-backed queries: -- **Current:** The `pipelines` subcommand uses the existing `/pipelines` REST API endpoint -- **Future:** The `dora` and `copilot` subcommands are placeholders awaiting upstream API support +- **Pipelines:** Fully functional - queries the `/pipelines` REST API endpoint with filtering and formatting +- **DORA:** Partial - returns project metadata from REST API; full metric calculations require SQL against domain tables +- **Copilot:** Partial - returns project and connection metadata from REST API; usage metrics are in database tables not exposed via API -When DevLake exposes metrics APIs, the existing command structure will remain the same — only the implementation will change from returning an error to fetching actual metrics. +All queries use the query engine abstraction (`internal/query/engine.go`) with registered query definitions. When DevLake exposes metrics APIs in the future, only the query execution functions need to change - the command structure and engine remain the same. ## See Also diff --git a/internal/query/engine.go b/internal/query/engine.go index 3f69ef4..ae239b2 100644 --- a/internal/query/engine.go +++ b/internal/query/engine.go @@ -27,16 +27,16 @@ func (e *Engine) Execute(queryDef *QueryDef, params map[string]interface{}) (int return nil, fmt.Errorf("query %q has no execute function", queryDef.Name) } - // Validate required parameters + // Apply defaults and validate parameters for _, param := range queryDef.Params { - if param.Required { - if _, ok := params[param.Name]; !ok { - // Check if there's a default value - if param.Default != "" { - params[param.Name] = param.Default - } else { - return nil, fmt.Errorf("required parameter %q not provided", param.Name) - } + if _, ok := params[param.Name]; !ok { + // Parameter not provided + if param.Default != "" { + // Apply default value + params[param.Name] = param.Default + } else if param.Required { + // Required parameter missing with no default + return nil, fmt.Errorf("required parameter %q not provided", param.Name) } } } diff --git a/internal/query/pipelines.go b/internal/query/pipelines.go index b6663a1..c5c876a 100644 --- a/internal/query/pipelines.go +++ b/internal/query/pipelines.go @@ -58,7 +58,12 @@ func executePipelinesQuery(client *devlake.Client, params map[string]interface{} if l, ok := params["limit"].(int); ok { limit = l } else if l, ok := params["limit"].(string); ok { - fmt.Sscanf(l, "%d", &limit) + var parsedLimit int + n, err := fmt.Sscanf(l, "%d", &parsedLimit) + if err != nil || n != 1 { + return nil, fmt.Errorf("invalid limit value %q: must be a valid integer", l) + } + limit = parsedLimit } // Query pipelines via API From 5a69bdcc59158ff207ffd60aa15b7ef10d23d11f Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:44:46 +0000 Subject: [PATCH 5/6] Fix URL injection vulnerability in ListPipelines - Replace manual string concatenation with url.Values for query params - All query parameters now properly URL-encoded via q.Encode() - Remove unused strings import - Prevents injection of additional query parameters via --status flag - All validation passes: go build, go test, go vet Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- internal/devlake/client.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/devlake/client.go b/internal/devlake/client.go index 61020ed..9831e3a 100644 --- a/internal/devlake/client.go +++ b/internal/devlake/client.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "net/url" - "strings" "time" ) @@ -525,22 +524,22 @@ type PipelineListResponse struct { // blueprintID filters by blueprint (0 = no filter). // page and pageSize control pagination (0 = use defaults). func (c *Client) ListPipelines(status string, blueprintID, page, pageSize int) (*PipelineListResponse, error) { - path := "/pipelines?" - params := []string{} + path := "/pipelines" + q := url.Values{} if status != "" { - params = append(params, "status="+status) + q.Set("status", status) } if blueprintID > 0 { - params = append(params, fmt.Sprintf("blueprint_id=%d", blueprintID)) + q.Set("blueprint_id", fmt.Sprintf("%d", blueprintID)) } if page > 0 { - params = append(params, fmt.Sprintf("page=%d", page)) + q.Set("page", fmt.Sprintf("%d", page)) } if pageSize > 0 { - params = append(params, fmt.Sprintf("pagesize=%d", pageSize)) + q.Set("pagesize", fmt.Sprintf("%d", pageSize)) } - if len(params) > 0 { - path += strings.Join(params, "&") + if len(q) > 0 { + path += "?" + q.Encode() } return doGet[PipelineListResponse](c, path) } From 5c6898cdd469adc60151982bcbb08a74f6f7396e Mon Sep 17 00:00:00 2001 From: "anthropic-code-agent[bot]" <242468646+Claude@users.noreply.github.com> Date: Thu, 12 Mar 2026 20:00:29 +0000 Subject: [PATCH 6/6] Address GitHub review feedback on query commands 1. Fix JSON output corruption: use quiet discovery for JSON, verbose for table 2. Add --format validation: return error for invalid format values 3. Fix nil map safety: initialize params map if nil in engine.Execute 4. Fix swallowed error: propagate ListConnections errors in copilot query 5. Add tests: query_pipelines_test.go validates JSON output and format validation All validation passes: go build, go test, go vet Co-authored-by: ewega <26189114+ewega@users.noreply.github.com> --- cmd/query_pipelines.go | 20 +++-- cmd/query_pipelines_test.go | 174 ++++++++++++++++++++++++++++++++++++ internal/query/copilot.go | 5 +- internal/query/engine.go | 5 ++ 4 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 cmd/query_pipelines_test.go diff --git a/cmd/query_pipelines.go b/cmd/query_pipelines.go index 558f26c..a809fcf 100644 --- a/cmd/query_pipelines.go +++ b/cmd/query_pipelines.go @@ -41,23 +41,31 @@ func init() { } func runQueryPipelines(cmd *cobra.Command, args []string) error { + // Validate format flag + if queryPipelinesFormat != "json" && queryPipelinesFormat != "table" { + return fmt.Errorf("invalid --format value %q: must be 'json' or 'table'", queryPipelinesFormat) + } + // Discover DevLake instance - var disc *devlake.DiscoveryResult var client *devlake.Client var err error - if !outputJSON && queryPipelinesFormat != "table" { - client, disc, err = discoverClient(cfgURL) + // Use quiet discovery for JSON output, verbose for table + if outputJSON || queryPipelinesFormat == "json" { + // Quiet discovery for JSON output + disc, err := devlake.Discover(cfgURL) if err != nil { return fmt.Errorf("discovering DevLake: %w", err) } + client = devlake.NewClient(disc.URL) } else { - // Quiet discovery for JSON/table output - disc, err = devlake.Discover(cfgURL) + // Verbose discovery for table output + var disc *devlake.DiscoveryResult + client, disc, err = discoverClient(cfgURL) if err != nil { return fmt.Errorf("discovering DevLake: %w", err) } - client = devlake.NewClient(disc.URL) + _ = disc // disc is used by discoverClient for output } // Get the query definition diff --git a/cmd/query_pipelines_test.go b/cmd/query_pipelines_test.go new file mode 100644 index 0000000..462fdbd --- /dev/null +++ b/cmd/query_pipelines_test.go @@ -0,0 +1,174 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/DevExpGBB/gh-devlake/internal/devlake" + "github.com/DevExpGBB/gh-devlake/internal/query" +) + +func TestQueryPipelines_InvalidFormat(t *testing.T) { + queryPipelinesFormat = "invalid" + t.Cleanup(func() { queryPipelinesFormat = "json" }) + + err := runQueryPipelines(nil, nil) + if err == nil { + t.Fatal("expected error for invalid --format, got nil") + } + if !strings.Contains(err.Error(), "invalid --format value") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestQueryPipelines_JSONOutputNoBanner(t *testing.T) { + // Mock DevLake API + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ping" { + w.WriteHeader(http.StatusOK) + return + } + if r.URL.Path == "/pipelines" { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(devlake.PipelineListResponse{ + Pipelines: []devlake.Pipeline{ + { + ID: 123, + Status: "TASK_COMPLETED", + FinishedTasks: 10, + TotalTasks: 10, + }, + }, + Count: 1, + }) + return + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + + // Set URL to mock server + origURL := cfgURL + cfgURL = srv.URL + t.Cleanup(func() { cfgURL = origURL }) + + // Set format to JSON + origFormat := queryPipelinesFormat + queryPipelinesFormat = "json" + t.Cleanup(func() { queryPipelinesFormat = origFormat }) + + // Capture stdout + orig := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + os.Stdout = w + t.Cleanup(func() { os.Stdout = orig }) + + // Run the command + if err := runQueryPipelines(nil, nil); err != nil { + t.Fatalf("runQueryPipelines returned error: %v", err) + } + + w.Close() + var buf bytes.Buffer + buf.ReadFrom(r) + out := buf.String() + + // Verify no discovery banners in output + if strings.Contains(out, "Discovering DevLake") { + t.Error("JSON output should not contain discovery banner") + } + if strings.Contains(out, "🔍") { + t.Error("JSON output should not contain emoji banners") + } + + // Verify valid JSON + trimmed := strings.TrimSpace(out) + var pipelines []query.PipelineResult + if err := json.Unmarshal([]byte(trimmed), &pipelines); err != nil { + t.Fatalf("output is not valid JSON: %v — got: %q", err, out) + } + + if len(pipelines) != 1 { + t.Fatalf("expected 1 pipeline, got %d", len(pipelines)) + } + if pipelines[0].ID != 123 { + t.Errorf("expected ID=123, got %d", pipelines[0].ID) + } + if pipelines[0].Status != "TASK_COMPLETED" { + t.Errorf("expected status=TASK_COMPLETED, got %q", pipelines[0].Status) + } +} + +func TestQueryPipelines_GlobalJSONFlag(t *testing.T) { + // Mock DevLake API + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ping" { + w.WriteHeader(http.StatusOK) + return + } + if r.URL.Path == "/pipelines" { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(devlake.PipelineListResponse{ + Pipelines: []devlake.Pipeline{}, + Count: 0, + }) + return + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + + // Set URL to mock server + origURL := cfgURL + cfgURL = srv.URL + t.Cleanup(func() { cfgURL = origURL }) + + // Set global JSON flag + origJSON := outputJSON + outputJSON = true + t.Cleanup(func() { outputJSON = origJSON }) + + // Set format to table (should be overridden by --json) + origFormat := queryPipelinesFormat + queryPipelinesFormat = "table" + t.Cleanup(func() { queryPipelinesFormat = origFormat }) + + // Capture stdout + orig := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + os.Stdout = w + t.Cleanup(func() { os.Stdout = orig }) + + // Run the command + if err := runQueryPipelines(nil, nil); err != nil { + t.Fatalf("runQueryPipelines returned error: %v", err) + } + + w.Close() + var buf bytes.Buffer + buf.ReadFrom(r) + out := buf.String() + + // Verify no discovery banners in output + if strings.Contains(out, "Discovering DevLake") { + t.Error("JSON output with --json should not contain discovery banner") + } + + // Verify valid JSON + trimmed := strings.TrimSpace(out) + var pipelines []query.PipelineResult + if err := json.Unmarshal([]byte(trimmed), &pipelines); err != nil { + t.Fatalf("output is not valid JSON: %v — got: %q", err, out) + } +} diff --git a/internal/query/copilot.go b/internal/query/copilot.go index 6eb1c5b..a1fe93b 100644 --- a/internal/query/copilot.go +++ b/internal/query/copilot.go @@ -50,12 +50,11 @@ func executeCopilotQuery(client *devlake.Client, params map[string]interface{}) // Check if gh-copilot plugin is configured connections, err := client.ListConnections("gh-copilot") if err != nil { - // Plugin might not be available - connections = []devlake.Connection{} + return nil, fmt.Errorf("listing gh-copilot connections: %w", err) } availableData := map[string]interface{}{ - "projectName": proj.Name, + "projectName": proj.Name, "copilotConnectionsFound": len(connections), } diff --git a/internal/query/engine.go b/internal/query/engine.go index ae239b2..c3dfc97 100644 --- a/internal/query/engine.go +++ b/internal/query/engine.go @@ -27,6 +27,11 @@ func (e *Engine) Execute(queryDef *QueryDef, params map[string]interface{}) (int return nil, fmt.Errorf("query %q has no execute function", queryDef.Name) } + // Initialize params if nil to avoid panics when applying defaults + if params == nil { + params = make(map[string]interface{}) + } + // Apply defaults and validate parameters for _, param := range queryDef.Params { if _, ok := params[param.Name]; !ok {