diff --git a/grafana/dashboards/ai-usage.json b/grafana/dashboards/ai-usage.json new file mode 100644 index 000000000..9059255d0 --- /dev/null +++ b/grafana/dashboards/ai-usage.json @@ -0,0 +1,474 @@ +{ + "uid": "gittensory-ai-usage", + "title": "Gittensory - AI usage", + "tags": ["gittensory", "ai"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-7d", "to": "now" }, + "description": "Consolidated AI-usage dashboard (replaces the old Claude/Codex/ORB AI-usage dashboards, and the AI-cost/token panels formerly in the infra dashboard). Two structurally different sources, kept in clearly separate sections: (1) the durable, cross-provider ai_usage_events log (migration 0109) — every configured provider (Ollama, Claude Code, Codex, OpenAI-compatible, Anthropic BYOK) and every AI feature (ai_review_pr, ai_slop_pr, embeddings, visual_vision, screenshot_table_vision, agent_private_summary, agent_public_summary, pr_intelligence_comment, linked_issue_satisfaction, issue_plan, unlinked_issue_verify, chat_qa, intent_routing, ai_key_change), filterable by $provider/$feature/$model and repo/PR-attributable; (2) Claude Code's own OTEL session telemetry, a separate non-durable data source with no repo/PR attribution that cannot be joined to (1) — kept in its own section rather than force-merged. A row of live Prometheus counters sits between them for real-time signal the durable log can lag behind (it's written after a call completes).", + "templating": { + "list": [ + { + "name": "provider", + "label": "Provider", + "type": "query", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "query": { "queryType": "table", "rawQueryText": "SELECT DISTINCT provider FROM ai_usage_events WHERE provider IS NOT NULL ORDER BY provider" }, + "includeAll": true, + "allValue": "$__all", + "multi": false, + "current": { "text": "All", "value": "$__all" }, + "refresh": 2, + "sort": 1 + }, + { + "name": "feature", + "label": "Feature", + "type": "query", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "query": { "queryType": "table", "rawQueryText": "SELECT DISTINCT feature FROM ai_usage_events ORDER BY feature" }, + "includeAll": true, + "allValue": "$__all", + "multi": false, + "current": { "text": "All", "value": "$__all" }, + "refresh": 2, + "sort": 1 + }, + { + "name": "model", + "label": "Model", + "type": "query", + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "query": { + "queryType": "table", + "rawQueryText": "SELECT DISTINCT model FROM ai_usage_events WHERE ('$provider' = '$__all' OR provider = '$provider') ORDER BY model" + }, + "includeAll": true, + "allValue": "$__all", + "multi": false, + "current": { "text": "All", "value": "$__all" }, + "refresh": 2, + "sort": 1, + "description": "Narrows to the selected Provider's own models (e.g. Ollama's separate bge-m3/qwen3-vl/qwen3 use cases)." + }, + { + "name": "claudeModel", + "label": "Claude Model (OTEL)", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(claude_code_cost_usage_USD_total, model)", + "query": { "query": "label_values(claude_code_cost_usage_USD_total, model)", "refId": "A" }, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { "text": "All", "value": "$__all" }, + "refresh": 2, + "description": "Only scopes the separate 'Claude Code native OTEL telemetry' section below — unrelated to Provider/Feature/Model above, which scope the durable ai_usage_events log." + } + ] + }, + "annotations": { "list": [] }, + "links": [], + "panels": [ + { + "id": 1, + "type": "row", + "title": "Summary (durable log, filtered)", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } + }, + { + "id": 2, + "type": "stat", + "title": "Total AI events", + "description": "Every ai_usage_events row with a non-null provider (i.e. a real, attributed AI call) in the selected time range, filtered by Provider/Feature/Model.", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}", + "rawQueryText": "SELECT count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Total tokens", + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT coalesce(sum(total_tokens), 0) AS total_tokens FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}", + "rawQueryText": "SELECT coalesce(sum(total_tokens), 0) AS total_tokens FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Total cost", + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "fixed", "fixedColor": "green" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}", + "rawQueryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Events missing real usage", + "description": "Completed AI calls (status='ok') with no real usage captured (no provider attribution, or every numeric field still at its zero default). Only scoped by Feature + time range — not by Provider/Model, since a NULL provider is exactly the kind of row this panel exists to surface, and filtering it out by provider would hide the very rows it's meant to catch. Excludes quota_exceeded/disabled/unavailable (no call was attempted) and ai_key_change (not an AI call).", + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE status = 'ok' AND feature <> 'ai_key_change' AND ('$feature' = '$__all' OR feature = '$feature') AND (provider IS NULL OR (input_tokens = 0 AND output_tokens = 0 AND total_tokens = 0 AND cost_usd = 0)) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}", + "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE status = 'ok' AND feature <> 'ai_key_change' AND ('$feature' = '$__all' OR feature = '$feature') AND (provider IS NULL OR (input_tokens = 0 AND output_tokens = 0 AND total_tokens = 0 AND cost_usd = 0)) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}" + } + ] + }, + { + "id": 6, + "type": "row", + "title": "Usage by category (what's using the most, and is anything spiking)", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } + }, + { + "id": 7, + "type": "timeseries", + "title": "Tokens by feature, over time", + "description": "Stacked daily token totals per feature within the selected Provider/Model filter — a spike in one feature's stack is the signal to investigate.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "queryType": "time series", + "timeColumns": ["time"], + "metricColumn": "feature", + "queryText": "SELECT date(created_at) AS time, feature, coalesce(sum(total_tokens), 0) AS tokens FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY date(created_at), feature ORDER BY time", + "rawQueryText": "SELECT date(created_at) AS time, feature, coalesce(sum(total_tokens), 0) AS tokens FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY date(created_at), feature ORDER BY time" + } + ] + }, + { + "id": 8, + "type": "piechart", + "title": "Cost share by feature", + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 7 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD" } }, + "options": { "pieType": "donut", "legend": { "showLegend": true, "placement": "right", "values": ["percent", "value"] }, "reduceOptions": { "calcs": ["lastNotNull"] }, "displayLabels": ["percent"] }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT feature, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY feature ORDER BY cost_usd DESC", + "rawQueryText": "SELECT feature, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY feature ORDER BY cost_usd DESC" + } + ] + }, + { + "id": 9, + "type": "piechart", + "title": "Events by provider", + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 7 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "pieType": "donut", "legend": { "showLegend": true, "placement": "right", "values": ["percent", "value"] }, "reduceOptions": { "calcs": ["lastNotNull"] }, "displayLabels": ["percent"] }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT provider, count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider ORDER BY events DESC", + "rawQueryText": "SELECT provider, count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider ORDER BY events DESC" + } + ] + }, + { + "id": 10, + "type": "table", + "title": "Usage by provider + model + effort", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT provider, model, effort, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50", + "rawQueryText": "SELECT provider, model, effort, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50" + } + ] + }, + { + "id": 11, + "type": "row", + "title": "By repo / PR", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 } + }, + { + "id": 12, + "type": "table", + "title": "Top repos by AI usage", + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 24 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY repo, provider ORDER BY events DESC LIMIT 50", + "rawQueryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY repo, provider ORDER BY events DESC LIMIT 50" + } + ] + }, + { + "id": 13, + "type": "table", + "title": "Recent AI events", + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 24 }, + "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, + "targets": [ + { + "refId": "A", + "queryType": "table", + "queryText": "SELECT created_at, feature, provider, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} ORDER BY created_at DESC LIMIT 100", + "rawQueryText": "SELECT created_at, feature, provider, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE provider IS NOT NULL AND ('$provider' = '$__all' OR provider = '$provider') AND ('$feature' = '$__all' OR feature = '$feature') AND ('$model' = '$__all' OR model = '$model') AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} ORDER BY created_at DESC LIMIT 100" + } + ] + }, + { + "id": 14, + "type": "row", + "title": "Live Prometheus counters (all providers, real-time — the durable log above is written after a call completes)", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 } + }, + { + "id": 15, + "type": "timeseries", + "title": "Cumulative AI cost (USD) by provider", + "description": "gittensory's own monotonic cost counter (gittensory_ai_cost_usd_total) — continuously scraped, so it draws a clean cumulative line. Resets to 0 on a stack restart.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [{ "refId": "A", "expr": "sum by (provider) (gittensory_ai_cost_usd_total) or vector(0)", "legendFormat": "{{provider}}" }] + }, + { + "id": 16, + "type": "timeseries", + "title": "Tokens/min by provider", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { + "refId": "A", + "expr": "sum by (provider) ((rate(gittensory_ai_input_tokens_total[5m]) + rate(gittensory_ai_output_tokens_total[5m])) * 60)", + "legendFormat": "{{provider}}" + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "AI requests + fallbacks (last 1h)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "refId": "A", "expr": "sum by (model, effort) (increase(gittensory_ai_requests_total[1h]))", "legendFormat": "{{model}} · {{effort}}" }, + { "refId": "B", "expr": "sum by (primary, fallback) (increase(gittensory_ai_review_model_fallback_total[1h]))", "legendFormat": "fallback {{primary}}→{{fallback}}" } + ] + }, + { + "id": 18, + "type": "timeseries", + "title": "Total tokens by provider + kind", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "refId": "A", "expr": "sum by (provider, kind) (gittensory_ai_input_tokens_total)", "legendFormat": "{{provider}} {{kind}} in" }, + { "refId": "B", "expr": "sum by (provider, kind) (gittensory_ai_output_tokens_total)", "legendFormat": "{{provider}} {{kind}} out" } + ] + }, + { + "id": 19, + "type": "timeseries", + "title": "Codex CLI attempts by model and effort", + "description": "Codex-specific: raw CLI attempt counters, including attempts that never produced a usable response (so never landed a durable ai_usage_events row).", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "none" } } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "expr": "sum by (model, effort) (increase(gittensory_ai_requests_total{provider=\"codex\"}[$__rate_interval]))", + "legendFormat": "{{model}} / {{effort}}" + } + ] + }, + { + "id": 20, + "type": "timeseries", + "title": "Codex reported tokens by direction", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { "refId": "A", "expr": "sum by (kind) (increase(gittensory_ai_input_tokens_total{provider=\"codex\"}[$__rate_interval]))", "legendFormat": "input {{kind}}" }, + { "refId": "B", "expr": "sum by (kind) (increase(gittensory_ai_output_tokens_total{provider=\"codex\"}[$__rate_interval]))", "legendFormat": "output {{kind}}" } + ] + }, + { + "id": 21, + "type": "row", + "title": "Claude Code native OTEL telemetry (session-scoped CLI metrics — NOT joinable with the durable log above)", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 } + }, + { + "id": 22, + "type": "stat", + "title": "Total cost", + "description": "Each headless review is a session-scoped cumulative counter, so totals use last_over_time (sum of each session's final value) — increase()/rate() would read flat because a session exports its total in one shot.", + "gridPos": { "h": 5, "w": 5, "x": 0, "y": 60 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 2, "color": { "mode": "fixed", "fixedColor": "green" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background_solid", "graphMode": "none", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(last_over_time(claude_code_cost_usage_USD_total{model=~\"$claudeModel\"}[$__range]))" }] + }, + { + "id": 23, + "type": "stat", + "title": "Total tokens", + "gridPos": { "h": 5, "w": 5, "x": 5, "y": 60 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(last_over_time(claude_code_token_usage_tokens_total{model=~\"$claudeModel\"}[$__range]))" }] + }, + { + "id": 24, + "type": "stat", + "title": "Sessions", + "gridPos": { "h": 5, "w": 5, "x": 10, "y": 60 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(last_over_time(claude_code_session_count_total[$__range]))" }] + }, + { + "id": 25, + "type": "stat", + "title": "Active time", + "gridPos": { "h": 5, "w": 4, "x": 15, "y": 60 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "fixed", "fixedColor": "orange" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(last_over_time(claude_code_active_time_seconds_total[$__range]))" }] + }, + { + "id": 26, + "type": "stat", + "title": "Lines of code", + "gridPos": { "h": 5, "w": 5, "x": 19, "y": 60 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "yellow" } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum(last_over_time(claude_code_lines_of_code_count_total[$__range]))" }] + }, + { + "id": 27, + "type": "timeseries", + "title": "Cost by model (per scrape interval)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 65 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "gradientMode": "opacity" } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom", "displayMode": "table", "calcs": ["max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [{ "refId": "A", "expr": "sum by (model) (last_over_time(claude_code_cost_usage_USD_total{model=~\"$claudeModel\"}[$__rate_interval]))", "legendFormat": "{{model}}" }] + }, + { + "id": 28, + "type": "piechart", + "title": "Tokens by type", + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 65 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "options": { "pieType": "donut", "legend": { "showLegend": true, "placement": "right", "values": ["percent", "value"] }, "reduceOptions": { "calcs": ["lastNotNull"] }, "displayLabels": ["percent"] }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum by (type) (last_over_time(claude_code_token_usage_tokens_total{model=~\"$claudeModel\"}[$__range]))", "legendFormat": "{{type}}" }] + }, + { + "id": 29, + "type": "bargauge", + "title": "Cost by model", + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 65 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "continuous-GrYlRd" } } }, + "options": { "orientation": "horizontal", "displayMode": "gradient", "valueMode": "color", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "refId": "A", "instant": true, "expr": "sum by (model) (last_over_time(claude_code_cost_usage_USD_total{model=~\"$claudeModel\"}[$__range]))", "legendFormat": "{{model}}" }] + }, + { + "id": 30, + "type": "timeseries", + "title": "Tokens by type (per scrape interval)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 73 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 0, "stacking": { "mode": "normal" } } } }, + "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [{ "refId": "A", "expr": "sum by (type) (last_over_time(claude_code_token_usage_tokens_total{model=~\"$claudeModel\"}[$__rate_interval]))", "legendFormat": "{{type}}" }] + }, + { + "id": 31, + "type": "table", + "title": "Cost by model × effort", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 73 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "Cost", "desc": true }] }, + "fieldConfig": { + "defaults": { "custom": { "filterable": true } }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Cost" }, + "properties": [{ "id": "unit", "value": "currencyUSD" }, { "id": "decimals", "value": 4 }, { "id": "custom.cellOptions", "value": { "type": "gauge", "mode": "gradient" } }] + } + ] + }, + "targets": [{ "refId": "A", "format": "table", "instant": true, "expr": "sum by (model, effort) (last_over_time(claude_code_cost_usage_USD_total{model=~\"$claudeModel\"}[$__range]))" }], + "transformations": [{ "id": "organize", "options": { "renameByName": { "Value": "Cost" }, "excludeByName": { "Time": true } } }] + } + ] +} diff --git a/grafana/dashboards/claude-usage.json b/grafana/dashboards/claude-usage.json deleted file mode 100644 index b2933c4bc..000000000 --- a/grafana/dashboards/claude-usage.json +++ /dev/null @@ -1,391 +0,0 @@ -{ - "uid": "gittensory-claude", - "title": "Gittensory — Claude usage (OTEL)", - "tags": ["gittensory", "claude", "ai"], - "timezone": "browser", - "schemaVersion": 39, - "version": 4, - "refresh": "30s", - "time": { "from": "now-7d", "to": "now" }, - "description": "Claude Code CLI/session telemetry via OpenTelemetry — NOT the durable ORB ai_usage_events DB attribution (see the 'Gittensory - ORB AI usage' dashboard for that, filterable by provider/model/repo/PR). Each headless review is a session-scoped cumulative counter, so totals use last_over_time (sum of each session's final value) — increase()/rate() would read flat because a session exports its total in one shot.", - "templating": { - "list": [ - { - "name": "model", - "label": "Model", - "type": "query", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(claude_code_cost_usage_USD_total, model)", - "query": { - "query": "label_values(claude_code_cost_usage_USD_total, model)", - "refId": "A" - }, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { "text": "All", "value": "$__all" }, - "refresh": 2 - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Summary (over the selected time range)", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } - }, - - { - "id": 2, - "type": "stat", - "title": "Total cost", - "gridPos": { "h": 5, "w": 5, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 2, - "color": { "mode": "fixed", "fixedColor": "green" } - } - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "background_solid", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "instant": true, - "expr": "sum(last_over_time(claude_code_cost_usage_USD_total{model=~\"$model\"}[$__range]))" - } - ] - }, - - { - "id": 3, - "type": "stat", - "title": "Total tokens", - "gridPos": { "h": 5, "w": 5, "x": 5, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "blue" } - } - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "instant": true, - "expr": "sum(last_over_time(claude_code_token_usage_tokens_total{model=~\"$model\"}[$__range]))" - } - ] - }, - - { - "id": 4, - "type": "stat", - "title": "Sessions", - "gridPos": { "h": 5, "w": 5, "x": 10, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "purple" } - } - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "instant": true, - "expr": "sum(last_over_time(claude_code_session_count_total[$__range]))" - } - ] - }, - - { - "id": 5, - "type": "stat", - "title": "Active time", - "gridPos": { "h": 5, "w": 4, "x": 15, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "fixed", "fixedColor": "orange" } - } - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "instant": true, - "expr": "sum(last_over_time(claude_code_active_time_seconds_total[$__range]))" - } - ] - }, - - { - "id": 6, - "type": "stat", - "title": "Lines of code", - "gridPos": { "h": 5, "w": 5, "x": 19, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "yellow" } - } - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "instant": true, - "expr": "sum(last_over_time(claude_code_lines_of_code_count_total[$__range]))" - } - ] - }, - - { - "id": 7, - "type": "row", - "title": "Breakdowns", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } - }, - - { - "id": 8, - "type": "timeseries", - "title": "Cost by model (per scrape interval)", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "custom": { - "drawStyle": "bars", - "fillOpacity": 70, - "lineWidth": 1, - "gradientMode": "opacity" - } - } - }, - "options": { - "legend": { - "showLegend": true, - "placement": "bottom", - "displayMode": "table", - "calcs": ["max"] - }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "refId": "A", - "expr": "sum by (model) (last_over_time(claude_code_cost_usage_USD_total{model=~\"$model\"}[$__rate_interval]))", - "legendFormat": "{{model}}" - } - ] - }, - - { - "id": 9, - "type": "piechart", - "title": "Tokens by type", - "gridPos": { "h": 8, "w": 6, "x": 12, "y": 7 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short" } }, - "options": { - "pieType": "donut", - "legend": { - "showLegend": true, - "placement": "right", - "values": ["percent", "value"] - }, - "reduceOptions": { "calcs": ["lastNotNull"] }, - "displayLabels": ["percent"] - }, - "targets": [ - { - "refId": "A", - "instant": true, - "expr": "sum by (type) (last_over_time(claude_code_token_usage_tokens_total{model=~\"$model\"}[$__range]))", - "legendFormat": "{{type}}" - } - ] - }, - - { - "id": 10, - "type": "bargauge", - "title": "Cost by model", - "gridPos": { "h": 8, "w": 6, "x": 18, "y": 7 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 4, - "color": { "mode": "continuous-GrYlRd" } - } - }, - "options": { - "orientation": "horizontal", - "displayMode": "gradient", - "valueMode": "color", - "reduceOptions": { "calcs": ["lastNotNull"] } - }, - "targets": [ - { - "refId": "A", - "instant": true, - "expr": "sum by (model) (last_over_time(claude_code_cost_usage_USD_total{model=~\"$model\"}[$__range]))", - "legendFormat": "{{model}}" - } - ] - }, - - { - "id": 11, - "type": "timeseries", - "title": "Tokens by type (per scrape interval)", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "lineWidth": 0, - "stacking": { "mode": "normal" } - } - } - }, - "options": { - "legend": { "showLegend": true, "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } - }, - "targets": [ - { - "refId": "A", - "expr": "sum by (type) (last_over_time(claude_code_token_usage_tokens_total{model=~\"$model\"}[$__rate_interval]))", - "legendFormat": "{{type}}" - } - ] - }, - - { - "id": 12, - "type": "table", - "title": "Cost by model × effort", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "sortBy": [{ "displayName": "Cost", "desc": true }] - }, - "fieldConfig": { - "defaults": { "custom": { "filterable": true } }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "Cost" }, - "properties": [ - { "id": "unit", "value": "currencyUSD" }, - { "id": "decimals", "value": 4 }, - { - "id": "custom.cellOptions", - "value": { "type": "gauge", "mode": "gradient" } - } - ] - } - ] - }, - "targets": [ - { - "refId": "A", - "format": "table", - "instant": true, - "expr": "sum by (model, effort) (last_over_time(claude_code_cost_usage_USD_total{model=~\"$model\"}[$__range]))" - } - ], - "transformations": [ - { - "id": "organize", - "options": { - "renameByName": { "Value": "Cost" }, - "excludeByName": { "Time": true } - } - } - ] - }, - - { - "id": 13, - "type": "timeseries", - "title": "Cumulative review cost", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 23 }, - "description": "gittensory's own monotonic cost counter (gittensory_ai_cost_usd_total) — continuously scraped, so it draws a clean cumulative line, complementing the per-session OTEL metrics above. Resets to 0 on a stack restart.", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "decimals": 2, - "color": { "mode": "fixed", "fixedColor": "green" }, - "custom": { - "drawStyle": "line", - "fillOpacity": 25, - "lineWidth": 2, - "showPoints": "never", - "gradientMode": "opacity", - "lineInterpolation": "smooth", - "spanNulls": true - } - }, - "overrides": [] - }, - "options": { - "legend": { - "showLegend": true, - "placement": "bottom", - "displayMode": "table", - "calcs": ["lastNotNull"] - }, - "tooltip": { "mode": "multi" } - }, - "targets": [ - { - "refId": "A", - "expr": "sum by (provider) (gittensory_ai_cost_usd_total)", - "legendFormat": "{{provider}}" - } - ] - } - ] -} diff --git a/grafana/dashboards/codex-usage.json b/grafana/dashboards/codex-usage.json deleted file mode 100644 index 72e67ac10..000000000 --- a/grafana/dashboards/codex-usage.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "uid": "gittensory-codex", - "title": "Gittensory - Codex usage (self-host)", - "tags": ["gittensory", "codex", "ai"], - "timezone": "browser", - "schemaVersion": 39, - "version": 4, - "refresh": "30s", - "time": { "from": "now-7d", "to": "now" }, - "description": "Codex review observability for the self-hosted stack. Prometheus panels are subscription-CLI attempt counters for the selected range. Durable review records come from ai_usage_events (migration 0109) and must be read with their status. Two different filters are used deliberately: the token/cost panels (real usage sums) filter strictly on provider='codex', since an unattributed row always contributes 0 to a sum either way. The record-count/status/recent-events panels instead match 'codex' as a reviewer-name token in either provider OR the model label (\"+\"-joined reviewer names, e.g. 'codex', 'codex:o4-mini', 'codex+claude-code') — a quota-blocked call never gets provider attribution (no call ever ran) but IS attributable via its configured model label, so filtering by provider alone silently drops every quota_exceeded row for a codex-configured repo. estimated_neurons is a heuristic budget proxy, not billing-grade accounting.", - "panels": [ - { - "id": 1, - "type": "row", - "title": "Summary", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } - }, - { - "id": 2, - "type": "stat", - "title": "Reported cost", - "description": "USD cost reported by Codex JSON/JSONL output in the selected range, if available. Subscription CLIs may not emit this.", - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "fixed", "fixedColor": "green" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, - "targets": [{ "refId": "A", "instant": true, "expr": "sum(increase(gittensory_ai_cost_usd_total{provider=\"codex\"}[$__range])) or vector(0)" }] - }, - { - "id": 3, - "type": "stat", - "title": "CLI attempts", - "description": "Codex CLI attempts in the selected range. Failed or timed-out attempts can still appear here.", - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, - "targets": [{ "refId": "A", "instant": true, "expr": "sum(increase(gittensory_ai_requests_total{provider=\"codex\"}[$__range])) or vector(0)" }] - }, - { - "id": 4, - "type": "stat", - "title": "Reported CLI tokens", - "description": "Tokens reported by Codex CLI output in the selected range. This is not billing-grade cost accounting.", - "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area", "justifyMode": "center" }, - "targets": [{ "refId": "A", "instant": true, "expr": "sum(increase(gittensory_ai_total_tokens_total{provider=\"codex\"}[$__range])) or (sum(increase(gittensory_ai_input_tokens_total{provider=\"codex\"}[$__range])) + sum(increase(gittensory_ai_output_tokens_total{provider=\"codex\"}[$__range]))) or vector(0)" }] - }, - { - "id": 5, - "type": "stat", - "title": "Successful review records", - "description": "Durable ai_review_pr records with status=ok where codex was the reviewer — matched by provider attribution OR the configured model label (see dashboard description), so this stays consistent with the status-breakdown panel below rather than under-counting successes where the CLI's response reported no usage envelope at all.", - "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "green" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%'))", - "rawQueryText": "SELECT count(*) AS reviews FROM ai_usage_events WHERE feature = 'ai_review_pr' AND status = 'ok' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%'))" - } - ] - }, - { - "id": 6, - "type": "row", - "title": "Live counters", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } - }, - { - "id": 7, - "type": "timeseries", - "title": "CLI attempts by model and effort", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "none" } } } }, - "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, - "targets": [ - { - "refId": "A", - "expr": "sum by (model, effort) (increase(gittensory_ai_requests_total{provider=\"codex\"}[$__rate_interval]))", - "legendFormat": "{{model}} / {{effort}}" - } - ] - }, - { - "id": 8, - "type": "timeseries", - "title": "Reported tokens by direction", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, - "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, - "targets": [ - { - "refId": "A", - "expr": "sum by (kind) (increase(gittensory_ai_input_tokens_total{provider=\"codex\"}[$__rate_interval]))", - "legendFormat": "input {{kind}}" - }, - { - "refId": "B", - "expr": "sum by (kind) (increase(gittensory_ai_output_tokens_total{provider=\"codex\"}[$__rate_interval]))", - "legendFormat": "output {{kind}}" - } - ] - }, - { - "id": 9, - "type": "row", - "title": "Durable review records (status-aware)", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 } - }, - { - "id": 10, - "type": "timeseries", - "title": "Durable token usage by day", - "description": "Real input/output tokens from ai_usage_events (migration 0109), populated when the Codex CLI reports usage. Replaces the old estimated-neuron-only proxy for this panel. Filters strictly on provider='codex' (not the broader model-label match used by the status/recent-events panels below): a quota-blocked or otherwise unattributed row contributes 0 tokens regardless of which filter is used, so the stricter filter is equivalent here and simpler.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, - "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, - "targets": [ - { - "refId": "A", - "queryType": "time series", - "timeColumns": ["time"], - "queryText": "SELECT date(created_at) AS time, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", - "rawQueryText": "SELECT date(created_at) AS time, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" - } - ] - }, - { - "id": 11, - "type": "piechart", - "title": "Review record status", - "description": "Includes quota_exceeded rows for a codex-configured repo: those rows never get provider attribution (the call never ran), but the model label already reflects the configured reviewer at write time (record() computes it via reviewerModelLabel() regardless of status), so matching 'codex' as a reviewer-name token there recovers them. A plain provider='codex' filter would silently make every quota-blocked codex review invisible here.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short" } }, - "options": { "pieType": "donut", "legend": { "showLegend": true, "placement": "right", "values": ["percent", "value"] }, "reduceOptions": { "calcs": ["lastNotNull"] }, "displayLabels": ["percent"] }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) GROUP BY status ORDER BY count DESC", - "rawQueryText": "SELECT status, count(*) AS count FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) GROUP BY status ORDER BY count DESC" - } - ] - }, - { - "id": 13, - "type": "timeseries", - "title": "Durable cost by day", - "description": "Real USD cost from ai_usage_events (migration 0109), populated when the Codex CLI reports a cost field. Filters strictly on provider='codex' for the same reason as the token-usage panel above — an unattributed row contributes $0 regardless of filter.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "custom": { "drawStyle": "bars", "fillOpacity": 70, "lineWidth": 1 } } }, - "options": { "legend": { "showLegend": true, "placement": "bottom" }, "tooltip": { "mode": "multi" } }, - "targets": [ - { - "refId": "A", - "queryType": "time series", - "timeColumns": ["time"], - "queryText": "SELECT date(created_at) AS time, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time", - "rawQueryText": "SELECT date(created_at) AS time, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE feature = 'ai_review_pr' AND provider = 'codex' GROUP BY date(created_at) ORDER BY time" - } - ] - }, - { - "id": 12, - "type": "table", - "title": "Recent Codex-attributed review events", - "description": "Includes quota-blocked codex-configured rows (matched via the model label, not just provider attribution — see the status-breakdown panel's description) so an operator can see recent failures/blocks, not only rows with a completed usage response. The provider column itself is not displayed here; the model column already reflects the configured reviewer for every row, including blocked ones.", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 32 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, - "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) ORDER BY created_at DESC LIMIT 100", - "rawQueryText": "SELECT created_at, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, estimated_neurons, detail, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE feature = 'ai_review_pr' AND (provider = 'codex' OR (('+' || model || '+') LIKE '%+codex+%' OR ('+' || model || '+') LIKE '%+codex:%')) ORDER BY created_at DESC LIMIT 100" - } - ] - } - ] -} diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 807f27689..ad1933703 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -1209,182 +1209,6 @@ } ] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 73 - }, - "id": 108, - "title": "AI Usage & Cost (per provider)", - "type": "row" - }, - { - "type": "timeseries", - "title": "Tokens/min by provider", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "id": 109, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 74 - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (provider) ((rate(gittensory_ai_input_tokens_total[5m]) + rate(gittensory_ai_output_tokens_total[5m])) * 60)", - "legendFormat": "{{provider}}" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "lineWidth": 2, - "fillOpacity": 10 - }, - "unit": "short" - } - } - }, - { - "type": "timeseries", - "title": "AI requests + fallbacks (last 1h)", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "id": 110, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 74 - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (model, effort) (increase(gittensory_ai_requests_total[1h]))", - "legendFormat": "{{model}} \u00b7 {{effort}}" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (primary, fallback) (increase(gittensory_ai_review_model_fallback_total[1h]))", - "legendFormat": "fallback {{primary}}\u2192{{fallback}}" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "lineWidth": 2, - "fillOpacity": 10 - }, - "unit": "short" - } - } - }, - { - "type": "timeseries", - "title": "Cumulative AI cost (USD) by provider", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "id": 111, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 82 - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (provider) (gittensory_ai_cost_usd_total) or vector(0)", - "legendFormat": "{{provider}}" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "lineWidth": 2, - "fillOpacity": 10 - }, - "unit": "currencyUSD" - } - } - }, - { - "type": "timeseries", - "title": "Total tokens by provider + kind", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "id": 112, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 82 - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (provider, kind) (gittensory_ai_input_tokens_total)", - "legendFormat": "{{provider}} {{kind}} in" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (provider, kind) (gittensory_ai_output_tokens_total)", - "legendFormat": "{{provider}} {{kind}} out" - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "lineWidth": 2, - "fillOpacity": 10 - }, - "unit": "short" - } - } - }, { "collapsed": false, "gridPos": { diff --git a/grafana/dashboards/orb-ai-usage.json b/grafana/dashboards/orb-ai-usage.json deleted file mode 100644 index 823163314..000000000 --- a/grafana/dashboards/orb-ai-usage.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "uid": "gittensory-orb-ai-usage", - "title": "Gittensory - ORB AI usage", - "tags": ["gittensory", "ai", "orb"], - "timezone": "browser", - "schemaVersion": 39, - "version": 4, - "refresh": "30s", - "time": { "from": "now-7d", "to": "now" }, - "description": "Durable, provider-neutral AI usage from ai_usage_events (migration 0109) — every configured provider (Codex, Claude Code, OpenAI-compatible, Ollama, Anthropic BYOK), not just Codex. Every AI feature (ai_review_pr, agent_private_summary, agent_public_summary, pr_intelligence_comment, ai_slop_pr, issue_plan) threads real usage through recordAiUsageEvent on a completed call; a row still lands as estimated_neurons-only when the call never happened (quota_exceeded/disabled/unavailable), is a BYOK call (not yet usage-tracked), or the provider omitted a parseable usage field — see the 'Events missing real usage' panel. See 'Gittensory - Codex usage (self-host)' for a Codex-focused view and 'Gittensory — Claude usage (OTEL)' for Claude Code's own CLI/session telemetry (a separate, non-durable data source).", - "panels": [ - { - "id": 1, - "type": "row", - "title": "Summary (all providers)", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } - }, - { - "id": 2, - "type": "stat", - "title": "Total AI events", - "description": "Every ai_usage_events row with a non-null provider (i.e. a real, attributed AI call), across all features.", - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL", - "rawQueryText": "SELECT count(*) AS events FROM ai_usage_events WHERE provider IS NOT NULL" - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Total tokens", - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT coalesce(sum(total_tokens), 0) AS total_tokens FROM ai_usage_events WHERE provider IS NOT NULL", - "rawQueryText": "SELECT coalesce(sum(total_tokens), 0) AS total_tokens FROM ai_usage_events WHERE provider IS NOT NULL" - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Total cost", - "gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "currencyUSD", "decimals": 4, "color": { "mode": "fixed", "fixedColor": "green" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL", - "rawQueryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL" - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Events missing real usage", - "description": "Completed AI calls (status='ok', a response was actually returned) with no real usage captured — either no provider attribution at all, or a provider attributed with input_tokens/output_tokens/total_tokens/cost_usd all still at their zero default. (input_tokens/output_tokens/total_tokens/cost_usd are NOT NULL DEFAULT 0 per migration 0109 — recordAiUsageEvent's finiteNumber() coercion means they can never be SQL NULL, so a zero-value check, not an IS NULL check, is the real \"no data\" signal here; coerceAiUsage extracts provider independently of the numeric fields, so a provider name with no parseable token counts is a real, reachable row shape.) Excludes quota_exceeded/disabled/unavailable rows (no call was attempted, so no usage was ever possible) and ai_key_change audit rows (not an AI call at all — recordAiKeyChange's status is always \"set\"/\"replace\"/\"delete\", never \"ok\", but the feature is excluded explicitly here too so this panel doesn't rely on that invariant holding in another file). A nonzero count here means either a BYOK call (callAiProvider does not yet capture usage) or a provider response that omitted a parseable usage field.", - "gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "orange" } } }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE status = 'ok' AND feature <> 'ai_key_change' AND (provider IS NULL OR (input_tokens = 0 AND output_tokens = 0 AND total_tokens = 0 AND cost_usd = 0))", - "rawQueryText": "SELECT count(*) AS unattributed FROM ai_usage_events WHERE status = 'ok' AND feature <> 'ai_key_change' AND (provider IS NULL OR (input_tokens = 0 AND output_tokens = 0 AND total_tokens = 0 AND cost_usd = 0))" - } - ] - }, - { - "id": 6, - "type": "row", - "title": "By provider / model / effort", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } - }, - { - "id": 7, - "type": "table", - "title": "Usage by provider", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, - "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT provider, count(*) AS events, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC", - "rawQueryText": "SELECT provider, count(*) AS events, coalesce(sum(input_tokens), 0) AS input_tokens, coalesce(sum(output_tokens), 0) AS output_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider ORDER BY events DESC" - } - ] - }, - { - "id": 8, - "type": "table", - "title": "Usage by provider + model + effort", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, - "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT provider, model, effort, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50", - "rawQueryText": "SELECT provider, model, effort, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL GROUP BY provider, model, effort ORDER BY events DESC LIMIT 50" - } - ] - }, - { - "id": 9, - "type": "row", - "title": "By repo / PR", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 } - }, - { - "id": 10, - "type": "table", - "title": "Top repos by AI usage", - "gridPos": { "h": 10, "w": 12, "x": 0, "y": 16 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "events", "desc": true }] }, - "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50", - "rawQueryText": "SELECT json_extract(metadata_json, '$.repoFullName') AS repo, provider, count(*) AS events, coalesce(sum(total_tokens), 0) AS total_tokens, coalesce(sum(cost_usd), 0) AS cost_usd FROM ai_usage_events WHERE provider IS NOT NULL AND json_extract(metadata_json, '$.repoFullName') IS NOT NULL GROUP BY repo, provider ORDER BY events DESC LIMIT 50" - } - ] - }, - { - "id": 11, - "type": "table", - "title": "Recent AI events (all providers)", - "gridPos": { "h": 10, "w": 12, "x": 12, "y": 16 }, - "datasource": { "type": "frser-sqlite-datasource", "uid": "gittensory-db" }, - "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [{ "displayName": "created_at", "desc": true }] }, - "fieldConfig": { "defaults": { "custom": { "filterable": true } } }, - "targets": [ - { - "refId": "A", - "queryType": "table", - "queryText": "SELECT created_at, feature, provider, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE provider IS NOT NULL ORDER BY created_at DESC LIMIT 100", - "rawQueryText": "SELECT created_at, feature, provider, model, effort, status, input_tokens, output_tokens, total_tokens, cost_usd, json_extract(metadata_json, '$.repoFullName') AS repo, json_extract(metadata_json, '$.pullNumber') AS pr FROM ai_usage_events WHERE provider IS NOT NULL ORDER BY created_at DESC LIMIT 100" - } - ] - } - ] -} diff --git a/grafana/dashboards/resource-hub.json b/grafana/dashboards/resource-hub.json index 9a0e0a170..1e2183c1e 100644 --- a/grafana/dashboards/resource-hub.json +++ b/grafana/dashboards/resource-hub.json @@ -66,7 +66,7 @@ "gridPos": { "h": 13, "w": 12, "x": 12, "y": 0 }, "options": { "mode": "markdown", - "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[Claude usage (OTEL)](/d/gittensory-claude)** — cost / tokens / model / effort from the review CLI.\n- **[Codex usage (self-host)](/d/gittensory-codex)** — Codex review counts, token counters, and durable AI usage rows.\n- **[Gittensory (infra)](/d/gittensory)** — AI usage & cost, queue, jobs, HTTP.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | infra dashboard → *AI Usage & Cost* |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting." + "content": "## 📊 Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** — live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** — gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** — durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** — queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** — utilization/VRAM for a self-hosted Ollama GPU box.\n\n## 📈 Metrics & logs\n- **Prometheus** — [targets](http://localhost:9090/targets) · [graph](http://localhost:9090)\n- **Alertmanager** — [alerts](http://localhost:9093)\n- **Loki** — query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** — release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## 🩺 Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` → 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## 📚 Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) — setup, configuration, AI, REES, RAG, operations, and troubleshooting." } } ] diff --git a/test/unit/selfhost-grafana-ai-usage-dashboard.test.ts b/test/unit/selfhost-grafana-ai-usage-dashboard.test.ts new file mode 100644 index 000000000..ca612bd18 --- /dev/null +++ b/test/unit/selfhost-grafana-ai-usage-dashboard.test.ts @@ -0,0 +1,258 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +type DashboardTarget = { + expr?: string; + legendFormat?: string; + queryText?: string; + rawQueryText?: string; +}; + +type DashboardPanel = { + id?: number; + title?: string; + datasource?: { type?: string; uid?: string }; + targets?: DashboardTarget[]; +}; + +type TemplateVar = { + name: string; + type: string; + datasource?: { type?: string }; + query?: { rawQueryText?: string; query?: string }; + includeAll?: boolean; + multi?: boolean; + allValue?: string; +}; + +type Dashboard = { + uid: string; + title: string; + panels: DashboardPanel[]; + templating: { list: TemplateVar[] }; +}; + +const dashboardsDir = join(process.cwd(), "grafana/dashboards"); +const dashboardPath = join(dashboardsDir, "ai-usage.json"); +const timeFrom = "${__from:date:seconds}"; +const timeTo = "${__to:date:seconds}"; +const tmpRoots: string[] = []; + +const sqliteCliAvailable = (() => { + try { + execFileSync("sqlite3", ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +})(); + +function readDashboard(): Dashboard { + return JSON.parse(readFileSync(dashboardPath, "utf8")) as Dashboard; +} + +/** Excludes panel 5 ("Events missing real usage") — it's DELIBERATELY not scoped by $provider/$model, + * since NULL provider is exactly the condition it exists to surface (see its own dedicated test below). */ +function sqliteTargets(dashboard = readDashboard()): DashboardTarget[] { + return dashboard.panels + .filter((panel) => panel.id !== 5) + .flatMap((panel) => panel.targets ?? []) + .filter((target) => typeof target.queryText === "string" && target.queryText.includes("ai_usage_events")); +} + +function targetForPanel(panelId: number): DashboardTarget { + const panel = readDashboard().panels.find((candidate) => candidate.id === panelId); + const target = panel?.targets?.[0]; + if (!target?.queryText) throw new Error(`missing SQL query target for panel ${panelId}`); + return target; +} + +/** Simulate Grafana's own template-variable substitution for a direct sqlite3 CLI run: default every + * variable to "All" ('$__all' both sides of its OR clause) unless a caller substitutes a real value + * first, and expand the time-range placeholders to a fixed window. */ +function expandGrafanaRange(query: string): string { + const from = Math.floor(Date.parse("2026-07-01T00:00:00Z") / 1000); + const to = Math.floor(Date.parse("2026-07-02T00:00:00Z") / 1000); + return query + .replaceAll(timeFrom, String(from)) + .replaceAll(timeTo, String(to)) + .replaceAll("'$provider'", "'$__all'") + .replaceAll("'$feature'", "'$__all'") + .replaceAll("'$model'", "'$__all'"); +} + +function tmpRoot(): string { + const dir = mkdtempSync(join(tmpdir(), "gittensory-grafana-ai-usage-")); + tmpRoots.push(dir); + return dir; +} + +function sqlite(db: string, sql: string): string { + return execFileSync("sqlite3", [db, sql], { encoding: "utf8" }).trim(); +} + +afterEach(() => { + for (const dir of tmpRoots.splice(0)) rmSync(dir, { force: true, recursive: true }); +}); + +describe("Gittensory - AI usage dashboard (Phase B2 consolidation)", () => { + it("replaces the 3 old AI-usage dashboards, not just adds a 4th", () => { + const files = readdirSync(dashboardsDir); + expect(files).toContain("ai-usage.json"); + expect(files).not.toContain("claude-usage.json"); + expect(files).not.toContain("codex-usage.json"); + expect(files).not.toContain("orb-ai-usage.json"); + }); + + it("declares dynamic, query-backed $provider/$feature/$model template variables scoped to ai_usage_events, plus a separate $claudeModel for the OTEL section", () => { + const vars = readDashboard().templating.list; + const byName = Object.fromEntries(vars.map((v) => [v.name, v])); + + expect(byName.provider?.type).toBe("query"); + expect(byName.provider?.datasource?.type).toBe("frser-sqlite-datasource"); + expect(byName.provider?.query?.rawQueryText).toBe("SELECT DISTINCT provider FROM ai_usage_events WHERE provider IS NOT NULL ORDER BY provider"); + expect(byName.provider?.includeAll).toBe(true); + + expect(byName.feature?.type).toBe("query"); + expect(byName.feature?.query?.rawQueryText).toBe("SELECT DISTINCT feature FROM ai_usage_events ORDER BY feature"); + expect(byName.feature?.includeAll).toBe(true); + + expect(byName.model?.type).toBe("query"); + expect(byName.model?.query?.rawQueryText).toContain("SELECT DISTINCT model FROM ai_usage_events WHERE"); + expect(byName.model?.query?.rawQueryText).toContain("'$provider'"); + expect(byName.model?.includeAll).toBe(true); + + expect(byName.claudeModel?.type).toBe("query"); + expect(byName.claudeModel?.datasource?.type).toBe("prometheus"); + }); + + it("scopes every ai_usage_events panel query to $provider/$feature/$model AND the selected time window", () => { + const targets = sqliteTargets(); + expect(targets.length).toBeGreaterThan(0); + for (const target of targets) { + expect(target.queryText).toContain("('$provider' = '$__all' OR provider = '$provider')"); + expect(target.queryText).toContain("('$feature' = '$__all' OR feature = '$feature')"); + expect(target.queryText).toContain("('$model' = '$__all' OR model = '$model')"); + expect(target.queryText).toContain("unixepoch(created_at) >="); + expect(target.queryText).toContain("unixepoch(created_at) <"); + } + }); + + it("scopes the 'events missing real usage' panel by Feature and time only, never by Provider/Model (those are exactly the columns it's trying to catch as absent)", () => { + const target = targetForPanel(5); + expect(target.queryText).toContain("('$feature' = '$__all' OR feature = '$feature')"); + expect(target.queryText).not.toContain("'$provider'"); + expect(target.queryText).not.toContain("'$model'"); + expect(target.queryText).toContain("provider IS NULL"); + }); + + it("carries over the exact Prometheus expressions from the removed dashboards, byte-for-byte (no copy-paste drift)", () => { + const targets = readDashboard().panels.flatMap((panel) => panel.targets ?? []); + // From gittensory.json's removed "AI Usage & Cost" row. + expect(targets.some((t) => t.expr === "sum by (provider) (gittensory_ai_cost_usd_total) or vector(0)")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (provider) ((rate(gittensory_ai_input_tokens_total[5m]) + rate(gittensory_ai_output_tokens_total[5m])) * 60)")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (model, effort) (increase(gittensory_ai_requests_total[1h]))")).toBe(true); + expect(targets.some((t) => t.expr === "sum by (primary, fallback) (increase(gittensory_ai_review_model_fallback_total[1h]))")).toBe(true); + // From codex-usage.json. + expect(targets.some((t) => t.expr === "sum by (model, effort) (increase(gittensory_ai_requests_total{provider=\"codex\"}[$__rate_interval]))")).toBe(true); + // From claude-usage.json's OTEL section (uses $claudeModel, not $model, to stay independent of the durable-log filters). + expect(targets.some((t) => t.expr === "sum(last_over_time(claude_code_cost_usage_USD_total{model=~\"$claudeModel\"}[$__range]))")).toBe(true); + }); + + it("keeps the Claude OTEL section on its own $claudeModel variable, never the durable log's $provider/$feature/$model", () => { + const dashboard = readDashboard(); + const otelRowIndex = dashboard.panels.findIndex((p) => p.title?.includes("Claude Code native OTEL")); + expect(otelRowIndex).toBeGreaterThan(-1); + const otelPanels = dashboard.panels.slice(otelRowIndex + 1); + const otelTargets = otelPanels.flatMap((p) => p.targets ?? []); + expect(otelTargets.length).toBeGreaterThan(0); + for (const target of otelTargets) { + if (target.expr) { + expect(target.expr).not.toMatch(/\$provider\b|\$feature\b|(? { + const root = tmpRoot(); + const db = join(root, "reporting.sqlite"); + sqlite( + db, + ` + CREATE TABLE ai_usage_events ( + feature TEXT NOT NULL, + model TEXT NOT NULL, + provider TEXT, + effort TEXT, + status TEXT NOT NULL, + estimated_neurons INTEGER NOT NULL DEFAULT 0, + input_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, + cost_usd REAL NOT NULL DEFAULT 0, + detail TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL + ); + INSERT INTO ai_usage_events (feature, model, provider, status, total_tokens, cost_usd, created_at) + VALUES + ('ai_review_pr', 'claude-sonnet-5', 'claude-code', 'ok', 1000, 0.5, '2026-07-01T12:00:00Z'), + ('ai_slop_pr', 'gpt-5.5', 'codex', 'ok', 500, 0.2, '2026-07-01T13:00:00Z'), + ('embeddings', 'bge-m3:latest', 'ollama', 'ok', 100, 0, '2026-07-01T14:00:00Z'), + ('embeddings', 'bge-m3:latest', 'ollama', 'ok', 200, 0, '2026-07-01T15:00:00Z'); + `, + ); + + const totalEventsQuery = targetForPanel(2).queryText!; + const allEvents = sqlite(db, expandGrafanaRange(totalEventsQuery)); + const ollamaOnly = sqlite(db, expandGrafanaRange(totalEventsQuery.replaceAll("'$provider'", "'ollama'"))); + const embeddingsOnly = sqlite(db, expandGrafanaRange(totalEventsQuery.replaceAll("'$feature'", "'embeddings'"))); + const specificModel = sqlite(db, expandGrafanaRange(totalEventsQuery.replaceAll("'$model'", "'gpt-5.5'"))); + + expect(allEvents).toBe("4"); + expect(ollamaOnly).toBe("2"); + expect(embeddingsOnly).toBe("2"); + expect(specificModel).toBe("1"); + + const totalTokensQuery = targetForPanel(3).queryText!; + expect(sqlite(db, expandGrafanaRange(totalTokensQuery.replaceAll("'$provider'", "'ollama'")))).toBe("300"); + }); + + (sqliteCliAvailable ? it : it.skip)("finds rows missing real usage regardless of which provider/model they came from", () => { + const root = tmpRoot(); + const db = join(root, "reporting.sqlite"); + sqlite( + db, + ` + CREATE TABLE ai_usage_events ( + feature TEXT NOT NULL, + model TEXT NOT NULL, + provider TEXT, + effort TEXT, + status TEXT NOT NULL, + estimated_neurons INTEGER NOT NULL DEFAULT 0, + input_tokens INTEGER NOT NULL DEFAULT 0, + output_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, + cost_usd REAL NOT NULL DEFAULT 0, + detail TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL + ); + INSERT INTO ai_usage_events (feature, model, provider, status, total_tokens, cost_usd, created_at) + VALUES + ('ai_slop_pr', 'byok:anthropic', NULL, 'ok', 0, 0, '2026-07-01T12:00:00Z'), + ('ai_review_pr', 'claude-sonnet-5', 'claude-code', 'ok', 1000, 0.5, '2026-07-01T13:00:00Z'), + ('embeddings', 'bge-m3', NULL, 'error', 0, 0, '2026-07-01T14:00:00Z'); + `, + ); + + const missingUsageQuery = targetForPanel(5).queryText!; + // 'ai_slop_pr' row: provider NULL, status ok -> counted. 'embeddings' row: status='error', not 'ok' -> excluded + // by the panel's own status='ok' filter (mirrors orb-ai-usage.json's original semantics). + expect(sqlite(db, expandGrafanaRange(missingUsageQuery))).toBe("1"); + }); +}); diff --git a/test/unit/selfhost-grafana-dashboard.test.ts b/test/unit/selfhost-grafana-dashboard.test.ts index 0aa2802b0..a1b0339c4 100644 --- a/test/unit/selfhost-grafana-dashboard.test.ts +++ b/test/unit/selfhost-grafana-dashboard.test.ts @@ -97,8 +97,21 @@ describe("Gittensory Self-Host Grafana dashboard", () => { expect(targets.some((target) => target.expr === "sum by (kind, key_scope, job_type) (rate(gittensory_jobs_rate_limit_admission_deferred_total[5m])) or vector(0)")).toBe(true); expect(targets.some((target) => target.expr === "sum by (kind, key_scope, job_type) (rate(gittensory_jobs_rate_limit_budget_deferred_total[5m])) or vector(0)")).toBe(true); expect(targets.some((target) => target.expr === "sum by (kind, key_scope, job_type) (rate(gittensory_jobs_rate_limited_by_type_total[5m])) or vector(0)")).toBe(true); - expect(targets.some((target) => target.expr === "sum by (primary, fallback) (increase(gittensory_ai_review_model_fallback_total[1h]))")).toBe(true); - expect(targets.some((target) => target.legendFormat === "fallback {{primary}}→{{fallback}}")).toBe(true); + // The AI request/fallback + cost/token panels moved to the consolidated grafana/dashboards/ai-usage.json + // (Phase B2, 2026-07) — see test/unit/selfhost-grafana-ai-usage-dashboard.test.ts for their coverage there. + }); + + it("no longer duplicates the AI cost/token panels moved to the consolidated ai-usage.json dashboard (Phase B2, 2026-07 fix)", () => { + const dashboard = readDashboard(selfhostDashboardPath); + const titles = dashboard.panels.map((panel) => panel.title); + const ids = dashboard.panels.map((panel) => panel.id); + + expect(titles).not.toContain("AI Usage & Cost (per provider)"); + expect(titles).not.toContain("Tokens/min by provider"); + expect(titles).not.toContain("AI requests + fallbacks (last 1h)"); + expect(titles).not.toContain("Cumulative AI cost (USD) by provider"); + expect(titles).not.toContain("Total tokens by provider + kind"); + expect(ids).not.toEqual(expect.arrayContaining([108, 109, 110, 111, 112])); }); it("keeps Orb dashboard panels zero-safe when telemetry counters are absent", () => {