diff --git a/.env.example b/.env.example index f5014c405..844b48eed 100644 --- a/.env.example +++ b/.env.example @@ -340,6 +340,10 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # LOKI_MEM_LIMIT=1g # --profile observability # TEMPO_MEM_LIMIT=1g # --profile observability # GRAFANA_MEM_LIMIT=512m # --profile observability +# ALERTMANAGER_MEM_LIMIT=256m # --profile observability +# DOCKER_PROXY_MEM_LIMIT=64m # --profile observability +# PROMTAIL_MEM_LIMIT=256m # --profile observability +# OTEL_COLLECTOR_MEM_LIMIT=512m # --profile observability # --- Continuous backup (optional; the Litestream sidecar in docker-compose.yml) --- # Blank is valid until --profile litestream is enabled. diff --git a/docker-compose.yml b/docker-compose.yml index 0cb369393..7e61da741 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -625,6 +625,10 @@ services: command: - "--config.file=/etc/alertmanager/alertmanager.yml" - "--storage.path=/alertmanager" + deploy: + resources: + limits: + memory: "${ALERTMANAGER_MEM_LIMIT:-256m}" grafana: image: grafana/grafana:13.1.0 @@ -766,6 +770,10 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock:ro networks: [docker-proxy] # isolated — only Promtail joins this network + deploy: + resources: + limits: + memory: "${DOCKER_PROXY_MEM_LIMIT:-64m}" promtail: image: grafana/promtail:3.6.11 @@ -789,6 +797,10 @@ services: # the only service on `docker-proxy`, so the inspect endpoint (and the env it exposes) is unreachable # from any other container. No Docker-socket mount — discovery goes through the proxy's read-only API. networks: [default, docker-proxy] + deploy: + resources: + limits: + memory: "${PROMTAIL_MEM_LIMIT:-256m}" # ── OTEL collector (--profile observability) ────────────────────────────── # Receives OTLP metrics from the Claude Code review CLI and re-exposes them on :8889 for Prometheus to @@ -805,6 +817,10 @@ services: # Mount our config at the image's default path so the stock entrypoint picks it up (no command override). volumes: - ./otel/otel-collector-config.yml:/etc/otelcol-contrib/config.yaml:ro + deploy: + resources: + limits: + memory: "${OTEL_COLLECTOR_MEM_LIMIT:-512m}" # ── Tempo (--profile observability) — trace store for Claude review spans ───── # Receives traces forwarded by the OTEL collector and serves them to Grafana (the Tempo data source). diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index 00e31cc13..5cca89c77 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -33,7 +33,7 @@ const gauges = new Map(); // force every reader (including renderMetrics' gauges loop) to branch on which shape a given entry holds. const gaugeVectors = new Map(); const histograms = new Map(); -const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ +export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_queue_pending", { help: "Current in-process queue depth.", type: "gauge" }], ["gittensory_queue_dead", { help: "Current in-process dead queue depth.", type: "gauge" }], ["gittensory_dlq_dead_lettered_recent", { help: "DLQ messages dead-lettered within the recent trailing window, sampled at scrape.", type: "gauge" }], @@ -140,6 +140,32 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_d1_table_row_count", { help: "Row count for a monitored D1 table, from the same probe as gittensory_d1_database_size_bytes, labeled by table.", type: "gauge" }], ["gittensory_signal_snapshots_rows_per_key", { help: "signal_snapshots row count divided by its distinct (signal_type, target_key) count, scoped to the latest-only-dedup signal types dedupeSignalSnapshots converges to ~1 row per key; -1 when the probe is disabled or has never completed a successful sample.", type: "gauge" }], ["gittensory_d1_probe_errors_total", { help: "D1 size/row-count Management API probe failures, by part (database_info/table_row_count).", type: "counter" }], + ["gittensory_agent_action_permission_denied_total", { help: "Agent actions denied for missing a required GitHub App write permission, by action class.", type: "counter" }], + ["gittensory_agent_action_permission_denied_suppressed_total", { help: "Repeat permission denials suppressed within the cooldown window (still counted here, but not re-audited), by action class.", type: "counter" }], + ["gittensory_ai_review_frozen_reuse_total", { help: "AI review passes that reused a frozen (maintainer-gated) prior verdict instead of re-running.", type: "counter" }], + ["gittensory_ai_review_one_shot_reuse_total", { help: "AI review passes that reused a one-shot prior verdict instead of re-running.", type: "counter" }], + ["gittensory_ai_review_paused_reuse_total", { help: "AI review passes that reused a prior verdict because the repo is paused.", type: "counter" }], + ["gittensory_ai_review_tiebreak_order_unstable_total", { help: "Dual-reviewer tiebreak passes where reviewer order was not stable, by combine mode.", type: "counter" }], + ["gittensory_grounding_cache_hit_total", { help: "Review grounding-context cache hits.", type: "counter" }], + ["gittensory_grounding_cache_miss_total", { help: "Review grounding-context cache misses.", type: "counter" }], + ["gittensory_impact_map_cache_hit_total", { help: "Impact-map cache hits.", type: "counter" }], + ["gittensory_impact_map_cache_miss_total", { help: "Impact-map cache misses.", type: "counter" }], + ["gittensory_installation_health_broker_probe_total", { help: "Installation-health broker probes, by result (ok/failed/mismatched_installation).", type: "counter" }], + ["gittensory_jobs_maintenance_admission_granted_under_pressure_total", { help: "Maintenance jobs admitted despite backpressure via the trickle-admission allowance.", type: "counter" }], + ["gittensory_jobs_maintenance_trickle_admitted_by_type_total", { help: "Maintenance jobs admitted via trickle admission, by job type.", type: "counter" }], + ["gittensory_linked_issue_satisfaction_cache_hit_total", { help: "Linked-issue satisfaction assessment cache hits.", type: "counter" }], + ["gittensory_linked_issue_satisfaction_cache_miss_total", { help: "Linked-issue satisfaction assessment cache misses.", type: "counter" }], + ["gittensory_linked_issue_satisfaction_cache_write_error_total", { help: "Linked-issue satisfaction assessment cache write errors.", type: "counter" }], + ["gittensory_open_pr_reconciliation_missing_total", { help: "Open PRs found missing from local tracking during reconciliation, by repo.", type: "counter" }], + ["gittensory_orb_relay_malformed_events_total", { help: "Orb relay batch entries dropped for missing/mistyped required fields (deliveryId/eventName/rawBody).", type: "counter" }], + ["gittensory_orb_relay_register_total", { help: "Orb relay registration attempts, by mode and result (registered/recovered/failed).", type: "counter" }], + ["gittensory_pr_outcomes_total", { help: "Recorded PR gate outcomes, by decision.", type: "counter" }], + ["gittensory_public_origin_acknowledged", { help: "1 when the configured public origin is acknowledged as reachable; 0 otherwise.", type: "gauge" }], + ["gittensory_repo_culture_profile_cache_hit_total", { help: "Repo-culture-profile cache hits.", type: "counter" }], + ["gittensory_repo_culture_profile_cache_miss_total", { help: "Repo-culture-profile cache misses.", type: "counter" }], + ["gittensory_review_memory_cache_hit_total", { help: "Review-memory cache hits.", type: "counter" }], + ["gittensory_review_memory_cache_miss_total", { help: "Review-memory cache misses.", type: "counter" }], + ["gittensory_review_memory_suppressed_total", { help: "Review-memory entries suppressed from surfacing, by repo.", type: "counter" }], ]; const metricMeta = new Map(DEFAULT_METRIC_META); diff --git a/test/unit/selfhost-compose-resource-limits.test.ts b/test/unit/selfhost-compose-resource-limits.test.ts index ef38b72aa..036ce9c7f 100644 --- a/test/unit/selfhost-compose-resource-limits.test.ts +++ b/test/unit/selfhost-compose-resource-limits.test.ts @@ -25,6 +25,10 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893) tempo: "${TEMPO_MEM_LIMIT:-1g}", grafana: "${GRAFANA_MEM_LIMIT:-512m}", runner: "${RUNNER_MEM_LIMIT:-2g}", + alertmanager: "${ALERTMANAGER_MEM_LIMIT:-256m}", + "docker-proxy": "${DOCKER_PROXY_MEM_LIMIT:-64m}", + promtail: "${PROMTAIL_MEM_LIMIT:-256m}", + "otel-collector": "${OTEL_COLLECTOR_MEM_LIMIT:-512m}", }; it("caps the core app and every heavyweight optional service with an operator-overridable memory limit", () => { @@ -53,6 +57,10 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893) "TEMPO_MEM_LIMIT", "GRAFANA_MEM_LIMIT", "RUNNER_MEM_LIMIT", + "ALERTMANAGER_MEM_LIMIT", + "DOCKER_PROXY_MEM_LIMIT", + "PROMTAIL_MEM_LIMIT", + "OTEL_COLLECTOR_MEM_LIMIT", ]) { expect(env, key).toContain(key); } diff --git a/test/unit/selfhost-metrics.test.ts b/test/unit/selfhost-metrics.test.ts index fddc5d31d..2cb60e70c 100644 --- a/test/unit/selfhost-metrics.test.ts +++ b/test/unit/selfhost-metrics.test.ts @@ -1,6 +1,8 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { backupAcknowledgedGaugeValue } from "../../src/selfhost/health"; -import { counterValue, gauge, gaugeVector, hitRatio, incr, observe, registerMetricMeta, renderMetrics, resetMetrics, setSelfHostedMetricsMode } from "../../src/selfhost/metrics"; +import { DEFAULT_METRIC_META, counterValue, gauge, gaugeVector, hitRatio, incr, observe, registerMetricMeta, renderMetrics, resetMetrics, setSelfHostedMetricsMode } from "../../src/selfhost/metrics"; afterEach(() => { resetMetrics(); @@ -354,3 +356,32 @@ describe("hitRatio (#2090)", () => { expect(counterValue("gittensory_redis_gh_response_cache_total", { result: "set" })).toBe(0); }); }); + +describe("DEFAULT_METRIC_META completeness (drift guard, 2026-07 fix)", () => { + // Recursively walk src/ collecting every incr(/gauge(/gaugeVector(/observe( call site that uses a literal + // string metric name (dynamically-built names can't be statically checked this way, and there are none + // today). Every one of those names must carry HELP/TYPE metadata, or renderMetrics() silently emits it as a + // bare, undocumented sample forever -- exactly the class of gap #1943's audit found ~15 of. + function collectTsFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) out.push(...collectTsFiles(full)); + else if (entry.name.endsWith(".ts") && !entry.name.endsWith(".d.ts")) out.push(full); + } + return out; + } + + it("every literal metric name emitted anywhere in src/ has a registered DEFAULT_METRIC_META entry", () => { + const registered = new Set(DEFAULT_METRIC_META.map(([name]) => name)); + const used = new Set(); + const pattern = /\b(?:incr|gauge|gaugeVector|observe)\(\s*"([a-z0-9_]+)"/g; + for (const file of collectTsFiles(join(process.cwd(), "src"))) { + const contents = readFileSync(file, "utf8"); + for (const match of contents.matchAll(pattern)) used.add(match[1]!); + } + expect(used.size).toBeGreaterThan(50); // sanity: the scan found the real call sites, not an empty tree + const missing = [...used].filter((name) => !registered.has(name)).sort(); + expect(missing).toEqual([]); + }); +});