+ The metrics path is the app's own /metrics (scraped directly by Prometheus)
+ plus, for Claude Code's OTEL telemetry specifically, app or smoke process → OTEL collector →
+ its Prometheus exporter (:8889) → Prometheus.
+
otel-collector:4318/v1/metrics, the collector
+ is not reachable from the app container.
+ otel-collector:8889/metrics, the collector's Prometheus exporter is unhealthy
+ or the pipeline in otel/otel-collector-config.yml is misconfigured.
+ /metrics check fails, that is unrelated to the OTEL
+ collector — check the app container directly (docker compose logs gittensory
+ ).
+ src/{" "}
+ — a renamed or removed metric a dashboard still references renders as a permanent
+ zero/no-data, not an error.
+ The trace path is app or smoke process → OTEL collector → Tempo → Grafana. Tempo is only diff --git a/package.json b/package.json index 112052e64..7858f174e 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "test:coverage": "vitest run --coverage --pool=forks", "test:smoke:production": "node scripts/smoke-production.mjs", "test:smoke:observability": "node scripts/smoke-observability-traces.mjs", + "test:smoke:observability:metrics": "node scripts/smoke-observability-metrics.mjs", "test:smoke:browser:install": "playwright install chromium", "test:smoke:browser": "node scripts/smoke-ui-browser.mjs", "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:driver-parity && npm run test --workspace @jsonbored/gittensory-engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", diff --git a/scripts/smoke-observability-metrics.mjs b/scripts/smoke-observability-metrics.mjs new file mode 100755 index 000000000..c63425c59 --- /dev/null +++ b/scripts/smoke-observability-metrics.mjs @@ -0,0 +1,88 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; + +const composeService = process.env.SELFHOST_SERVICE ?? "gittensory"; +const timeoutMs = Number(process.env.OBSERVABILITY_SMOKE_TIMEOUT_MS ?? "30000"); +const pollIntervalMs = Number( + process.env.OBSERVABILITY_SMOKE_POLL_MS ?? "1000", +); +const metricName = `gittensory_selfhost_smoke_${Date.now()}_total`; + +await main(); + +async function main() { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) + throw new Error("OBSERVABILITY_SMOKE_TIMEOUT_MS must be a positive number"); + if (!Number.isFinite(pollIntervalMs) || pollIntervalMs <= 0) + throw new Error("OBSERVABILITY_SMOKE_POLL_MS must be a positive number"); + + const script = ` +const metricName = ${JSON.stringify(metricName)}; +const now = BigInt(Date.now()) * 1000000n; +const body = { + resourceMetrics: [{ + resource: { + attributes: [ + { key: "service.name", value: { stringValue: "gittensory-selfhost-smoke" } } + ] + }, + scopeMetrics: [{ + scope: { name: "gittensory-selfhost-smoke" }, + metrics: [{ + name: metricName, + sum: { + dataPoints: [{ + startTimeUnixNano: String(now), + timeUnixNano: String(now), + asInt: "1", + attributes: [{ key: "smoke.kind", value: { stringValue: "metrics" } }] + }], + aggregationTemporality: 2, + isMonotonic: true + } + }] + }] + }] +}; +const push = await fetch("http://otel-collector:4318/v1/metrics", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body) +}); +if (!push.ok) throw new Error("collector rejected smoke metric: " + push.status + " " + await push.text()); +const deadline = Date.now() + ${JSON.stringify(timeoutMs)}; +let last = ""; +while (Date.now() <= deadline) { + const res = await fetch("http://otel-collector:8889/metrics"); + if (res.ok) { + const text = await res.text(); + if (text.includes(metricName)) { + // Second check: the app's own /metrics is basic-shape sane (real HELP/TYPE lines exist), not just + // that the process answers 200. Independent of the collector path above -- this is the app's own + // in-process registry (src/selfhost/metrics.ts), not something the collector could mask a break in. + const appRes = await fetch("http://localhost:8787/metrics"); + if (!appRes.ok) throw new Error("app /metrics returned " + appRes.status); + const appText = await appRes.text(); + if (!appText.includes("# HELP gittensory_uptime_seconds") || !appText.includes("# TYPE gittensory_uptime_seconds gauge")) { + throw new Error("app /metrics is missing expected HELP/TYPE shape for gittensory_uptime_seconds"); + } + console.log(JSON.stringify({ ok: true, metricName })); + process.exit(0); + } + last = "collector /metrics did not contain the smoke metric yet"; + } else { + last = res.status + " " + await res.text(); + } + await new Promise((resolve) => setTimeout(resolve, ${JSON.stringify(pollIntervalMs)})); +} +throw new Error("otel-collector did not re-expose smoke metric " + metricName + ": " + last.slice(0, 300)); +`; + + execFileSync( + "docker", + ["compose", "exec", "-T", composeService, "node", "-e", script], + { + stdio: "inherit", + }, + ); +} diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts index 57484c129..1e18e7663 100644 --- a/test/unit/selfhost-observability-config.test.ts +++ b/test/unit/selfhost-observability-config.test.ts @@ -89,6 +89,24 @@ describe("self-host observability trace config", () => { expect(script).toContain("selfhost.observability.smoke"); }); + it("ships an operator smoke probe that verifies collector-to-Prometheus-exporter metrics retrieval, plus the app's own /metrics shape (2026-07 fix)", () => { + const script = readFileSync( + join(process.cwd(), "scripts/smoke-observability-metrics.mjs"), + "utf8", + ); + + expect(script).toContain("http://otel-collector:4318/v1/metrics"); + expect(script).toContain("http://otel-collector:8889/metrics"); + expect(script).toContain("http://localhost:8787/metrics"); + expect(script).toContain("gittensory-selfhost-smoke"); + expect(script).toContain("# HELP gittensory_uptime_seconds"); + + const packageJson = JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8")); + expect(packageJson.scripts["test:smoke:observability:metrics"]).toBe( + "node scripts/smoke-observability-metrics.mjs", + ); + }); + it("wires Postgres and backup exporters without starting them on SQLite-only observability", () => { const compose = record(readYaml("docker-compose.yml")); const services = record(compose.services);