feat(observability): add a metrics-pipeline smoke test

apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx

@@ -299,6 +299,44 @@ curl -X DELETE "$QDRANT_URL/collections/gittensory"`}
         </li>
       </ul>
 
+      <h2>Grafana metrics/dashboards show no data</h2>
+      <p>
+        The metrics path is the app's own <code>/metrics</code> (scraped directly by Prometheus)
+        plus, for Claude Code's OTEL telemetry specifically, app or smoke process → OTEL collector →
+        its Prometheus exporter (<code>:8889</code>) → Prometheus.
+      </p>
+      <CodeBlock
+        lang="bash"
+        code={`docker compose --profile observability ps prometheus otel-collector grafana
+docker compose logs --tail=80 prometheus otel-collector grafana
+
+# Send one synthetic OTLP metric through the collector, read it back from its Prometheus exporter,
+# and sanity-check the app's own /metrics HELP/TYPE shape.
+npm run test:smoke:observability:metrics`}
+      />
+      <ul>
+        <li>
+          If the smoke command fails at <code>otel-collector:4318/v1/metrics</code>, the collector
+          is not reachable from the app container.
+        </li>
+        <li>
+          If it pushes successfully but cannot read it back from{" "}
+          <code>otel-collector:8889/metrics</code>, the collector's Prometheus exporter is unhealthy
+          or the pipeline in <code>otel/otel-collector-config.yml</code> is misconfigured.
+        </li>
+        <li>
+          If the app's own <code>/metrics</code> check fails, that is unrelated to the OTEL
+          collector — check the app container directly (<code>docker compose logs gittensory</code>
+          ).
+        </li>
+        <li>
+          If the smoke command passes but a Grafana dashboard panel is still blank, check that
+          panel's own PromQL expression against a metric name actually emitted in <code>src/</code>{" "}
+          — a renamed or removed metric a dashboard still references renders as a permanent
+          zero/no-data, not an error.
+        </li>
+      </ul>
+
       <h2>Grafana traces error or show no data</h2>
       <p>
         The trace path is app or smoke process → OTEL collector → Tempo → Grafana. Tempo is only

package.json

@@ -78,6 +78,7 @@
     "test:coverage": "vitest run --coverage --pool=forks",
     "test:smoke:production": "node scripts/smoke-production.mjs",
     "test:smoke:observability": "node scripts/smoke-observability-traces.mjs",
+    "test:smoke:observability:metrics": "node scripts/smoke-observability-metrics.mjs",
     "test:smoke:browser:install": "playwright install chromium",
     "test:smoke:browser": "node scripts/smoke-ui-browser.mjs",
     "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:driver-parity && npm run test --workspace @jsonbored/gittensory-engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",

scripts/smoke-observability-metrics.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+import { execFileSync } from "node:child_process";
+
+const composeService = process.env.SELFHOST_SERVICE ?? "gittensory";
+const timeoutMs = Number(process.env.OBSERVABILITY_SMOKE_TIMEOUT_MS ?? "30000");
+const pollIntervalMs = Number(
+  process.env.OBSERVABILITY_SMOKE_POLL_MS ?? "1000",
+);
+const metricName = `gittensory_selfhost_smoke_${Date.now()}_total`;
+
+await main();
+
+async function main() {
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0)
+    throw new Error("OBSERVABILITY_SMOKE_TIMEOUT_MS must be a positive number");
+  if (!Number.isFinite(pollIntervalMs) || pollIntervalMs <= 0)
+    throw new Error("OBSERVABILITY_SMOKE_POLL_MS must be a positive number");
+
+  const script = `
+const metricName = ${JSON.stringify(metricName)};
+const now = BigInt(Date.now()) * 1000000n;
+const body = {
+  resourceMetrics: [{
+    resource: {
+      attributes: [
+        { key: "service.name", value: { stringValue: "gittensory-selfhost-smoke" } }
+      ]
+    },
+    scopeMetrics: [{
+      scope: { name: "gittensory-selfhost-smoke" },
+      metrics: [{
+        name: metricName,
+        sum: {
+          dataPoints: [{
+            startTimeUnixNano: String(now),
+            timeUnixNano: String(now),
+            asInt: "1",
+            attributes: [{ key: "smoke.kind", value: { stringValue: "metrics" } }]
+          }],
+          aggregationTemporality: 2,
+          isMonotonic: true
+        }
+      }]
+    }]
+  }]
+};
+const push = await fetch("http://otel-collector:4318/v1/metrics", {
+  method: "POST",
+  headers: { "content-type": "application/json" },
+  body: JSON.stringify(body)
+});
+if (!push.ok) throw new Error("collector rejected smoke metric: " + push.status + " " + await push.text());
+const deadline = Date.now() + ${JSON.stringify(timeoutMs)};
+let last = "";
+while (Date.now() <= deadline) {
+  const res = await fetch("http://otel-collector:8889/metrics");
+  if (res.ok) {
+    const text = await res.text();
+    if (text.includes(metricName)) {
+      // Second check: the app's own /metrics is basic-shape sane (real HELP/TYPE lines exist), not just
+      // that the process answers 200. Independent of the collector path above -- this is the app's own
+      // in-process registry (src/selfhost/metrics.ts), not something the collector could mask a break in.
+      const appRes = await fetch("http://localhost:8787/metrics");
+      if (!appRes.ok) throw new Error("app /metrics returned " + appRes.status);
+      const appText = await appRes.text();
+      if (!appText.includes("# HELP gittensory_uptime_seconds") || !appText.includes("# TYPE gittensory_uptime_seconds gauge")) {
+        throw new Error("app /metrics is missing expected HELP/TYPE shape for gittensory_uptime_seconds");
+      }
+      console.log(JSON.stringify({ ok: true, metricName }));
+      process.exit(0);
+    }
+    last = "collector /metrics did not contain the smoke metric yet";
+  } else {
+    last = res.status + " " + await res.text();
+  }
+  await new Promise((resolve) => setTimeout(resolve, ${JSON.stringify(pollIntervalMs)}));
+}
+throw new Error("otel-collector did not re-expose smoke metric " + metricName + ": " + last.slice(0, 300));
+`;
+
+  execFileSync(
+    "docker",
+    ["compose", "exec", "-T", composeService, "node", "-e", script],
+    {
+      stdio: "inherit",
+    },
+  );
+}

test/unit/selfhost-observability-config.test.ts

@@ -89,6 +89,24 @@ describe("self-host observability trace config", () => {
     expect(script).toContain("selfhost.observability.smoke");
   });
 
+  it("ships an operator smoke probe that verifies collector-to-Prometheus-exporter metrics retrieval, plus the app's own /metrics shape (2026-07 fix)", () => {
+    const script = readFileSync(
+      join(process.cwd(), "scripts/smoke-observability-metrics.mjs"),
+      "utf8",
+    );
+
+    expect(script).toContain("http://otel-collector:4318/v1/metrics");
+    expect(script).toContain("http://otel-collector:8889/metrics");
+    expect(script).toContain("http://localhost:8787/metrics");
+    expect(script).toContain("gittensory-selfhost-smoke");
+    expect(script).toContain("# HELP gittensory_uptime_seconds");
+
+    const packageJson = JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8"));
+    expect(packageJson.scripts["test:smoke:observability:metrics"]).toBe(
+      "node scripts/smoke-observability-metrics.mjs",
+    );
+  });
+
   it("wires Postgres and backup exporters without starting them on SQLite-only observability", () => {
     const compose = record(readYaml("docker-compose.yml"));
     const services = record(compose.services);