feat(miner): generated env-var reference with CI drift check (#5179)

package.json

@@ -17,6 +17,8 @@
     "selfhost:postgres:migrate": "tsx scripts/migrate-selfhost-sqlite-to-postgres.ts",
     "selfhost:env-reference": "node scripts/gen-selfhost-env-reference.mjs",
     "selfhost:env-reference:check": "node scripts/gen-selfhost-env-reference.mjs --check",
+    "miner:env-reference": "node packages/gittensory-miner/scripts/generate-env-reference.mjs",
+    "miner:env-reference:check": "node packages/gittensory-miner/scripts/generate-env-reference.mjs --check",
     "command-reference": "node scripts/gen-command-reference.mjs",
     "command-reference:check": "node scripts/gen-command-reference.mjs --check",
     "selfhost:validate-observability": "node scripts/validate-observability-configs.mjs",
@@ -78,7 +80,7 @@
     "test:smoke:observability": "node scripts/smoke-observability-traces.mjs",
     "test:smoke:browser:install": "playwright install chromium",
     "test:smoke:browser": "node scripts/smoke-ui-browser.mjs",
-    "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:driver-parity && npm run test --workspace @jsonbored/gittensory-engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
+    "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:driver-parity && npm run test --workspace @jsonbored/gittensory-engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
     "test:release": "npm run test:ci && npm run changelog:check",
     "test:release:mcp": "npm run test:ci",
     "test:watch": "vitest",

packages/gittensory-miner/README.md

@@ -13,6 +13,9 @@ Current scope is intentionally small:
 - `--help` and `version` commands
 - startup npm version nudge (override with `--no-update-check` or `GITTENSORY_MINER_NO_UPDATE_CHECK=1`)
 
+Environment variables read by the miner are documented in [`docs/env-reference.md`](docs/env-reference.md).
+Regenerate that file with `npm run miner:env-reference` from the repo root after adding or removing env reads.
+
 Real miner commands land in follow-up issues.
 
 The package also includes the first metadata-only discovery primitive: `fetchCandidateIssues` lists open issue

packages/gittensory-miner/docs/env-reference.md

@@ -0,0 +1,26 @@
+# gittensory-miner environment variable reference
+
+Generated by `npm run miner:env-reference`. Do not edit manually.
+
+| Name | First reference | Default |
+| --- | --- | --- |
+| `GITTENSORY_MINER_AMS_POLICY_PATH` | `lib/ams-policy.js` | (none) |
+| `GITTENSORY_MINER_ATTEMPT_LOG_DB` | `lib/attempt-log.js` | (none) |
+| `GITTENSORY_MINER_CLAIM_LEDGER_DB` | `lib/claim-ledger.js` | (none) |
+| `GITTENSORY_MINER_CONFIG_DIR` | `lib/deny-hook-synthesis.js` | `""` |
+| `GITTENSORY_MINER_DENY_HOOK_SYNTHESIS_DB` | `lib/deny-hook-synthesis.js` | `""` |
+| `GITTENSORY_MINER_EVENT_LEDGER_DB` | `lib/event-ledger.js` | (none) |
+| `GITTENSORY_MINER_GOVERNOR_LEDGER_DB` | `lib/governor-ledger.js` | `""` |
+| `GITTENSORY_MINER_GOVERNOR_STATE_DB` | `lib/governor-state.js` | (none) |
+| `GITTENSORY_MINER_NO_UPDATE_CHECK` | `lib/update-check.js` | `""` |
+| `GITTENSORY_MINER_ORB_EXPORT_DB` | `lib/orb-export.js` | `""` |
+| `GITTENSORY_MINER_PLAN_STORE_DB` | `lib/plan-store.js` | `""` |
+| `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.js` | (none) |
+| `GITTENSORY_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.js` | `""` |
+| `GITTENSORY_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.js` | (none) |
+| `GITTENSORY_MINER_REPO_CLONE_DIR` | `lib/repo-clone.js` | `""` |
+| `GITTENSORY_MINER_RUN_STATE_DB` | `lib/run-state.js` | (none) |
+| `GITTENSORY_MINER_VERSION` | `lib/version.js` | `""` |
+| `GITTENSORY_MINER_WORKTREE_ALLOCATOR_DB` | `lib/worktree-allocator.js` | `""` |
+| `GITTENSORY_MINER_WORKTREE_DIR` | `lib/worktree-allocator.js` | `""` |
+| `MINER_CODING_AGENT_PROVIDER` | `lib/laptop-init.js` | (none) |

packages/gittensory-miner/scripts/generate-env-reference.d.mts

@@ -0,0 +1,39 @@
+export type MinerEnvReferenceRow = {
+  name: string;
+  firstReference: string;
+  defaultValue: string | null;
+};
+
+export type MinerEnvReferenceOptions = {
+  rootDir?: string;
+  sourceRoots?: readonly string[];
+};
+
+export type WriteMinerEnvReferenceOptions = MinerEnvReferenceOptions & {
+  outputPath?: string;
+  check?: boolean;
+};
+
+export declare const DEFAULT_OUTPUT_PATH: string;
+export declare const DEFAULT_SOURCE_ROOTS: readonly string[];
+
+export declare function isMinerEnvVar(name: string): boolean;
+
+export declare function collectMinerEnvDefaults(
+  rootDir: string,
+  sourceRoots?: readonly string[],
+): Map<string, string>;
+
+export declare function collectMinerEnvVars(
+  options?: MinerEnvReferenceOptions,
+): MinerEnvReferenceRow[];
+
+export declare function renderMinerEnvReferenceMarkdown(rows: MinerEnvReferenceRow[]): string;
+
+export declare function writeMinerEnvReference(
+  options?: WriteMinerEnvReferenceOptions,
+): {
+  changed: boolean;
+  outputPath: string;
+  rows: MinerEnvReferenceRow[];
+};

packages/gittensory-miner/scripts/generate-env-reference.mjs

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { dirname, extname, relative, resolve, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+import { collectSelfHostEnvVars } from "../../../scripts/gen-selfhost-env-reference.mjs";
+
+export const DEFAULT_OUTPUT_PATH = "packages/gittensory-miner/docs/env-reference.md";
+export const DEFAULT_SOURCE_ROOTS = [
+  "packages/gittensory-miner/bin",
+  "packages/gittensory-miner/lib",
+  "packages/gittensory-miner/scripts",
+];
+
+const MINER_ENV_PREFIXES = ["GITTENSORY_MINER_", "MINER_"];
+const SOURCE_FILE_EXTENSIONS = new Set([".cjs", ".js", ".mjs", ".ts", ".tsx"]);
+const DEFAULT_PATTERNS = [
+  /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*""/g,
+  /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*''/g,
+  /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*"([^"]*)"/g,
+  /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*'([^']*)'/g,
+  /env\.([A-Z][A-Z0-9_]*)\.trim\(\)\s*:\s*""/g,
+  /env\.([A-Z][A-Z0-9_]*)\.trim\(\)\s*:\s*''/g,
+  /\{\s*([A-Z][A-Z0-9_]*)\s*=\s*"([^"]*)"\s*\}\s*=\s*process\.env/g,
+  /\{\s*([A-Z][A-Z0-9_]*)\s*=\s*'([^']*)'\s*\}\s*=\s*process\.env/g,
+];
+
+export function isMinerEnvVar(name) {
+  return MINER_ENV_PREFIXES.some((prefix) => name.startsWith(prefix));
+}
+
+export function collectMinerEnvDefaults(rootDir, sourceRoots = DEFAULT_SOURCE_ROOTS) {
+  const defaults = new Map();
+  for (const file of sourceFiles(rootDir, sourceRoots)) {
+    const abs = resolve(rootDir, file);
+    const source = readFileSync(abs, "utf8");
+    for (const pattern of DEFAULT_PATTERNS) {
+      pattern.lastIndex = 0;
+      for (const match of source.matchAll(pattern)) {
+        const name = match[1];
+        if (!isMinerEnvVar(name)) continue;
+        const value = match[2] ?? "";
+        if (!defaults.has(name)) defaults.set(name, value);
+      }
+    }
+  }
+  return defaults;
+}
+
+export function collectMinerEnvVars({
+  rootDir = process.cwd(),
+  sourceRoots = DEFAULT_SOURCE_ROOTS,
+} = {}) {
+  const defaults = collectMinerEnvDefaults(rootDir, sourceRoots);
+  const rows = collectSelfHostEnvVars({ rootDir, sourceRoots })
+    .filter((row) => isMinerEnvVar(row.name))
+    .map((row) => ({
+      name: row.name,
+      firstReference: row.firstReference.replace(/^packages\/gittensory-miner\//, ""),
+      defaultValue: defaults.has(row.name) ? defaults.get(row.name) : null,
+    }));
+  return rows;
+}
+
+export function renderMinerEnvReferenceMarkdown(rows) {
+  const header = [
+    "# gittensory-miner environment variable reference",
+    "",
+    "Generated by `npm run miner:env-reference`. Do not edit manually.",
+    "",
+    "| Name | First reference | Default |",
+    "| --- | --- | --- |",
+  ];
+  const body = rows.map((row) => {
+    const defaultCell =
+      row.defaultValue === null || row.defaultValue === undefined
+        ? "(none)"
+        : row.defaultValue === ""
+          ? "`\"\"`"
+          : `\`${JSON.stringify(row.defaultValue)}\``;
+    return `| \`${row.name}\` | \`${row.firstReference}\` | ${defaultCell} |`;
+  });
+  return [...header, ...body, ""].join("\n");
+}
+
+export function writeMinerEnvReference({
+  rootDir = process.cwd(),
+  outputPath = DEFAULT_OUTPUT_PATH,
+  sourceRoots = DEFAULT_SOURCE_ROOTS,
+  check = false,
+} = {}) {
+  const rows = collectMinerEnvVars({ rootDir, sourceRoots });
+  const output = renderMinerEnvReferenceMarkdown(rows);
+  const absOutput = resolve(rootDir, outputPath);
+  const current = existsSync(absOutput) ? readFileSync(absOutput, "utf8") : null;
+  const changed = current !== output;
+  if (!check && changed) {
+    mkdirSync(dirname(absOutput), { recursive: true });
+    writeFileSync(absOutput, output);
+  }
+  return { changed, outputPath, rows };
+}
+
+function sourceFiles(rootDir, sourceRoots) {
+  const files = [];
+  for (const sourceRoot of sourceRoots) {
+    const abs = resolve(rootDir, sourceRoot);
+    if (!existsSync(abs)) continue;
+    const stat = statSync(abs);
+    if (stat.isFile()) {
+      if (!isSupportedSourceFile(abs)) throw new Error(`Unsupported source root file extension: ${sourceRoot}`);
+      files.push(toPosixPath(relative(rootDir, abs)));
+      continue;
+    }
+    if (!stat.isDirectory()) throw new Error(`Unsupported source root: ${sourceRoot}`);
+    for (const file of walkSourceFiles(abs)) {
+      files.push(toPosixPath(relative(rootDir, file)));
+    }
+  }
+  return files;
+}
+
+function walkSourceFiles(dir) {
+  const files = [];
+  const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
+  for (const entry of entries) {
+    const abs = resolve(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...walkSourceFiles(abs));
+    } else if (entry.isFile() && isSupportedSourceFile(abs) && !entry.name.endsWith(".d.ts")) {
+      files.push(abs);
+    }
+  }
+  return files;
+}
+
+function isSupportedSourceFile(file) {
+  return SOURCE_FILE_EXTENSIONS.has(extname(file));
+}
+
+function toPosixPath(path) {
+  return path.split(sep).join("/");
+}
+
+function main(argv) {
+  const check = argv.includes("--check");
+  const result = writeMinerEnvReference({ check });
+  if (check && result.changed) {
+    process.stderr.write(
+      `generate-env-reference: ${result.outputPath} is stale; run npm run miner:env-reference.\n`,
+    );
+    process.exit(1);
+  }
+  process.stdout.write(
+    `generate-env-reference: ${check ? "checked" : "wrote"} ${result.rows.length} env var references in ${result.outputPath}\n`,
+  );
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
+  main(process.argv.slice(2));
+}

scripts/gen-selfhost-env-reference.mjs

@@ -79,6 +79,9 @@ function collectEnvReads(source, fileName) {
       addRead(node.arguments[1].text);
     } else if (ts.isCallExpression(node) && isProcessEnvNameHelperCall(node)) {
       addRead(node.arguments[0].text);
+    } else if (ts.isCallExpression(node) && isEnvNameLiteralArgHelperCall(node)) {
+      const argIndex = ENV_NAME_LITERAL_ARG_HELPERS.get(node.expression.text);
+      addRead(node.arguments[argIndex].text);
     }
     ts.forEachChild(node, visit);
   };
@@ -101,6 +104,7 @@ function isStaticEnvHelperCall(node) {
 // isStaticEnvHelperCall above (envString) because these take the var NAME as arg[0], not arg[1] after a
 // container.
 const PROCESS_ENV_NAME_HELPERS = new Set(["parsePositiveIntEnv"]);
+const ENV_NAME_LITERAL_ARG_HELPERS = new Map([["resolveLocalStoreDbPath", 1]]);
 
 function isProcessEnvNameHelperCall(node) {
   return (
@@ -111,6 +115,16 @@ function isProcessEnvNameHelperCall(node) {
   );
 }
 
+function isEnvNameLiteralArgHelperCall(node) {
+  if (!ts.isIdentifier(node.expression)) return false;
+  const argIndex = ENV_NAME_LITERAL_ARG_HELPERS.get(node.expression.text);
+  return (
+    argIndex !== undefined &&
+    node.arguments.length > argIndex &&
+    ts.isStringLiteralLike(node.arguments[argIndex])
+  );
+}
+
 function bindingElementName(element) {
   const candidate = element.propertyName ?? element.name;
   if (ts.isIdentifier(candidate) || ts.isStringLiteralLike(candidate)) return candidate.text;