diff --git a/package.json b/package.json index 2d46c226b..112052e64 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "selfhost:postgres:migrate": "tsx scripts/migrate-selfhost-sqlite-to-postgres.ts", "selfhost:env-reference": "node scripts/gen-selfhost-env-reference.mjs", "selfhost:env-reference:check": "node scripts/gen-selfhost-env-reference.mjs --check", + "miner:env-reference": "node packages/gittensory-miner/scripts/generate-env-reference.mjs", + "miner:env-reference:check": "node packages/gittensory-miner/scripts/generate-env-reference.mjs --check", "command-reference": "node scripts/gen-command-reference.mjs", "command-reference:check": "node scripts/gen-command-reference.mjs --check", "selfhost:validate-observability": "node scripts/validate-observability-configs.mjs", @@ -78,7 +80,7 @@ "test:smoke:observability": "node scripts/smoke-observability-traces.mjs", "test:smoke:browser:install": "playwright install chromium", "test:smoke:browser": "node scripts/smoke-ui-browser.mjs", - "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:driver-parity && npm run test --workspace @jsonbored/gittensory-engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", + "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:driver-parity && npm run test --workspace @jsonbored/gittensory-engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", "test:release": "npm run test:ci && npm run changelog:check", "test:release:mcp": "npm run test:ci", "test:watch": "vitest", diff --git a/packages/gittensory-miner/README.md b/packages/gittensory-miner/README.md index a0ca58d01..85c8083e8 100644 --- a/packages/gittensory-miner/README.md +++ b/packages/gittensory-miner/README.md @@ -13,6 +13,9 @@ Current scope is intentionally small: - `--help` and `version` commands - startup npm version nudge (override with `--no-update-check` or `GITTENSORY_MINER_NO_UPDATE_CHECK=1`) +Environment variables read by the miner are documented in [`docs/env-reference.md`](docs/env-reference.md). +Regenerate that file with `npm run miner:env-reference` from the repo root after adding or removing env reads. + Real miner commands land in follow-up issues. The package also includes the first metadata-only discovery primitive: `fetchCandidateIssues` lists open issue diff --git a/packages/gittensory-miner/docs/env-reference.md b/packages/gittensory-miner/docs/env-reference.md new file mode 100644 index 000000000..e5e9d4727 --- /dev/null +++ b/packages/gittensory-miner/docs/env-reference.md @@ -0,0 +1,26 @@ +# gittensory-miner environment variable reference + +Generated by `npm run miner:env-reference`. Do not edit manually. + +| Name | First reference | Default | +| --- | --- | --- | +| `GITTENSORY_MINER_AMS_POLICY_PATH` | `lib/ams-policy.js` | (none) | +| `GITTENSORY_MINER_ATTEMPT_LOG_DB` | `lib/attempt-log.js` | (none) | +| `GITTENSORY_MINER_CLAIM_LEDGER_DB` | `lib/claim-ledger.js` | (none) | +| `GITTENSORY_MINER_CONFIG_DIR` | `lib/deny-hook-synthesis.js` | `""` | +| `GITTENSORY_MINER_DENY_HOOK_SYNTHESIS_DB` | `lib/deny-hook-synthesis.js` | `""` | +| `GITTENSORY_MINER_EVENT_LEDGER_DB` | `lib/event-ledger.js` | (none) | +| `GITTENSORY_MINER_GOVERNOR_LEDGER_DB` | `lib/governor-ledger.js` | `""` | +| `GITTENSORY_MINER_GOVERNOR_STATE_DB` | `lib/governor-state.js` | (none) | +| `GITTENSORY_MINER_NO_UPDATE_CHECK` | `lib/update-check.js` | `""` | +| `GITTENSORY_MINER_ORB_EXPORT_DB` | `lib/orb-export.js` | `""` | +| `GITTENSORY_MINER_PLAN_STORE_DB` | `lib/plan-store.js` | `""` | +| `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.js` | (none) | +| `GITTENSORY_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.js` | `""` | +| `GITTENSORY_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.js` | (none) | +| `GITTENSORY_MINER_REPO_CLONE_DIR` | `lib/repo-clone.js` | `""` | +| `GITTENSORY_MINER_RUN_STATE_DB` | `lib/run-state.js` | (none) | +| `GITTENSORY_MINER_VERSION` | `lib/version.js` | `""` | +| `GITTENSORY_MINER_WORKTREE_ALLOCATOR_DB` | `lib/worktree-allocator.js` | `""` | +| `GITTENSORY_MINER_WORKTREE_DIR` | `lib/worktree-allocator.js` | `""` | +| `MINER_CODING_AGENT_PROVIDER` | `lib/laptop-init.js` | (none) | diff --git a/packages/gittensory-miner/scripts/generate-env-reference.d.mts b/packages/gittensory-miner/scripts/generate-env-reference.d.mts new file mode 100644 index 000000000..98318573f --- /dev/null +++ b/packages/gittensory-miner/scripts/generate-env-reference.d.mts @@ -0,0 +1,39 @@ +export type MinerEnvReferenceRow = { + name: string; + firstReference: string; + defaultValue: string | null; +}; + +export type MinerEnvReferenceOptions = { + rootDir?: string; + sourceRoots?: readonly string[]; +}; + +export type WriteMinerEnvReferenceOptions = MinerEnvReferenceOptions & { + outputPath?: string; + check?: boolean; +}; + +export declare const DEFAULT_OUTPUT_PATH: string; +export declare const DEFAULT_SOURCE_ROOTS: readonly string[]; + +export declare function isMinerEnvVar(name: string): boolean; + +export declare function collectMinerEnvDefaults( + rootDir: string, + sourceRoots?: readonly string[], +): Map; + +export declare function collectMinerEnvVars( + options?: MinerEnvReferenceOptions, +): MinerEnvReferenceRow[]; + +export declare function renderMinerEnvReferenceMarkdown(rows: MinerEnvReferenceRow[]): string; + +export declare function writeMinerEnvReference( + options?: WriteMinerEnvReferenceOptions, +): { + changed: boolean; + outputPath: string; + rows: MinerEnvReferenceRow[]; +}; diff --git a/packages/gittensory-miner/scripts/generate-env-reference.mjs b/packages/gittensory-miner/scripts/generate-env-reference.mjs new file mode 100644 index 000000000..153dbff77 --- /dev/null +++ b/packages/gittensory-miner/scripts/generate-env-reference.mjs @@ -0,0 +1,160 @@ +#!/usr/bin/env node +import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; +import { dirname, extname, relative, resolve, sep } from "node:path"; +import { pathToFileURL } from "node:url"; +import { collectSelfHostEnvVars } from "../../../scripts/gen-selfhost-env-reference.mjs"; + +export const DEFAULT_OUTPUT_PATH = "packages/gittensory-miner/docs/env-reference.md"; +export const DEFAULT_SOURCE_ROOTS = [ + "packages/gittensory-miner/bin", + "packages/gittensory-miner/lib", + "packages/gittensory-miner/scripts", +]; + +const MINER_ENV_PREFIXES = ["GITTENSORY_MINER_", "MINER_"]; +const SOURCE_FILE_EXTENSIONS = new Set([".cjs", ".js", ".mjs", ".ts", ".tsx"]); +const DEFAULT_PATTERNS = [ + /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*""/g, + /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*''/g, + /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*"([^"]*)"/g, + /env\.([A-Z][A-Z0-9_]*)\s*\?\?\s*'([^']*)'/g, + /env\.([A-Z][A-Z0-9_]*)\.trim\(\)\s*:\s*""/g, + /env\.([A-Z][A-Z0-9_]*)\.trim\(\)\s*:\s*''/g, + /\{\s*([A-Z][A-Z0-9_]*)\s*=\s*"([^"]*)"\s*\}\s*=\s*process\.env/g, + /\{\s*([A-Z][A-Z0-9_]*)\s*=\s*'([^']*)'\s*\}\s*=\s*process\.env/g, +]; + +export function isMinerEnvVar(name) { + return MINER_ENV_PREFIXES.some((prefix) => name.startsWith(prefix)); +} + +export function collectMinerEnvDefaults(rootDir, sourceRoots = DEFAULT_SOURCE_ROOTS) { + const defaults = new Map(); + for (const file of sourceFiles(rootDir, sourceRoots)) { + const abs = resolve(rootDir, file); + const source = readFileSync(abs, "utf8"); + for (const pattern of DEFAULT_PATTERNS) { + pattern.lastIndex = 0; + for (const match of source.matchAll(pattern)) { + const name = match[1]; + if (!isMinerEnvVar(name)) continue; + const value = match[2] ?? ""; + if (!defaults.has(name)) defaults.set(name, value); + } + } + } + return defaults; +} + +export function collectMinerEnvVars({ + rootDir = process.cwd(), + sourceRoots = DEFAULT_SOURCE_ROOTS, +} = {}) { + const defaults = collectMinerEnvDefaults(rootDir, sourceRoots); + const rows = collectSelfHostEnvVars({ rootDir, sourceRoots }) + .filter((row) => isMinerEnvVar(row.name)) + .map((row) => ({ + name: row.name, + firstReference: row.firstReference.replace(/^packages\/gittensory-miner\//, ""), + defaultValue: defaults.has(row.name) ? defaults.get(row.name) : null, + })); + return rows; +} + +export function renderMinerEnvReferenceMarkdown(rows) { + const header = [ + "# gittensory-miner environment variable reference", + "", + "Generated by `npm run miner:env-reference`. Do not edit manually.", + "", + "| Name | First reference | Default |", + "| --- | --- | --- |", + ]; + const body = rows.map((row) => { + const defaultCell = + row.defaultValue === null || row.defaultValue === undefined + ? "(none)" + : row.defaultValue === "" + ? "`\"\"`" + : `\`${JSON.stringify(row.defaultValue)}\``; + return `| \`${row.name}\` | \`${row.firstReference}\` | ${defaultCell} |`; + }); + return [...header, ...body, ""].join("\n"); +} + +export function writeMinerEnvReference({ + rootDir = process.cwd(), + outputPath = DEFAULT_OUTPUT_PATH, + sourceRoots = DEFAULT_SOURCE_ROOTS, + check = false, +} = {}) { + const rows = collectMinerEnvVars({ rootDir, sourceRoots }); + const output = renderMinerEnvReferenceMarkdown(rows); + const absOutput = resolve(rootDir, outputPath); + const current = existsSync(absOutput) ? readFileSync(absOutput, "utf8") : null; + const changed = current !== output; + if (!check && changed) { + mkdirSync(dirname(absOutput), { recursive: true }); + writeFileSync(absOutput, output); + } + return { changed, outputPath, rows }; +} + +function sourceFiles(rootDir, sourceRoots) { + const files = []; + for (const sourceRoot of sourceRoots) { + const abs = resolve(rootDir, sourceRoot); + if (!existsSync(abs)) continue; + const stat = statSync(abs); + if (stat.isFile()) { + if (!isSupportedSourceFile(abs)) throw new Error(`Unsupported source root file extension: ${sourceRoot}`); + files.push(toPosixPath(relative(rootDir, abs))); + continue; + } + if (!stat.isDirectory()) throw new Error(`Unsupported source root: ${sourceRoot}`); + for (const file of walkSourceFiles(abs)) { + files.push(toPosixPath(relative(rootDir, file))); + } + } + return files; +} + +function walkSourceFiles(dir) { + const files = []; + const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + const abs = resolve(dir, entry.name); + if (entry.isDirectory()) { + files.push(...walkSourceFiles(abs)); + } else if (entry.isFile() && isSupportedSourceFile(abs) && !entry.name.endsWith(".d.ts")) { + files.push(abs); + } + } + return files; +} + +function isSupportedSourceFile(file) { + return SOURCE_FILE_EXTENSIONS.has(extname(file)); +} + +function toPosixPath(path) { + return path.split(sep).join("/"); +} + +function main(argv) { + const check = argv.includes("--check"); + const result = writeMinerEnvReference({ check }); + if (check && result.changed) { + process.stderr.write( + `generate-env-reference: ${result.outputPath} is stale; run npm run miner:env-reference.\n`, + ); + process.exit(1); + } + process.stdout.write( + `generate-env-reference: ${check ? "checked" : "wrote"} ${result.rows.length} env var references in ${result.outputPath}\n`, + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)); +} diff --git a/scripts/gen-selfhost-env-reference.mjs b/scripts/gen-selfhost-env-reference.mjs index ab10fb6b6..ab8148f52 100644 --- a/scripts/gen-selfhost-env-reference.mjs +++ b/scripts/gen-selfhost-env-reference.mjs @@ -79,6 +79,9 @@ function collectEnvReads(source, fileName) { addRead(node.arguments[1].text); } else if (ts.isCallExpression(node) && isProcessEnvNameHelperCall(node)) { addRead(node.arguments[0].text); + } else if (ts.isCallExpression(node) && isEnvNameLiteralArgHelperCall(node)) { + const argIndex = ENV_NAME_LITERAL_ARG_HELPERS.get(node.expression.text); + addRead(node.arguments[argIndex].text); } ts.forEachChild(node, visit); }; @@ -101,6 +104,7 @@ function isStaticEnvHelperCall(node) { // isStaticEnvHelperCall above (envString) because these take the var NAME as arg[0], not arg[1] after a // container. const PROCESS_ENV_NAME_HELPERS = new Set(["parsePositiveIntEnv"]); +const ENV_NAME_LITERAL_ARG_HELPERS = new Map([["resolveLocalStoreDbPath", 1]]); function isProcessEnvNameHelperCall(node) { return ( @@ -111,6 +115,16 @@ function isProcessEnvNameHelperCall(node) { ); } +function isEnvNameLiteralArgHelperCall(node) { + if (!ts.isIdentifier(node.expression)) return false; + const argIndex = ENV_NAME_LITERAL_ARG_HELPERS.get(node.expression.text); + return ( + argIndex !== undefined && + node.arguments.length > argIndex && + ts.isStringLiteralLike(node.arguments[argIndex]) + ); +} + function bindingElementName(element) { const candidate = element.propertyName ?? element.name; if (ts.isIdentifier(candidate) || ts.isStringLiteralLike(candidate)) return candidate.text; diff --git a/test/unit/miner-env-reference-script.test.ts b/test/unit/miner-env-reference-script.test.ts new file mode 100644 index 000000000..9feaaa62a --- /dev/null +++ b/test/unit/miner-env-reference-script.test.ts @@ -0,0 +1,119 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + collectMinerEnvVars, + isMinerEnvVar, + renderMinerEnvReferenceMarkdown, + writeMinerEnvReference, +} from "../../packages/gittensory-miner/scripts/generate-env-reference.mjs"; + +function fixtureRoot(): string { + const root = mkdtempSync(join(tmpdir(), "gt-miner-env-reference-")); + mkdirSync(join(root, "packages", "gittensory-miner", "lib"), { recursive: true }); + writeFileSync( + join(root, "packages", "gittensory-miner", "lib", "a.js"), + [ + "export function readEnv(env = process.env) {", + " const ignored = process.env.OTHER_APP_VAR;", + " const config = env.GITTENSORY_MINER_CONFIG_DIR ?? '';", + " const provider = env.MINER_CODING_AGENT_PROVIDER ?? '';", + " return resolveLocalStoreDbPath('state.sqlite3', 'GITTENSORY_MINER_RUN_STATE_DB', env);", + "}", + "", + ].join("\n"), + ); + writeFileSync( + join(root, "packages", "gittensory-miner", "lib", "b.js"), + [ + "export function duplicate(env = process.env) {", + " return env.GITTENSORY_MINER_CONFIG_DIR;", + "}", + "", + ].join("\n"), + ); + return root; +} + +describe("generate-env-reference (#5179)", () => { + it("filters to GITTENSORY_MINER_* and MINER_* vars only", () => { + expect(isMinerEnvVar("GITTENSORY_MINER_CONFIG_DIR")).toBe(true); + expect(isMinerEnvVar("MINER_CODING_AGENT_PROVIDER")).toBe(true); + expect(isMinerEnvVar("OTHER_APP_VAR")).toBe(false); + }); + + it("extracts static env reads, resolveLocalStoreDbPath names, and first references", () => { + expect(collectMinerEnvVars({ rootDir: fixtureRoot() })).toEqual([ + { + name: "GITTENSORY_MINER_CONFIG_DIR", + firstReference: "lib/a.js", + defaultValue: "", + }, + { + name: "GITTENSORY_MINER_RUN_STATE_DB", + firstReference: "lib/a.js", + defaultValue: null, + }, + { + name: "MINER_CODING_AGENT_PROVIDER", + firstReference: "lib/a.js", + defaultValue: "", + }, + ]); + }); + + it("renders a deterministic markdown table with names, references, and defaults", () => { + expect( + renderMinerEnvReferenceMarkdown([ + { name: "GITTENSORY_MINER_CONFIG_DIR", firstReference: "lib/a.js", defaultValue: "" }, + { name: "GITTENSORY_MINER_RUN_STATE_DB", firstReference: "lib/run-state.js", defaultValue: null }, + ]), + ).toBe( + [ + "# gittensory-miner environment variable reference", + "", + "Generated by `npm run miner:env-reference`. Do not edit manually.", + "", + "| Name | First reference | Default |", + "| --- | --- | --- |", + "| `GITTENSORY_MINER_CONFIG_DIR` | `lib/a.js` | `\"\"` |", + "| `GITTENSORY_MINER_RUN_STATE_DB` | `lib/run-state.js` | (none) |", + "", + ].join("\n"), + ); + }); + + it("writes the generated doc and reports stale output in check mode", () => { + const root = fixtureRoot(); + const outputPath = "packages/gittensory-miner/docs/env-reference.md"; + const outputAbs = join(root, outputPath); + + const written = writeMinerEnvReference({ rootDir: root, outputPath }); + expect(written.changed).toBe(true); + expect(existsSync(outputAbs)).toBe(true); + const generated = readFileSync(outputAbs, "utf8"); + expect(generated).toContain("GITTENSORY_MINER_CONFIG_DIR"); + + expect(writeMinerEnvReference({ rootDir: root, outputPath, check: true }).changed).toBe(false); + + writeFileSync(outputAbs, "stale\n"); + const stale = writeMinerEnvReference({ rootDir: root, outputPath, check: true }); + expect(stale.changed).toBe(true); + expect(readFileSync(outputAbs, "utf8")).toBe("stale\n"); + + const rewritten = writeMinerEnvReference({ rootDir: root, outputPath }); + expect(rewritten.changed).toBe(true); + expect(readFileSync(outputAbs, "utf8")).toBe(generated); + }); + + it("REGRESSION: output is byte-for-byte deterministic across two runs on an unchanged tree", () => { + const root = fixtureRoot(); + const outputPath = "packages/gittensory-miner/docs/env-reference.md"; + writeMinerEnvReference({ rootDir: root, outputPath }); + const first = readFileSync(join(root, outputPath), "utf8"); + writeMinerEnvReference({ rootDir: root, outputPath }); + const second = readFileSync(join(root, outputPath), "utf8"); + expect(second).toBe(first); + }); +});