From fc165a16a577a1674b6f782118290284707b9577 Mon Sep 17 00:00:00 2001 From: davion-knight <298846663+davion-knight@users.noreply.github.com> Date: Sun, 12 Jul 2026 07:38:41 -0500 Subject: [PATCH] feat(miner-deployment): add docker-compose.miner.yml for AMS fleet mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a compose file so operators run the miner as a long-lived worker instead of a hand-assembled docker run (#5177): a `miner` service built from the package Dockerfile, `restart: unless-stopped`, SQLite state on a named `miner-data` volume, and credentials via an env file (.gittensory-miner.env) rather than inlined — so tokens stay out of the compose file and out of `docker inspect`. Documents `--scale miner=N` plus the honest caveat that a shared volume corrupts the SQLite ledgers, with the per-worker isolation patterns (separate compose projects or distinct config dirs; k8s StatefulSet for built-in isolation), in the compose comments and DEPLOYMENT.md. Ships a scanner-safe .env.example (empty values) and a validation test asserting the service contract + that no credential is hardcoded. Closes #5177 --- .gitignore | 3 + .../.gittensory-miner.env.example | 11 ++++ packages/gittensory-miner/DEPLOYMENT.md | 25 +++++--- .../gittensory-miner/docker-compose.miner.yml | 42 ++++++++++++++ test/unit/miner-docker-compose.test.ts | 57 +++++++++++++++++++ 5 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 packages/gittensory-miner/.gittensory-miner.env.example create mode 100644 packages/gittensory-miner/docker-compose.miner.yml create mode 100644 test/unit/miner-docker-compose.test.ts diff --git a/.gitignore b/.gitignore index a55b9c157..e10fde543 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ output/ .env .env.* !.env.example +# Operator-filled miner secrets for docker-compose fleet mode (#5177); the .example is committed. +.gittensory-miner.env +!.gittensory-miner.env.example !.env.selfhost.example # Docker Compose secret files: the actual secret VALUES an operator drops into # secrets/.txt for the native `secrets:` mount (docker-compose.yml), read via the existing generic diff --git a/packages/gittensory-miner/.gittensory-miner.env.example b/packages/gittensory-miner/.gittensory-miner.env.example new file mode 100644 index 000000000..60d2bd61b --- /dev/null +++ b/packages/gittensory-miner/.gittensory-miner.env.example @@ -0,0 +1,11 @@ +# Env file for docker-compose.miner.yml (#5177). Copy to `.gittensory-miner.env`, fill in real values, and do +# NOT commit the filled-in copy. Values are intentionally EMPTY here (a real-looking value in a committed file +# trips secret scanners). Compose reads `KEY=value` lines; leave a key blank or delete it if unused. +# +# Required: a GitHub token the miner uses to read issues and open PRs. +GITHUB_TOKEN= +# Optional: only for the coding-agent providers you enable (claude-cli / codex-cli / agent-sdk). +ANTHROPIC_API_KEY= +OPENAI_API_KEY= +# Note: the state dir is fixed at /data/miner (the Dockerfile default + the compose volume mount), so it is +# intentionally NOT an override here — see the comment in docker-compose.miner.yml. diff --git a/packages/gittensory-miner/DEPLOYMENT.md b/packages/gittensory-miner/DEPLOYMENT.md index d0a5f06f0..4e38dffa5 100644 --- a/packages/gittensory-miner/DEPLOYMENT.md +++ b/packages/gittensory-miner/DEPLOYMENT.md @@ -2,13 +2,13 @@ Two form factors for running `@jsonbored/gittensory-miner`: **laptop mode** (single machine, zero Docker) and **fleet mode** (containerized workers with a shared data volume). Both are 100% client-side for core operation — the miner never uploads source and never requires a hosted Gittensory callback to boot. Credentials (GitHub tokens, etc.) stay on the operator's machine or in their own secret store; nothing is baked into images. -| | Laptop mode | Fleet mode | -|---|---|---| -| **Best for** | One contributor machine, local experimentation | Many parallel miner attempts on a host or small cluster | -| **Dependencies** | Node.js `>=22.13.0` only | Docker (or compatible runtime) + Node image or custom image | -| **State** | SQLite files under `~/.config/gittensory-miner/` (override with `GITTENSORY_MINER_CONFIG_DIR`) | Same SQLite layout on a mounted `/data` (or `GITTENSORY_MINER_CONFIG_DIR`) volume | -| **Setup** | `npm install -g @jsonbored/gittensory-miner` or workspace build | `docker build` + `docker run` with env + volume (see below) | -| **Footprint** | One Node process, local disk for ledgers/queues | One container per worker; scale horizontally by adding containers | +| | Laptop mode | Fleet mode | +| ---------------- | ---------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | +| **Best for** | One contributor machine, local experimentation | Many parallel miner attempts on a host or small cluster | +| **Dependencies** | Node.js `>=22.13.0` only | Docker (or compatible runtime) + Node image or custom image | +| **State** | SQLite files under `~/.config/gittensory-miner/` (override with `GITTENSORY_MINER_CONFIG_DIR`) | Same SQLite layout on a mounted `/data` (or `GITTENSORY_MINER_CONFIG_DIR`) volume | +| **Setup** | `npm install -g @jsonbored/gittensory-miner` or workspace build | `docker build` + `docker run` with env + volume (see below) | +| **Footprint** | One Node process, local disk for ledgers/queues | One container per worker; scale horizontally by adding containers | ## Laptop mode walkthrough @@ -69,6 +69,17 @@ The image entrypoint is `gittensory-miner`; pass subcommands after the image nam The repo-root [`docker-compose.yml`](../../docker-compose.yml) documents the **self-hosted review stack** (the `gittensory` API/orb), not the miner CLI. Miners are clients of that stack (or of github.com directly) and do not require it to run locally. +### Docker Compose (fleet mode) + +Instead of a hand-assembled `docker run`, [`docker-compose.miner.yml`](docker-compose.miner.yml) defines a long-lived `miner` service (built from this package's Dockerfile, `restart: unless-stopped`, state on a named `miner-data` volume). Credentials come from an env file, never inlined: + +```sh +cp .gittensory-miner.env.example .gittensory-miner.env # fill in GITHUB_TOKEN (+ optional provider keys) +docker compose -f docker-compose.miner.yml up -d --build +``` + +**Scaling to N parallel workers.** `docker compose -f docker-compose.miner.yml up -d --scale miner=N` gives every replica the **same** `miner-data` volume — and the miner's SQLite ledgers are **not** safe for concurrent access, so N replicas on one volume will contend/corrupt. To run N **isolated** workers, give each its own state: run N separate compose projects (`docker compose -p miner-1 …`, `-p miner-2 …` — `-p` namespaces the volume) or point each at a distinct `GITTENSORY_MINER_CONFIG_DIR` on its own mount. For built-in isolated horizontal scaling, use the Kubernetes StatefulSet in [`k8s/`](../../k8s/) (per-pod volumes). + ## Invariants - Core miner bookkeeping (claims, plans, queues, ledgers) works offline after install. diff --git a/packages/gittensory-miner/docker-compose.miner.yml b/packages/gittensory-miner/docker-compose.miner.yml new file mode 100644 index 000000000..6b3c2f4be --- /dev/null +++ b/packages/gittensory-miner/docker-compose.miner.yml @@ -0,0 +1,42 @@ +# docker-compose for gittensory-miner (AMS) fleet mode (#5177). +# +# Runs the miner as a long-lived CLI worker instead of a hand-assembled `docker run`. Build context is the +# MONOREPO ROOT (../..) because the Dockerfile needs the full workspace on disk before `npm ci` — see the +# comments in packages/gittensory-miner/Dockerfile. +# +# Usage: +# 1. Create the env file (never commit real values): cp .gittensory-miner.env.example .gittensory-miner.env +# 2. Start one worker: docker compose -f docker-compose.miner.yml up -d --build +# +# Scaling to N parallel workers: +# docker compose -f docker-compose.miner.yml up -d --scale miner=N +# BUT: `--scale` gives every replica the SAME `miner-data` volume, and the miner's SQLite ledgers +# (claim-ledger.sqlite3, plan-store.sqlite3, …) are NOT safe for concurrent multi-process access — N replicas +# sharing one volume WILL corrupt/contend on those files. To run N ISOLATED workers, give each its own state: +# • run N separate compose projects, each with its own volume: +# docker compose -p miner-1 -f docker-compose.miner.yml up -d +# docker compose -p miner-2 -f docker-compose.miner.yml up -d # …etc; -p namespaces the volume +# • or point each worker at a distinct GITTENSORY_MINER_CONFIG_DIR on its own mount. +# (A single-shared-volume `--scale` is only safe for one active worker; Kubernetes StatefulSet per-pod PVCs, see +# k8s/, are the built-in answer for isolated horizontal scaling.) +services: + miner: + build: + context: ../.. + dockerfile: packages/gittensory-miner/Dockerfile + image: gittensory-miner:latest + # ENTRYPOINT is `gittensory-miner`; `run` is the continuous fleet-worker loop. + command: ["run"] + restart: unless-stopped + # Operator-supplied secrets/config (GITHUB_TOKEN, optional provider keys). Copy the .example, fill it in, + # and NEVER commit the real file — env vars stay out of the compose file and out of `docker inspect` args. + env_file: + - .gittensory-miner.env + # The state dir is fixed at /data/miner — it is the Dockerfile's built-in GITTENSORY_MINER_CONFIG_DIR default + # AND the volume mount below. It is deliberately NOT set here (and NOT offered as an env-file override): + # Compose's `environment` would silently override an env-file value, and the mount is pinned to /data/miner, + # so exposing it as a knob would be a silently-ignored footgun. To relocate state, change the volume mount. + volumes: + - miner-data:/data/miner +volumes: + miner-data: diff --git a/test/unit/miner-docker-compose.test.ts b/test/unit/miner-docker-compose.test.ts new file mode 100644 index 000000000..2724dafac --- /dev/null +++ b/test/unit/miner-docker-compose.test.ts @@ -0,0 +1,57 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +// Validation for the AMS fleet-mode compose file (#5177). Not `src/**` logic, so Codecov doesn't gate it — but +// the structural contract (service built from the package Dockerfile, named volume for SQLite persistence, +// restart policy, credentials via env-file not inlined) is asserted here as a real test, and the env example is +// checked to ship only empty (secret-scanner-safe) placeholders. +const MINER_DIR = join(process.cwd(), "packages/gittensory-miner"); +const compose = parse( + readFileSync(join(MINER_DIR, "docker-compose.miner.yml"), "utf8"), +) as Record; +const envExample = readFileSync( + join(MINER_DIR, ".gittensory-miner.env.example"), + "utf8", +); + +describe("docker-compose.miner.yml (#5177)", () => { + it("defines a miner service built from the package Dockerfile (monorepo-root context)", () => { + const miner = compose.services?.miner; + expect(miner).toBeTruthy(); + expect(miner.build.dockerfile).toBe("packages/gittensory-miner/Dockerfile"); + expect(miner.build.context).toBe("../.."); + expect(miner.command).toContain("run"); + }); + + it("persists state on a named volume and restarts unless stopped", () => { + const miner = compose.services.miner; + expect(miner.restart).toBe("unless-stopped"); + expect(miner.volumes).toContain("miner-data:/data/miner"); + expect(compose.volumes).toHaveProperty("miner-data"); + }); + + it("sources credentials from an env file and pins nothing overridable in `environment`", () => { + const miner = compose.services.miner; + expect(miner.env_file).toContain(".gittensory-miner.env"); + const envBlock = JSON.stringify(miner.environment ?? {}); + // no secret-named key is assigned a value directly in the compose `environment` block + expect(envBlock).not.toMatch(/TOKEN|API_KEY|SECRET|PASSWORD/i); + // regression: GITTENSORY_MINER_CONFIG_DIR must NOT be pinned in `environment` — Compose's `environment` + // silently overrides `env_file`, so pinning it here would make any env-file override a dead footgun. + expect(envBlock).not.toMatch(/GITTENSORY_MINER_CONFIG_DIR/); + }); + + it("gitignores the operator's real env file while keeping the committed example", () => { + const gitignore = readFileSync(join(process.cwd(), ".gitignore"), "utf8"); + expect(gitignore).toMatch(/^\.gittensory-miner\.env$/m); + expect(gitignore).toMatch(/^!\.gittensory-miner\.env\.example$/m); + }); + + it("ships an env example with empty (scanner-safe) placeholder values for every credential", () => { + for (const key of ["GITHUB_TOKEN", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"]) { + expect(envExample).toMatch(new RegExp(`^${key}=\\s*$`, "m")); + } + }); +});