diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 000000000..fb5d8e460 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,62 @@ +# Kubernetes manifests — gittensory-miner (AMS) fleet mode + +Example manifests for running **N isolated miner workers** on a small Kubernetes cluster, as an alternative to +`docker run` / docker-compose on a single host. Built on the existing +[`packages/gittensory-miner/Dockerfile`](../packages/gittensory-miner/Dockerfile) image (see +[`DEPLOYMENT.md`](../packages/gittensory-miner/DEPLOYMENT.md) for the fleet-mode overview). These are starting +points — review resource sizing, storage class, and your registry before applying to a real cluster. + +## Why a StatefulSet (not a Deployment) + +The miner keeps all state in **local SQLite ledgers** (`claim-ledger.sqlite3`, `plan-store.sqlite3`, …) under +`GITTENSORY_MINER_CONFIG_DIR` (`/data/miner`). Those stores are **not safe for concurrent multi-pod access**, so +each worker needs its **own** volume. A Deployment can only mount a single shared PVC across every replica; a +**StatefulSet's `volumeClaimTemplates`** give each replica its own PersistentVolumeClaim — so scaling to N +replicas yields N workers with fully isolated state. That per-pod isolation is the whole reason these manifests +use a StatefulSet. + +## Deploy + +1. **Build and push the image** from the monorepo root, then set `image:` in `miner-deployment.yaml`: + ```sh + docker build -f packages/gittensory-miner/Dockerfile -t /gittensory-miner:latest . + docker push /gittensory-miner:latest + ``` +2. **Create the Secret** (fill in real values first — never commit the filled-in copy): + ```sh + cp k8s/miner-secret.example.yaml k8s/miner-secret.yaml # edit in your real GITHUB_TOKEN + provider keys + kubectl apply -f k8s/miner-secret.yaml + ``` +3. **Deploy the workers:** + ```sh + kubectl apply -f k8s/miner-deployment.yaml + ``` + +## Scale + +Each replica is one isolated worker with its own volume. Scale the fleet with: + +```sh +kubectl scale statefulset/gittensory-miner --replicas= +``` + +or by editing `replicas:` in `miner-deployment.yaml` and re-applying. New replicas each get a fresh +per-pod PVC from the `volumeClaimTemplate`; scaling down retains the PVCs (Kubernetes does not delete them +automatically), so a scaled-back worker resumes its own state when scaled up again. + +## Notes + +- **Secrets** are injected at runtime via the Secret — the image contains no credentials. `ANTHROPIC_API_KEY` / + `OPENAI_API_KEY` are marked `optional`, so a worker running only the providers you configure starts cleanly + without the others. +- **Resources** default to a modest CLI-worker baseline (`250m`/`512Mi` request, `1`/`1Gi` limit) with headroom + for a coding-agent subprocess. Tune for your providers and cluster. +- **Storage** uses the cluster's default StorageClass at `2Gi` per pod; uncomment `storageClassName` in the + `volumeClaimTemplate` if you need a specific class. +- **Image tag** — the example uses `:latest` (which defaults to `imagePullPolicy: Always`, re-pulling on every + restart). For production, push and pin an immutable tag (e.g. a version or digest). +- **Probes** — no `livenessProbe`/`readinessProbe` is defined: the worker is a CLI loop, not a served endpoint, + so there's no health port to probe. Add a process-based `livenessProbe` (e.g. an `exec` check) if your + platform expects one. +- **Filesystem ownership** — `fsGroup`/`runAsGroup` are set so the non-root user owns its PVC and can write the + SQLite files; without them the worker cannot create state on a root-owned `ReadWriteOnce` volume. diff --git a/k8s/miner-deployment.yaml b/k8s/miner-deployment.yaml new file mode 100644 index 000000000..e1b33b607 --- /dev/null +++ b/k8s/miner-deployment.yaml @@ -0,0 +1,88 @@ +# Kubernetes workload for gittensory-miner (AMS) fleet mode (#5181). +# +# This is a StatefulSet, NOT a Deployment: the miner keeps all of its state in local SQLite ledgers +# (claim-ledger.sqlite3, plan-store.sqlite3, …) under GITTENSORY_MINER_CONFIG_DIR, and those stores are NOT +# safe for concurrent multi-pod access. A Deployment can only mount one shared PVC across every replica; a +# StatefulSet's volumeClaimTemplates give each replica its OWN PersistentVolumeClaim, so N workers run with +# fully isolated state — the safety property this manifest exists to guarantee. See k8s/README.md. +# +# Image: build from packages/gittensory-miner/Dockerfile at the monorepo root and push to your registry: +# docker build -f packages/gittensory-miner/Dockerfile -t /gittensory-miner:latest . +# Then set `image:` below. Secrets come from k8s/miner-secret.example.yaml (apply that first). +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gittensory-miner + labels: + app.kubernetes.io/name: gittensory-miner + app.kubernetes.io/component: fleet-worker +spec: + serviceName: gittensory-miner + # Scale the fleet by editing `replicas` (each replica is one isolated worker) or: + # kubectl scale statefulset/gittensory-miner --replicas= + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: gittensory-miner + template: + metadata: + labels: + app.kubernetes.io/name: gittensory-miner + app.kubernetes.io/component: fleet-worker + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + # runAsGroup + fsGroup so the non-root user owns the mounted PVC and can create its SQLite files on a + # root-owned ReadWriteOnce volume; OnRootMismatch avoids re-chowning a large volume on every restart. + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: miner + # Replace with the image you built + pushed from packages/gittensory-miner/Dockerfile. + image: gittensory-miner:latest + # ENTRYPOINT is `gittensory-miner`; `run` is the continuous fleet-worker loop. + args: ["run"] + env: + - name: GITTENSORY_MINER_CONFIG_DIR + value: /data/miner + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: gittensory-miner-secrets + key: GITHUB_TOKEN + # Coding-agent provider credentials — optional; present only for the providers you enable. + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: + name: gittensory-miner-secrets + key: ANTHROPIC_API_KEY + optional: true + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: gittensory-miner-secrets + key: OPENAI_API_KEY + optional: true + volumeMounts: + - name: miner-data + mountPath: /data/miner + # A CLI worker, not a server: modest baseline with headroom for a coding-agent subprocess. + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + # Per-pod PersistentVolumeClaim — each replica gets its OWN volume, never one shared across replicas. + volumeClaimTemplates: + - metadata: + name: miner-data + spec: + accessModes: ["ReadWriteOnce"] + # storageClassName: fast-ssd # set this if your cluster's default StorageClass isn't what you want + resources: + requests: + storage: 2Gi diff --git a/k8s/miner-secret.example.yaml b/k8s/miner-secret.example.yaml new file mode 100644 index 000000000..2971cc6ca --- /dev/null +++ b/k8s/miner-secret.example.yaml @@ -0,0 +1,27 @@ +# Secret template for gittensory-miner fleet workers (#5181). +# +# Values are intentionally EMPTY — this is a committed example, and a real-looking placeholder string in a +# secret-named field trips secret scanners. Fill them in out-of-band; do NOT commit the filled-in copy. +# +# Recommended (keeps secret values out of any file entirely): +# kubectl create secret generic gittensory-miner-secrets \ +# --from-literal=GITHUB_TOKEN= \ +# --from-literal=ANTHROPIC_API_KEY= --from-literal=OPENAI_API_KEY= +# +# Or copy + edit this file, then apply it (stringData takes raw values; Kubernetes base64-encodes them): +# cp k8s/miner-secret.example.yaml k8s/miner-secret.yaml # then fill in the empty values +# kubectl apply -f k8s/miner-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: gittensory-miner-secrets + labels: + app.kubernetes.io/name: gittensory-miner +type: Opaque +stringData: + # Required: a GitHub token the miner uses to read issues and open PRs. + GITHUB_TOKEN: "" + # Optional: only for the coding-agent providers you enable (claude-cli / codex-cli / agent-sdk). + # Delete a line entirely for any provider you do not use. + ANTHROPIC_API_KEY: "" + OPENAI_API_KEY: "" diff --git a/test/unit/miner-k8s-manifests.test.ts b/test/unit/miner-k8s-manifests.test.ts new file mode 100644 index 000000000..301170d6b --- /dev/null +++ b/test/unit/miner-k8s-manifests.test.ts @@ -0,0 +1,116 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +// Validation for the example K8s manifests (#5181). These are static infra artifacts (not src/** logic), so +// Codecov's patch gate doesn't apply to the YAML — but the safety property the manifests exist to guarantee +// (per-pod isolated SQLite storage, never a shared PVC across replicas) is asserted here as a real test, and the +// structural validator is exercised against both a well-formed manifest (passes) and a malformed one (fails). + +const K8S_DIR = join(process.cwd(), "k8s"); +const readManifest = (name: string): Record => + parse(readFileSync(join(K8S_DIR, name), "utf8")) as Record; + +/** Minimal well-formedness check every Kubernetes resource must satisfy; throws on the first violation. */ +function validateK8sResource(doc: unknown): { kind: string } { + if (typeof doc !== "object" || doc === null) + throw new Error("manifest is not a mapping"); + const m = doc as Record; + for (const field of ["apiVersion", "kind", "metadata"] as const) { + if (m[field] == null) + throw new Error(`manifest missing required field: ${field}`); + } + const metadata = m.metadata as Record; + if (typeof metadata.name !== "string" || metadata.name.length === 0) { + throw new Error("manifest missing metadata.name"); + } + return { kind: String(m.kind) }; +} + +/** The safety property this issue introduces: each replica gets its OWN volume, never one shared claim. */ +function assertPerPodStorage(statefulSet: Record): void { + const spec = statefulSet.spec as Record | undefined; + const vcts = spec?.volumeClaimTemplates; + if (!Array.isArray(vcts) || vcts.length === 0) { + throw new Error( + "no volumeClaimTemplates: replicas would not get per-pod storage", + ); + } + const template = spec?.template as Record | undefined; + const podSpec = template?.spec as Record | undefined; + const podVolumes = + (podSpec?.volumes as Array> | undefined) ?? []; + for (const volume of podVolumes) { + if (volume.persistentVolumeClaim) + throw new Error("a shared PVC across replicas is not allowed"); + } +} + +describe("k8s miner manifests (#5181)", () => { + const deployment = readManifest("miner-deployment.yaml"); + + it("miner-deployment.yaml is a well-formed StatefulSet", () => { + expect(validateK8sResource(deployment).kind).toBe("StatefulSet"); + }); + + it("gives each replica its own SQLite volume (per-pod PVC, never a shared claim)", () => { + expect(() => assertPerPodStorage(deployment)).not.toThrow(); + const spec = deployment.spec as Record; + expect((spec.volumeClaimTemplates as unknown[]).length).toBeGreaterThan(0); + expect(typeof spec.replicas).toBe("number"); + }); + + it("runs the continuous worker with config dir, a secret-sourced token, and resource bounds", () => { + const containers = (deployment.spec as any).template.spec + .containers as Array>; + const container = containers[0]; + if (!container) + throw new Error("no container in the StatefulSet pod template"); + expect(container.args).toContain("run"); + const env = container.env as Array>; + const configDir = env.find((e) => e.name === "GITTENSORY_MINER_CONFIG_DIR"); + expect(configDir?.value).toBe("/data/miner"); + const token = env.find((e) => e.name === "GITHUB_TOKEN"); + expect(token?.valueFrom?.secretKeyRef?.key).toBe("GITHUB_TOKEN"); + expect(container.resources.requests).toBeTruthy(); + expect(container.resources.limits).toBeTruthy(); + }); + + it("secret template is a well-formed Secret exposing GITHUB_TOKEN", () => { + const secret = readManifest("miner-secret.example.yaml"); + expect(validateK8sResource(secret).kind).toBe("Secret"); + expect( + (secret.stringData as Record).GITHUB_TOKEN, + ).toBeDefined(); + }); + + it("the structural validator rejects a malformed manifest", () => { + const malformed = parse("apiVersion: apps/v1\nmetadata:\n name: broken\n"); // no `kind` + expect(() => validateK8sResource(malformed)).toThrow( + /missing required field: kind/, + ); + }); + + it("the per-pod-storage check rejects a shared-PVC configuration", () => { + const shared = { + spec: { + volumeClaimTemplates: [{ metadata: { name: "data" } }], + template: { + spec: { + volumes: [ + { name: "data", persistentVolumeClaim: { claimName: "shared" } }, + ], + }, + }, + }, + }; + expect(() => assertPerPodStorage(shared)).toThrow(/shared PVC/); + }); + + it("the per-pod-storage check rejects a config with no volumeClaimTemplates", () => { + expect(() => + assertPerPodStorage({ spec: { template: { spec: {} } } }), + ).toThrow(/no volumeClaimTemplates/); + }); +});