diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3346c1e..7dc87c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -557,6 +557,14 @@ jobs: needs: [vet] steps: - uses: actions/checkout@v4 + # Validate the release config (dockers, docker_signs, manifests, + # signs, nfpms, homebrew_casks) parses before a tag ever relies on + # it — a malformed .goreleaser.yaml otherwise aborts the release. + - name: goreleaser check + uses: goreleaser/goreleaser-action@v6 + with: + version: latest + args: check - name: shellcheck packaging scripts run: | sudo apt-get update -qq diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c79581..08da521 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,12 @@ jobs: uses: sigstore/cosign-installer@v3 - name: install syft (SBOM) uses: anchore/sbom-action/download-syft@v0 + - name: set up QEMU (multi-arch image builds) + if: vars.PUBLISH_CONTAINERS == 'true' + uses: docker/setup-qemu-action@v3 + - name: set up Docker Buildx + if: vars.PUBLISH_CONTAINERS == 'true' + uses: docker/setup-buildx-action@v3 - name: docker login (GHCR) uses: docker/login-action@v3 with: @@ -51,16 +57,15 @@ jobs: uses: goreleaser/goreleaser-action@v6 with: version: latest - # --skip=docker: GHCR package-write is not yet enabled on the - # org, so the image+manifest push fails with - # `denied: permission_denied: write_package` — and that - # failure aborts goreleaser's whole publish phase, taking the - # binaries/.deb/.rpm/GitHub Release down with it. Skipping - # docker lets the rest of the release ship; drop this flag - # once the org grants Actions package-write and re-tag to - # publish the image. (--skip=docker covers docker_manifests - # too; there is no separate manifest skip key.) - args: release --clean --skip=docker + # Container images publish to GHCR only when the repo/org + # variable PUBLISH_CONTAINERS == 'true'. Until the org enables + # Actions package-write on ghcr.io (otherwise the push fails + # `denied: permission_denied: write_package` and aborts the + # whole release), --skip=docker is passed so the binaries / + # .deb / .rpm / Homebrew / GitHub Release still ship. Flip the + # variable to 'true' once GHCR is enabled, then re-tag to + # publish images. (--skip=docker covers docker_manifests too.) + args: release --clean ${{ vars.PUBLISH_CONTAINERS == 'true' && '' || '--skip=docker' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Separate fine-grained PAT (contents:write on the homebrew-tap diff --git a/.goreleaser.yaml b/.goreleaser.yaml index feba5e5..974058c 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -285,6 +285,19 @@ docker_manifests: - "ghcr.io/cybertec-postgresql/pg_hardstorage:latest-amd64" - "ghcr.io/cybertec-postgresql/pg_hardstorage:latest-arm64" +# Keyless cosign signing of the published images + manifests (Sigstore + +# GitHub Actions OIDC), mirroring the blob `signs:` stanza. Only runs in +# the docker phase, i.e. when PUBLISH_CONTAINERS enables image publishing. +docker_signs: + - id: cosign-images + cmd: cosign + artifacts: all + args: + - "sign" + - "--yes" + - "${artifact}@${digest}" + output: true + release: github: owner: cybertec-postgresql diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a7b995..5dc9a2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,35 @@ on-disk and on-the-wire schema (backup manifests, configuration, output JSON, and the on-disk chunk envelope): an agent built against a given schema version keeps reading that version for at least 24 months after a successor lands. +## [1.0.3] — 2026-06-24 + +### Documentation: correctness sweep + cloud-support accuracy + +Audited the documentation against the codebase and corrected false or +stale claims. The big one: pg_hardstorage backs up self-managed +PostgreSQL over the physical replication protocol (`BASE_BACKUP` + a +physical slot); fully-managed DBaaS — Amazon RDS, Aurora, Cloud SQL, +Azure Database, Neon, Supabase — do **not** expose `BASE_BACKUP` and are +out of scope. Every "works on managed PG" claim was removed (web-verified +against each vendor's replication docs). Also fixed: feature counts (six +storage backends, one LLM provider), PG-version support (15–18; 15/16/17 +CI-required, 18 allow-failure), nonexistent CLI flags in tutorials / ops +guides, broken in-repo file paths, stale version strings, and the +AES-256-GCM-SIV-vs-GCM and cosign-vs-Ed25519 descriptions. CNPG-I, Rekor +anchoring, skill signing, and the FIPS image are now clearly marked +roadmap. Download / verify examples use a `VERSION` variable so they no +longer go stale. + +### Packaging: wire container-image publishing (GHCR) + +The release pipeline can now build and publish multi-arch (amd64/arm64) +distroless images to GHCR with keyless cosign image signatures. Publishing +is gated on the `PUBLISH_CONTAINERS` repo variable — set it once the org +enables Actions package-write on `ghcr.io`; until then the release ships +binaries / `.deb` / `.rpm` / Homebrew as before. Image-level SLSA +provenance remains roadmap. A `goreleaser check` step now validates the +release config in CI. + ## [Unreleased] ### Packaging: remove the obsolete homebrew-formula.json manifest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9fd6410..606beac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ contributions, but the bar is "this would land in the project anyway" ## Before you start -1. **Read `docs/SPEC.md`.** It's the single source of truth for what +1. **Read `SPEC.md`** (at the repo root). It's the single source of truth for what the system is and isn't supposed to do. Most of what looks like a missing feature is on a later milestone (v0.5 or v1.0); arguing for an earlier landing is a great use of an issue. diff --git a/SECURITY.md b/SECURITY.md index 7b57bd0..bf05915 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -48,4 +48,4 @@ The systemd units we ship apply the standard hardening set (`NoNewPrivileges=yes`, `ProtectSystem=strict`, `PrivateTmp=yes`, `SystemCallFilter=@system-service`, etc.). If you're running outside systemd, the equivalent posture is documented in -[docs/operator-guide.md](docs/operator-guide.md). +[docs/operations/operator-guide.md](docs/operations/operator-guide.md). diff --git a/docs/compliance/fedramp.md b/docs/compliance/fedramp.md index 9bc6270..a8728a4 100644 --- a/docs/compliance/fedramp.md +++ b/docs/compliance/fedramp.md @@ -63,7 +63,7 @@ JSON is `fedramp`. | CP-9 | System backup | Full backup pipeline | `pg_hardstorage backup ...` | `backup.create` | | CP-9(1) | Testing for reliability and integrity | Verifier subsystem + post-restore `pg_verifybackup` | `pg_hardstorage verify ...` | `verify.run` | | CP-9(3) | Separate storage for critical information | Cross-region replication | `pg_hardstorage repo replicate ...` | `replicate.completed` | -| CP-9(8) | Cryptographic protection | AES-256-GCM-SIV per chunk; Ed25519 manifest signatures | (automatic) | `backup.create` | +| CP-9(8) | Cryptographic protection | AES-256-GCM per chunk; Ed25519 manifest signatures | (automatic) | `backup.create` | | CP-10 | System recovery and reconstitution | Restore command + sandbox `pg_verifybackup` gate | `pg_hardstorage restore ...` | `restore.completed` | --- @@ -85,9 +85,9 @@ JSON is `fedramp`. | SC-7 | Boundary protection | Air-gap mode rejects public endpoints; storage plugin per-region scoping | `pg_hardstorage residency set ...` | (config) | | SC-8 | Transmission confidentiality | TLS 1.2+ on all storage backends; mTLS control plane | (config) | — | | SC-12 | Cryptographic key establishment and management | KMS-backed RKEK; per-tenant KEK | `pg_hardstorage kms ...` | `kms.*` | -| SC-13 | Cryptographic protection | AES-256-GCM-SIV (default); AES-256-GCM (FIPS variant) | (automatic) | `backup.create` | +| SC-13 | Cryptographic protection | AES-256-GCM (shipping today); AES-256-GCM-SIV planned, also for FIPS variant when GCM-SIV lands | (automatic) | `backup.create` | | SC-28 | Protection of information at rest | Three-layer envelope encryption | (automatic) | `backup.create` | -| SC-28(1) | Cryptographic protection | AES-256-GCM-SIV per chunk; Ed25519 manifest sigs | (automatic) | `backup.create` | +| SC-28(1) | Cryptographic protection | AES-256-GCM per chunk; Ed25519 manifest sigs | (automatic) | `backup.create` | --- @@ -97,7 +97,7 @@ JSON is `fedramp`. | --- | --- | --- | --- | --- | | SI-3 | Malicious code protection | Tier-1 plugin model + cosign-signed releases | (build) | — | | SI-4 | System monitoring | Prometheus metrics; OpenTelemetry traces; insider-threat scanner | `pg_hardstorage insider scan` | `insider.scan` | -| SI-7 | Software, firmware, and information integrity | SLSA L3 build provenance + reproducible builds | (build-time) | (cosign attest) | +| SI-7 | Software, firmware, and information integrity | SLSA L3 build provenance + reproducible builds | (build-time) | (SLSA provenance via slsa-github-generator + cosign sign-blob) | | SI-7(1) | Integrity checks | Continuous-attestation `integrity` runs over the repo | `pg_hardstorage integrity run` | `integrity.run` | | SI-7(7) | Integration of detection and response | `doctor` emits structured remediation commands for operator action | `pg_hardstorage doctor` | `doctor.suggested_fix` | diff --git a/docs/compliance/gdpr-art-17-crypto-shred.md b/docs/compliance/gdpr-art-17-crypto-shred.md index 02a87a3..fc2af6a 100644 --- a/docs/compliance/gdpr-art-17-crypto-shred.md +++ b/docs/compliance/gdpr-art-17-crypto-shred.md @@ -36,9 +36,10 @@ Every encrypted chunk is sealed under a layered key system: wrapped under the RKEK. Stored in `manifest.json.encryption.wrapped_dek`. 3. **Per-chunk key** derived `Kc = HKDF-SHA256(BDEK, - info=chunk_hash)`. Cipher: AES-256-GCM-SIV by default - (RFC 8452, nonce-misuse resistant); AES-256-GCM with - random 96-bit nonce in FIPS mode. + info=chunk_hash)`. Cipher: AES-256-GCM with a random + 96-bit nonce is shipping today; AES-256-GCM-SIV (RFC 8452, + nonce-misuse resistant) is the planned default once a + validated implementation lands. A tenant's KEK wraps every BDEK for every backup of every deployment under that tenant. Destroy that one key and @@ -182,7 +183,7 @@ pg_hardstorage audit append kms.shred \ | Art. 17(1) — right to erasure | `kms shred` per-tenant KEK destruction | `kms.shred` | | Art. 17(2) — communication of erasure to recipients | Cross-region replicas hold the same wrapped DEKs; KEK destruction propagates implicitly. Replica audit chain records `kms.shred`. | `kms.shred` (replicated) | | Art. 30 — record of processing activities | Hash-chained audit log; `audit verify-chain` proves untampered. | `audit.*` | -| Art. 32(1)(a) — encryption of personal data | AES-256-GCM-SIV per chunk (FIPS-validated GCM in FIPS build). | `backup.create` (records `encryption.scheme`) | +| Art. 32(1)(a) — encryption of personal data | AES-256-GCM per chunk (AES-256-GCM-SIV planned; FIPS-validated GCM in FIPS build). | `backup.create` (records `encryption.scheme`) | | Art. 32(1)(b) — confidentiality, integrity | Ed25519-signed manifests, Merkle audit chain. | `backup.create`, `audit.*` | --- diff --git a/docs/compliance/hipaa.md b/docs/compliance/hipaa.md index 6741462..d40adf6 100644 --- a/docs/compliance/hipaa.md +++ b/docs/compliance/hipaa.md @@ -41,14 +41,14 @@ JSON is `hipaa`. | §164.312(a)(2)(i) | Unique user identification | RBAC actor identity recorded on every event | (automatic) | (every event records `actor`) | | §164.312(a)(2)(ii) | Emergency access procedure | JIT tokens + n-of-m approval | `pg_hardstorage jit issue ...`, `approval request ...` | `jit.issue`, `approval.request` | | §164.312(a)(2)(iii) | Automatic logoff | JIT tokens auto-expire (max 24h) | (automatic) | `jit.issue` | -| §164.312(a)(2)(iv) | Encryption and decryption (Addressable) | AES-256-GCM-SIV per chunk; FIPS variant uses AES-256-GCM | (automatic) | `backup.create` records `encryption.scheme` | +| §164.312(a)(2)(iv) | Encryption and decryption (Addressable) | AES-256-GCM per chunk (AES-256-GCM-SIV planned) | (automatic) | `backup.create` records `encryption.scheme` | | §164.312(b) | Audit controls | Hash-chained Merkle audit log | `pg_hardstorage audit verify-chain` | (every event) | | §164.312(c)(1) | Integrity | Ed25519-signed manifests + chunk SHA round-trip | `pg_hardstorage verify ...` | `verify.run` | | §164.312(c)(2) | Mechanism to authenticate ePHI | Manifest signatures + audit chain hashing | `pg_hardstorage audit verify-chain` | `verify.manifest_signature` (on fail) | | §164.312(d) | Person or entity authentication | mTLS for control plane ↔ agent; Ed25519 keypair per operator | (config) | (auth failures captured) | | §164.312(e)(1) | Transmission security | TLS 1.2+ for storage backends; air-gap mode for offline networks | (config) | — | | §164.312(e)(2)(i) | Integrity controls | End-to-end checksums; chunk SHA round-trip | (automatic) | `verify.scrub_mismatch` (on fail) | -| §164.312(e)(2)(ii) | Encryption (Addressable, transmission) | TLS 1.2+ on every storage backend; AES-256-GCM-SIV on the chunk itself | (config) | — | +| §164.312(e)(2)(ii) | Encryption (Addressable, transmission) | TLS 1.2+ on every storage backend; AES-256-GCM on the chunk itself | (config) | — | --- @@ -60,7 +60,7 @@ when: | Layer | Requirement | `pg_hardstorage` posture | | --- | --- | --- | -| At rest | NIST-validated cipher | AES-256-GCM-SIV (RFC 8452) by default; AES-256-GCM (FIPS 140-validated via BoringCrypto) in `pg-hardstorage-fips` build | +| At rest | NIST-validated cipher | AES-256-GCM (random 96-bit nonce) shipping today; AES-256-GCM-SIV (RFC 8452) is the planned default once a validated implementation lands. The `pg-hardstorage-fips` build uses AES-256-GCM (FIPS 140-validated via BoringCrypto) — BoringCrypto does not ship GCM-SIV, so the FIPS variant will likewise use GCM when GCM-SIV lands | | In transit (storage backend) | TLS 1.2+ | All cloud backends; on-prem via plugin config | | In transit (agent ↔ control plane) | TLS / mTLS | mTLS by default | | Key management | NIST SP 800-57 lifecycle | KMS-backed RKEK; per-tenant KEK; KEK rotation with documented audit trail | diff --git a/docs/compliance/iso-27001-control-mapping.md b/docs/compliance/iso-27001-control-mapping.md index a2d4ddc..7303f72 100644 --- a/docs/compliance/iso-27001-control-mapping.md +++ b/docs/compliance/iso-27001-control-mapping.md @@ -59,7 +59,7 @@ constrains the geography. | A.8.16 | Monitoring activities | Prometheus metrics; OpenTelemetry traces; insider-threat scan | `pg_hardstorage insider scan` | `insider.scan` | | A.8.17 | Clock synchronisation | UTC timestamps everywhere; NTP recommended in operator guide | (operator) | (timestamp on every event) | | A.8.21 | Security of network services | TLS 1.2+ for storage backends; mTLS for control plane ↔ agent | (config) | (network failures captured) | -| A.8.24 | Use of cryptography | AES-256-GCM-SIV + Ed25519 + SHA-256 + HKDF-SHA256 | (automatic) | `backup.create` records `encryption.scheme` | +| A.8.24 | Use of cryptography | AES-256-GCM + Ed25519 + SHA-256 + HKDF-SHA256 (AES-256-GCM-SIV planned) | (automatic) | `backup.create` records `encryption.scheme` | | A.8.25 | Secure development life cycle | SLSA L3 build provenance; reproducible builds verified weekly | (build-time) | (cosign attest) | | A.8.26 | Application security requirements | Static analysis + race detector + sanitizers in CI | (build-time) | — | | A.8.27 | Secure system architecture and engineering principles | Documented threat model + design north stars in `SPEC.md` | — | — | @@ -107,7 +107,7 @@ information-asset register entry should record: | Asset | `pg_hardstorage` backup repository at `s3://...` | | Owner | (operator org) | | Classification | Per-deployment via `classify` | -| Encryption | AES-256-GCM-SIV per chunk (FIPS variant: AES-256-GCM) | +| Encryption | AES-256-GCM per chunk (AES-256-GCM-SIV planned) | | Retention | Per `retention` policy in deployment config | | Audit | Hash-chained Merkle log; quarterly bundle export | | Region | Per `residency` policy | diff --git a/docs/compliance/pci-dss.md b/docs/compliance/pci-dss.md index 829d474..5c146d6 100644 --- a/docs/compliance/pci-dss.md +++ b/docs/compliance/pci-dss.md @@ -30,8 +30,8 @@ report. | Requirement | Description | Product feature | Command | Audit event | | --- | --- | --- | --- | --- | -| 3.5.1 | PAN rendered unreadable | AES-256-GCM-SIV on every chunk; per-chunk key derivation | (automatic) | `backup.create` records `encryption.scheme` | -| 3.5.1.1 | Strong cryptography | AES-256-GCM-SIV (RFC 8452) by default; AES-256-GCM (FIPS) in FIPS variant | (automatic) | `backup.create` | +| 3.5.1 | PAN rendered unreadable | AES-256-GCM on every chunk; per-chunk key derivation | (automatic) | `backup.create` records `encryption.scheme` | +| 3.5.1.1 | Strong cryptography | AES-256-GCM (random 96-bit nonce) shipping today; AES-256-GCM-SIV (RFC 8452) planned | (automatic) | `backup.create` | | 3.5.1.2 | Disk-level encryption alone is insufficient | Per-chunk encryption is in addition to any underlying disk encryption | (automatic) | `backup.create` | | 3.6.1.1 | Cryptographic key custody | KMS-backed RKEK; per-tenant KEK; on-disk keyring with mode 0600 | `pg_hardstorage kms inspect` | `kms.*` | | 3.6.1.2 | Cryptographic keys protected | Keys never appear in logs or output; `kms inspect` is read-only and shows only fingerprints | `pg_hardstorage kms inspect` | (read-only) | @@ -143,13 +143,15 @@ audit period plus the cosign signatures on the running binary: ```sh +VERSION=1.0.3 # the release / image tag you're attesting + # 1. Audit chain bundle pg_hardstorage audit export-bundle \ --repo s3://acme-pci-backups/ \ --since 2026-01-01T00:00:00Z \ --until 2026-04-01T00:00:00Z \ --include-anchors \ - -o ./qsa-q1-2026-audit.tar.gz + --out ./qsa-q1-2026-audit.tar.gz # 2. Compliance report (Markdown for the QSA) pg_hardstorage compliance report \ @@ -161,7 +163,7 @@ pg_hardstorage compliance report \ # 3. Build provenance attestation cosign verify-attestation \ --type slsaprovenance \ - ghcr.io/cybertec-postgresql/pg_hardstorage:v0.9.2 \ + "ghcr.io/cybertec-postgresql/pg_hardstorage:v${VERSION}" \ --certificate-identity-regexp \ "https://github.com/cybertec-postgresql/pg_hardstorage/.*" \ --certificate-oidc-issuer \ diff --git a/docs/compliance/slsa-l3-provenance.md b/docs/compliance/slsa-l3-provenance.md index e9cec6b..40b0294 100644 --- a/docs/compliance/slsa-l3-provenance.md +++ b/docs/compliance/slsa-l3-provenance.md @@ -20,8 +20,8 @@ SLSA L3 stack assembled from the standard tooling: | Source — verified history | Branch protection + signed commits | | Build — scripted | `Makefile` + GoReleaser config | | Build — build service | GitHub Actions, hosted runners | -| Build — non-falsifiable provenance | `cosign attest --predicate` SLSA v1.0 predicate | -| Build — provenance distributed | Attestation attached to each artifact in `ghcr.io` + GitHub Releases | +| Build — non-falsifiable provenance | `slsa-github-generator` SLSA v1.0 `.intoto.jsonl` attestation | +| Build — provenance distributed | `.intoto.jsonl` attestation attached to the GitHub Release alongside the artifacts | | Build — isolated | Hosted runner; no shared state | | Build — parameterless | Build inputs are the tag + commit; no operator-supplied params | | Build — hermetic | `go mod download` is the only network step; build runs offline thereafter | @@ -54,16 +54,65 @@ A successful verification confirms: - The Sigstore transparency log (Rekor) recorded the signing event. +The `cosign sign-blob` signing of every tarball, `.deb`, +`.rpm`, and the `checksums.txt` file is configured in +`.goreleaser.yaml` (the `cosign-keyless` sign stanza). + +--- + +## Verifying build provenance + +Separately from the blob signatures, each release carries a +non-falsifiable SLSA L3 build provenance attestation. It is +emitted by the `slsa-framework/slsa-github-generator` reusable +workflow (not by cosign) and uploaded to the GitHub Release as +a `.intoto.jsonl` file. The generator runs on an isolated, +locked-down runner that cannot see the goreleaser-built +artifacts directly — that isolation is what earns SLSA L3. + +Verify a downloaded artifact against its provenance with the +`slsa-verifier`: + +```sh +VERSION=1.0.3 # the release you're verifying +slsa-verifier verify-artifact \ + --provenance-path pg_hardstorage.intoto.jsonl \ + --source-uri github.com/cybertec-postgresql/pg_hardstorage \ + --source-tag "v${VERSION}" \ + "pg_hardstorage_${VERSION}_linux_amd64.tar.gz" +``` + +The provenance predicate documents: + +- The source repo + commit SHA the build started from. +- The build entry point (which workflow file). +- The build platform (`github_actions`). +- The materials (Go toolchain version). +- The build invocation parameters (the tag). + --- ## Container images -Container images carry the same cosign signature plus a -SLSA provenance attestation: +> **Gated on GHCR enablement.** The release pipeline is wired to build +> and publish multi-arch (amd64/arm64) images to GHCR with **keyless +> cosign image signatures** (the `docker_signs` stanza). Publishing is +> gated on the `PUBLISH_CONTAINERS` repo variable: until the org enables +> Actions package-write on `ghcr.io` and the variable is set to `true`, +> no images are published and the `cosign verify` example below has +> nothing to check yet. Once enabled, signed images ship on each release +> from v1.0.3 onward. Image-level **SLSA provenance attestation** (the +> `cosign verify-attestation` example below) is still roadmap — only the +> blob/tarball SLSA provenance documented above ships today. + +When images ship, they are intended to carry a cosign +signature plus an image-level SLSA provenance attestation: ```sh +VERSION=1.0.3 # the image tag you're verifying + # Verify the image signature -cosign verify ghcr.io/cybertec-postgresql/pg_hardstorage:v0.9.2 \ +cosign verify "ghcr.io/cybertec-postgresql/pg_hardstorage:v${VERSION}" \ --certificate-identity-regexp \ "https://github.com/cybertec-postgresql/pg_hardstorage/.*" \ --certificate-oidc-issuer \ @@ -72,40 +121,34 @@ cosign verify ghcr.io/cybertec-postgresql/pg_hardstorage:v0.9.2 \ # Inspect the SLSA provenance cosign verify-attestation \ --type slsaprovenance \ - ghcr.io/cybertec-postgresql/pg_hardstorage:v0.9.2 \ + "ghcr.io/cybertec-postgresql/pg_hardstorage:v${VERSION}" \ --certificate-identity-regexp \ "https://github.com/cybertec-postgresql/pg_hardstorage/.*" \ --certificate-oidc-issuer \ "https://token.actions.githubusercontent.com" ``` -The provenance predicate documents: - -- The source repo + commit SHA the build started from. -- The build entry point (which `Makefile` target, which - workflow file). -- The build platform (`github_actions`). -- The materials (Go toolchain version, base image digest). -- The build invocation parameters (the tag). - --- ## SBOMs Each release ships a Software Bill of Materials in SPDX -JSON format, generated by `syft`: +JSON format, generated by `syft` (configured via the +`sboms:` stanza in `.goreleaser.yaml`). One +`.spdx.sbom.json` is produced per release archive +and attached to the GitHub Release: ```sh -cosign download attestation \ - ghcr.io/cybertec-postgresql/pg_hardstorage:v0.9.2 \ - | jq -r '.payload | @base64d | fromjson | - select(.predicateType == "https://spdx.dev/Document") - | .predicate' +VERSION=1.0.3 # the release you're inspecting +# The SBOM is a plain file on the Release; inspect it directly +jq -r '.packages[].name' \ + "pg_hardstorage_${VERSION}_linux_amd64.tar.gz.spdx.sbom.json" ``` -The SBOM is attested separately so an auditor can verify -"this image's SBOM is what the build produced" without -trusting any third-party scanner. +Like every other release artifact, the SBOM file is itself +`cosign sign-blob`-signed (the `signs.artifacts: all` setting +covers it), so an auditor can verify the SBOM is the one the +build produced without trusting any third-party scanner. --- @@ -136,27 +179,26 @@ build config). --- -## FIPS variant +## FIPS variant (roadmap — not yet shipped) + +> **Not yet shipped.** A FIPS build target exists only as the +> local-only `make build-fips` Makefile target. The release +> pipeline builds no FIPS binary and publishes no +> `pg_hardstorage-fips` image, so there is no released FIPS +> artifact and no FIPS provenance attestation to verify today. -The FIPS build (`pg-hardstorage-fips`) uses +The intended FIPS build uses `GOEXPERIMENT=boringcrypto CGO_ENABLED=1` and links against -the BoringCrypto FIPS-validated module. The variant's -provenance attestation records: +the BoringCrypto FIPS-validated module. Once a FIPS artifact +is released, its provenance attestation would record: - `GOEXPERIMENT=boringcrypto` is set. - The validated module version. - The FIPS-strict configuration (`crypto/tls` reports FIPS on startup). -```sh -cosign verify-attestation \ - --type slsaprovenance \ - ghcr.io/cybertec-postgresql/pg_hardstorage-fips:v0.9.2 \ - ... -``` - -A FIPS deployment must verify against the FIPS image; -the standard image does NOT carry FIPS validation even +A FIPS deployment must verify against the FIPS artifact; +the standard build does NOT carry FIPS validation even though both come from the same source. --- diff --git a/docs/explanation/architecture-tour.md b/docs/explanation/architecture-tour.md index 653c749..6eec65a 100644 --- a/docs/explanation/architecture-tour.md +++ b/docs/explanation/architecture-tour.md @@ -45,10 +45,15 @@ One connection model, one credential, one mental model. The agent needs no OS-level access to the database host. It can be in a different VM, different region, different cloud, different K8s -cluster. It works on AWS RDS, GCP Cloud SQL, Azure DB, Aiven, -Supabase, Neon — anything that exposes a standard PG replication -endpoint. This is the largest single architectural difference from -pgBackRest, which assumes co-location and SSH. +cluster. It works against any self-managed PostgreSQL that exposes the +physical replication endpoint. This is the largest single +architectural difference from pgBackRest, which assumes co-location +and SSH. + +Fully-managed DBaaS — AWS RDS, Aurora, GCP Cloud SQL, Azure Database, +Neon, Supabase — do **not** expose `BASE_BACKUP` or physical +replication to customers, so they are out of scope: pg_hardstorage +cannot take a physical base backup of them. ### How diff --git a/docs/explanation/audit-chain.md b/docs/explanation/audit-chain.md index 41efbf0..30b54aa 100644 --- a/docs/explanation/audit-chain.md +++ b/docs/explanation/audit-chain.md @@ -278,7 +278,7 @@ correlating across separate logs. The exportable evidence bundle (`pg_hardstorage llm export-session`) includes a Merkle proof that the session's events anchor at specific positions in the chain, signed with -cosign. An auditor can verify the session is exactly what the +the agent's Ed25519 keyring. An auditor can verify the session is exactly what the chain says it was, with no trust in the binary's good-faith reporting. diff --git a/docs/explanation/comparison-pgbackrest-walg-barman.md b/docs/explanation/comparison-pgbackrest-walg-barman.md index f27c723..60ae16e 100644 --- a/docs/explanation/comparison-pgbackrest-walg-barman.md +++ b/docs/explanation/comparison-pgbackrest-walg-barman.md @@ -17,8 +17,11 @@ you can pick the right tool for your case. The short version of where `pg_hardstorage` differs: - **WAL via the replication protocol, not file-based archiving.** - Works on managed PG (RDS, Cloud SQL, Aiven, Neon). No SSH, no - archive_library install required. + No SSH, no `archive_command`, and no extension on the database + host — the agent backs up any self-managed PostgreSQL it can reach + over the physical replication endpoint, from a different VM, region, + or cluster. (Fully-managed DBaaS such as RDS or Cloud SQL don't + expose `BASE_BACKUP` and are out of scope.) - **Content-addressed dedup with no chain dependency.** Every backup is independently restorable; deleting one cannot break another. @@ -39,16 +42,17 @@ time to earn equivalent confidence. | | pg_hardstorage | pgBackRest | WAL-G | Barman | | --- | --- | --- | --- | --- | | **Primary WAL transport** | Replication protocol streaming | `archive_command` | `archive_command` | `archive_command` + streaming | -| **Works on managed PG (RDS / Cloud SQL)?** | Yes (no host access needed) | No (needs SSH or archive_library) | Yes (file-based) | Partial (streaming variant) | +| **Backs up self-managed PG without host/SSH access?** | Yes (replication connection only) | No (needs SSH or archive_library) | No (archive_command on host) | Partial (streaming variant) | +| **Backs up managed DBaaS (RDS / Cloud SQL)?** | No (`BASE_BACKUP` not exposed) | No | No | No | | **Backup chain model** | None (CAS, every backup independent) | Differential / incremental chained | Delta backups chained | Differential / incremental chained | | **Dedup** | Cross-backup, cross-deployment, cross-tenant-ish | Within incremental chain | Page-delta | Within chain | -| **Encryption default** | AES-256-GCM-SIV envelope, on by default | Optional, configurable | Optional | Optional | +| **Encryption default** | AES-256-GCM envelope, on by default | Optional, configurable | Optional | Optional | | **KMS support** | AWS / GCP / Azure / Vault / HSM | Per-deployment | AWS / GCP / Azure | Per-deployment | | **Audit log** | Hash-chained Merkle, transparency-anchored (v0.5+) | Standard log file | Standard log file | Standard log file | | **WORM** | First-class (S3 Object Lock, Azure immutable, NetApp SnapLock) | Backend-dependent | Backend-dependent | Backend-dependent | | **FIPS build** | `pg-hardstorage-fips` flavour | Build-time | Build-time | Build-time | | **Patroni integration** | REST-aware + permanent_slots + dual-slot + sync-target | Config integration | Patroni `bootstrap.method` | Standard PG replication | -| **K8s integration** | CNPG-I, WAL-G shim, pgBackRest shim, Helm charts | pgBackRest operators | WAL-G operators | Custom | +| **K8s integration** | WAL-G shim, pgBackRest shim, Helm charts (CNPG-I provider on the v0.5 roadmap) | pgBackRest operators | WAL-G operators | Custom | | **LLM helper** | First-class, audited, gated | n/a | n/a | n/a | | **License** | Apache 2.0 | MIT | Apache 2.0 | GPL-3 | @@ -75,10 +79,12 @@ pgBackRest is excellent and we're comfortable saying so: The places to think twice: -- **You can't SSH into the host.** Managed PG offerings - (RDS, Cloud SQL, Aiven) make pgBackRest's primary model - difficult or impossible. Workarounds exist but they're - workarounds. +- **You can't SSH into the host.** When the backup tool can't + get SSH/OS access to a self-managed primary (a locked-down VM, + a host another team owns, a container you don't control), + pgBackRest's co-location model is difficult; pg_hardstorage only + needs a replication connection. (Neither tool can back up a + managed DBaaS like RDS — `BASE_BACKUP` isn't exposed there.) - **You want backup chains that don't cascade.** pgBackRest's incremental backup chain creates dependencies; one corrupt @@ -155,9 +161,12 @@ The places to think twice: The cases where the architecture genuinely earns its differentiation: -- **Managed PostgreSQL** (RDS, Cloud SQL, Azure DB, Aiven, Neon). - No SSH, no archive_library install. Replication protocol - streaming works identically across all of them. +- **Self-managed PostgreSQL without host access.** Backup runs + over a replication connection, so the agent can live in a + different VM, region, or K8s cluster from the database — no SSH, + no `archive_command`, no extension on the host. (Managed DBaaS + like RDS or Cloud SQL are *not* supported: they don't expose + `BASE_BACKUP`.) - **Mixed fleet** of self-hosted + managed + K8s. Same binary, same config schema, same repo schema. The operator learns one @@ -178,9 +187,10 @@ differentiation: story (REST + permanent_slots + dual-slot + sync-target) rather than rolling your own. -- **K8s with the operator-shim model**. CNPG-I, WAL-G shim, and +- **K8s with the operator-shim model**. The WAL-G shim and pgBackRest shim let you swap into existing operator-managed - clusters without rewriting the operator. + clusters without rewriting the operator. (A native CNPG-I + provider is on the roadmap for v0.5.) - **Transparent Data Encryption (TDE) at the source.** PG forks that encrypt heap / index / WAL at rest — CYBERTEC PGEE, diff --git a/docs/explanation/comparison.md b/docs/explanation/comparison.md index 2ec71f4..015e18e 100644 --- a/docs/explanation/comparison.md +++ b/docs/explanation/comparison.md @@ -9,7 +9,7 @@ in learning a new workflow. Here is the reasoning. | Dimension | pgBackRest | pg_hardstorage | |---|---|---| -| **WAL transport** | archive_command or async archive-push over SSH | replication-protocol streaming (START_REPLICATION SLOT) — works on managed PG too | +| **WAL transport** | archive_command or async archive-push over SSH | replication-protocol streaming (START_REPLICATION SLOT) — no SSH or archive_command on the host | | **Agent locality** | co-located; needs SSH to primary | remote — libpq replication connection; agent can run anywhere | | **Dependency model** | C + Perl + libpq, requires OS access | single static Go binary, zero CGO (default), works without a compiler | | **Backup chain** | incremental depends on chain ancestors — a corrupt increment breaks the chain | chunk-CAS — every backup is independent; no chain to maintain | diff --git a/docs/explanation/content-addressed-storage.md b/docs/explanation/content-addressed-storage.md index dc14548..7c1bb5c 100644 --- a/docs/explanation/content-addressed-storage.md +++ b/docs/explanation/content-addressed-storage.md @@ -146,7 +146,7 @@ Self-describing. Readers at v0.1+ handle: - Envelope `0x02` — encryption-aware, the current default. Compression algos: `zstd`, `none`. Encryption algos: -`aes-256-gcm-siv` (default), `aes-256-gcm` (FIPS), `none`. +`aes-256-gcm` (shipping today), `aes-256-gcm-siv` (planned), `none`. The 24-month manifest schema commitment also covers the chunk envelope: a v0.1 reader will keep working with chunks written by diff --git a/docs/explanation/design-principles.md b/docs/explanation/design-principles.md index 860f9f2..ae80e5f 100644 --- a/docs/explanation/design-principles.md +++ b/docs/explanation/design-principles.md @@ -85,9 +85,9 @@ homelabs, to have them. Concrete consequences: -- The cipher default is AES-256-GCM-SIV (RFC 8452, nonce-misuse - resistant). AES-256-GCM with a random 96-bit nonce is the FIPS - fallback when `BoringCrypto` is in use. See [envelope +- AES-256-GCM (random 96-bit nonce) is the cipher shipping today; + AES-256-GCM-SIV (RFC 8452, nonce-misuse-resistant) is the planned + default once a validated implementation lands. See [envelope encryption](envelope-encryption.md). - The audit log is hash-chained on every write — no opt-in, no @@ -184,12 +184,18 @@ spawning a wrapper script. Not a polling loop on `pg_wal/`. Not SSH into the host to read files. The reasoning is a single sentence: **we want the agent to work -where SSH and `archive_library` cannot reach.** Managed PostgreSQL -on AWS RDS, GCP Cloud SQL, Azure Database, Aiven, Supabase, Neon — -none of them let you install a C extension, none of them let you -SSH into the host. All of them expose the standard PostgreSQL -replication endpoint. Streaming over the replication protocol -works on every one of them, identically to a self-hosted PG. +where SSH and `archive_library` cannot reach.** On self-managed +PostgreSQL you often can't install a C extension or SSH into the +host — a locked-down VM, a host another team owns, a container you +don't control. As long as the database exposes the standard +physical replication endpoint, streaming over the replication +protocol works identically to a co-located setup, from anywhere the +agent can reach it. + +Fully-managed DBaaS — AWS RDS, Aurora, GCP Cloud SQL, Azure Database, +Neon, Supabase — are a different case: they do **not** expose +`BASE_BACKUP` or physical replication to customers, so pg_hardstorage +cannot back them up. It targets PostgreSQL you run yourself. Bonus consequence: a persistent slot **closes the WAL gap window to zero by default**. PG holds WAL on disk until our agent ACKs. @@ -250,10 +256,12 @@ the higher-numbered one yields. Some real examples: complexity is hidden. - **Compliance vs WAL via replication protocol:** what about FIPS? - AES-256-GCM-SIV isn't FIPS. We ship a `pg-hardstorage-fips` - build with `GOEXPERIMENT=boringcrypto` that falls back to plain - AES-256-GCM with a random 96-bit nonce. Both principles are - served — at the cost of two build flavours. + GCM-SIV isn't FIPS, and the planned GCM-SIV default will not be + available in the FIPS build. We ship a `pg-hardstorage-fips` + build with `GOEXPERIMENT=boringcrypto`; it uses AES-256-GCM with + a random 96-bit nonce today, and will likewise use GCM when + GCM-SIV lands. Both principles are served — at the cost of two + build flavours. The principles are not aspirational. Every one of them is load-bearing in a specific subsystem; if you propose changing one, diff --git a/docs/explanation/envelope-encryption.md b/docs/explanation/envelope-encryption.md index 8c7ae5e..ea45df9 100644 --- a/docs/explanation/envelope-encryption.md +++ b/docs/explanation/envelope-encryption.md @@ -65,27 +65,33 @@ Three properties fall out of this layout: ## The cipher choices -There are two cipher modes the system actually ships: - -- **AES-256-GCM-SIV** (RFC 8452) is the default. It's - nonce-misuse resistant — accidentally reusing a nonce reveals - only that the same plaintext was encrypted twice, not the - plaintexts themselves. This matters because we derive `Kc` - deterministically from chunk hash, and we must not assume nonce - uniqueness across the whole chunk population. - -- **AES-256-GCM** with a random 96-bit nonce is the **FIPS - fallback**. BoringCrypto (which is what `GOEXPERIMENT=boringcrypto` - builds against) doesn't yet ship GCM-SIV, and we will not ship a - FIPS build that uses an unvalidated module. In FIPS mode the - agent computes a fresh 96-bit nonce per chunk and stores it in - the chunk envelope; the trade is "don't reuse nonces" replaces - "tolerates nonce reuse". - -The choice is build-time (`pg-hardstorage` vs `pg-hardstorage-fips`) -and surfaced in the manifest's `encryption.scheme` field -(`aes-256-gcm-siv` or `aes-256-gcm`). Readers handle both -transparently. +The cipher that ships today: + +- **AES-256-GCM** with a random 96-bit nonce is the cipher + shipping today. The agent computes a fresh 96-bit nonce per + chunk and stores it in the chunk envelope; correctness depends + on those nonces never repeating under a given key. + +The planned default: + +- **AES-256-GCM-SIV** (RFC 8452) is the planned default once a + validated implementation lands. It's nonce-misuse resistant — + accidentally reusing a nonce reveals only that the same + plaintext was encrypted twice, not the plaintexts themselves. + This matters because we derive `Kc` deterministically from chunk + hash, and we must not assume nonce uniqueness across the whole + chunk population. The Go standard library does not yet ship + GCM-SIV, so it is not implemented today. + +A FIPS contrast applies to the same shipping cipher. BoringCrypto +(which is what `GOEXPERIMENT=boringcrypto` builds against) doesn't +ship GCM-SIV either, and we will not ship a FIPS build that uses an +unvalidated module — so the FIPS variant (`pg-hardstorage-fips`) +will likewise use AES-256-GCM when GCM-SIV lands. + +The active scheme is surfaced in the manifest's `encryption.scheme` +field (`aes-256-gcm` today; `aes-256-gcm-siv` once it lands). +Readers handle both transparently. The on-disk chunk envelope is self-describing: diff --git a/docs/explanation/llm-safety-stack.md b/docs/explanation/llm-safety-stack.md index 3b2e890..1e1e8b5 100644 --- a/docs/explanation/llm-safety-stack.md +++ b/docs/explanation/llm-safety-stack.md @@ -252,9 +252,9 @@ Wrote signed evidence bundle to ./session-20260428T1423-db1-restore.evidence.tar - executed_commands.ndjson (every command actually run, exit code, duration) - audit_chain_proof.json (Merkle proof: this session's events anchor at chain pos 1428..1547) - skill_used.yaml (the exact skill file at the version used) - - skill_signature.sig (cosign signature) + - skill_signature.sig (Ed25519 signature, agent keyring) - model_metadata.json (provider, model id, model version, model fingerprint) - - signature.sig (cosign signature on the bundle) + - signature.sig (Ed25519 signature on the bundle) ``` The bundle is what an admin shows in a post-incident review or a @@ -284,10 +284,10 @@ functions baked into the binary. Implications: - **Skill isolation.** A bug in the postmortem skill cannot touch the restore skill. Each skill loads independently, has its own tool allowlist, its own guardrails, its own RBAC scope. -- **Signed.** Shipped skills are cosign-signed by the project - key; user-added skills are signed by the operator's local - cosign key. Loading an unsigned skill emits a critical audit - event and requires `--allow-unsigned-skill` confirmation. +- **Inspectable, with signing on the roadmap.** Skills load as + plain, reviewable YAML today. Cryptographic skill signing and + signature verification (a project key for shipped skills, the + operator's key for local ones) are planned, not yet shipped. - **Linted + golden-tested.** `pg_hardstorage llm skill lint ` validates the schema and static-checks the tool list (no banned tools, no missing required ones). diff --git a/docs/explanation/threat-model.md b/docs/explanation/threat-model.md index 4ffad37..e6f72c3 100644 --- a/docs/explanation/threat-model.md +++ b/docs/explanation/threat-model.md @@ -137,7 +137,7 @@ any object. **Defences:** -- **Per-chunk authentication.** AES-256-GCM(-SIV) is an AEAD +- **Per-chunk authentication.** AES-256-GCM is an AEAD cipher. Substituting a chunk's ciphertext for a different ciphertext fails authentication on read — the agent rejects the modified chunk with a structured error. Restore aborts. @@ -258,8 +258,9 @@ they have the controls. - Anomalous patterns (off-hours bulk reads, novel principals, unusual download volume) trigger alerts via the configured Sinks. -- The transparency-log-anchored audit chain means the attacker - cannot silently rewrite the evidence after the fact. +- The hash-chained, transparency-log-anchored (self-hosted) audit + chain means the attacker cannot silently rewrite the evidence + after the fact. The customer's external IAM and threat-detection systems are expected to cover this case. We provide the evidence trail. diff --git a/docs/explanation/wal-pipeline.md b/docs/explanation/wal-pipeline.md index 1222242..f301957 100644 --- a/docs/explanation/wal-pipeline.md +++ b/docs/explanation/wal-pipeline.md @@ -26,11 +26,14 @@ the agent decides which streaming mode to run. Three separate properties make the choice easy: -1. **It works on managed PostgreSQL.** RDS, Cloud SQL, Azure - Database, Aiven, Supabase, Neon — none of them let you install a - C extension or SSH into the host. All of them expose the - replication endpoint. Pull-based, file-based archive paths can - fail invisibly on these providers; the replication slot can't. +1. **It needs no host access.** On self-managed PostgreSQL you + often can't (or don't want to) SSH into the database host or + install a C extension. The replication endpoint is enough: the + agent runs from a different VM, region, or cluster. A persistent + slot also can't fail invisibly the way pull-based, file-based + archive paths can. (Fully-managed DBaaS — RDS, Cloud SQL, Azure + Database, Neon, Supabase — do **not** expose `BASE_BACKUP` or + physical replication, so they are out of scope.) 2. **A persistent slot closes the WAL gap window.** PG holds WAL in `pg_wal/` until the consumer (us) confirms the LSN with a @@ -229,8 +232,8 @@ Both shim into the same chunker pipeline: install for paranoid double-archiving. PG 15+. - **`archive_command` shim** — `/usr/bin/pg_hardstorage wal push %p` - for managed PG that doesn't expose `archive_library` and where - customers want classical archiving alongside streaming. Same + for self-managed PG that can't load `archive_library` and where + operators want classical archiving alongside streaming. Same chunker pipeline. Both paths feed the same content-addressed chunk store. CAS diff --git a/docs/faq.md b/docs/faq.md index 8f1d002..e5e7616 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -25,27 +25,58 @@ a documentation gap. ## Getting started -#### Will pg_hardstorage work against AWS RDS, GCP Cloud SQL, Aiven, Supabase, or Neon? - -Yes. The entire data plane runs over the PostgreSQL -**replication protocol** on a normal database connection — -no SSH, no OS access, no `archive_library` install needed. -Anything that exposes a standard PG replication endpoint to -a replication-role user works. This is the single biggest -architectural difference from pgBackRest; see the +#### Will pg_hardstorage work against AWS RDS, Aurora, GCP Cloud SQL, Azure Database, Neon, or Supabase? + +No. pg_hardstorage's data plane runs over the PostgreSQL +**physical replication protocol** — a physical replication +slot plus `BASE_BACKUP`. Fully-managed DBaaS providers do +not expose `BASE_BACKUP` (or physical replication) to +customers, so pg_hardstorage cannot take a physical base +backup of them. It targets PostgreSQL you run yourself. + +**Why, per provider** (checked June 2026 against each +vendor's own replication documentation): RDS and Aurora are +documented not to support external physical replication / +`pg_basebackup`; Cloud SQL and Azure Database expose only +logical replication / decoding to external consumers +(physical replication there is internal managed-HA, not a +customer-reachable `BASE_BACKUP` endpoint); Neon uses a +custom storage architecture with no traditional +`pg_basebackup`; and hosted Supabase can *publish* via +logical replication but cannot serve as a physical replica +target. The general test: **if a service does not let an +external client open a physical replication connection and +run `BASE_BACKUP`, pg_hardstorage cannot back it up.** + +!!! note "What we have *not* verified" + This list is not exhaustive, and we have not run + pg_hardstorage end-to-end against every managed provider; + the above reflects each vendor's documented replication + capabilities, which can change over time. Treat the + `BASE_BACKUP` test as the source of truth — run + `pg_hardstorage wal preflight ` against your + endpoint to confirm it exposes what streaming needs before + relying on it. A self-hosted Postgres distribution you run + yourself (including self-hosted Supabase) counts as + self-managed and is supported. + +What the replication-protocol design *does* remove is the +need for host-level access. On self-managed PostgreSQL — +bare metal, VMs, containers, Patroni clusters, and operators +like CloudNativePG — the agent backs up over a normal database +connection with no SSH, no OS access, and no +`archive_library`/`archive_command` on the host, and it can +run in a different VM, region, or cluster from the database. +This is the single biggest architectural difference from +pgBackRest; see the [architecture tour](explanation/architecture-tour.md#2-wal-via-the-replication-protocol) for the full reasoning. -The one caveat: managed PG generally does not let you load -a custom `archive_library`, so the optional double-archive -path is unavailable. Streaming replication is the primary -path either way, so this changes nothing about backup -correctness. - #### What PostgreSQL versions are supported? -PG 15, 16, and 17 are first-class. PG 18 is supported in -the test matrix from v0.9 onward. PG 14 and earlier are +PG 15, 16, and 17 are first-class — required to pass CI. +PG 18 runs in the test matrix under allow-failure guardrails +while readiness is confirmed. PG 14 and earlier are out of scope — `BASE_BACKUP` non-exclusive mode and `pg_backup_start` / `pg_backup_stop` are PG 15-only APIs we rely on. @@ -558,9 +589,9 @@ Tier-2 plugins via your own channel; a checked-in #### How is this different from pgBackRest? -The short version: WAL via replication protocol (works on -managed PG), no chained incrementals (CAS dedup with no -chain dependency), encryption / KMS / audit chain / WORM +The short version: WAL via replication protocol (no SSH or +`archive_command` on the host), no chained incrementals (CAS +dedup with no chain dependency), encryption / KMS / audit chain / WORM on by default. pgBackRest's strengths — production maturity at very large scale, mature operator integrations — are real and we're explicit about them. Full comparison diff --git a/docs/glossary.md b/docs/glossary.md index f7591e6..f1993ef 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -28,12 +28,14 @@ the same turn, with explicit user confirmation. Off by default. See the [LLM safety stack](explanation/llm-safety-stack.md). -#### AES-256-GCM-SIV - -The default chunk-encryption cipher — RFC 8452, -nonce-misuse-resistant. AES-256-GCM with a random 96-bit -nonce is used in FIPS mode where BoringCrypto doesn't yet -expose GCM-SIV. See +#### AES-256-GCM + +The chunk-encryption cipher shipping today — a random +96-bit nonce per chunk. AES-256-GCM-SIV (RFC 8452, +nonce-misuse-resistant) is the planned default once a +validated implementation lands; the Go standard library +does not yet ship it, and BoringCrypto (FIPS) does not +expose it either. See [envelope encryption](explanation/envelope-encryption.md). #### Agent @@ -48,8 +50,8 @@ multiplexes every deployment on that host. See #### `archive_command` PG's per-segment WAL archive hook. pg_hardstorage offers -a thin shim (`pg_hardstorage wal push %p`) for managed-PG -environments where the streaming path can't be used; both +a thin shim (`pg_hardstorage wal push %p`) for setups that +want classical archiving alongside streaming; both shims feed the same content-addressed chunk store. See [wal pipeline](explanation/wal-pipeline.md). @@ -62,7 +64,8 @@ C) for the optional double-archiving path. #### Attestation -A cosign-signed claim attached to a manifest or a build +An Ed25519-signed claim (agent keyring) attached to a backup +manifest, or a cosign-signed claim attached to a release artefact, optionally anchored to a transparency log. Both backups and release binaries carry attestations. See [audit chain](explanation/audit-chain.md). @@ -71,8 +74,9 @@ backups and release binaries carry attestations. See The append-only, hash-chained Merkle log of every significant event (backup committed, restore started, KMS -rotated, LLM session opened, …). Periodic anchors land in -Rekor. Verifiable post-hoc via +rotated, LLM session opened, …). Periodic anchors land in a +transparency log (self-hosted today; external Rekor anchoring +is roadmap). Verifiable post-hoc via `pg_hardstorage audit verify-chain`. See [audit chain](explanation/audit-chain.md). @@ -175,10 +179,10 @@ coordination service needed. See #### cosign -The Sigstore CLI used to sign manifests, attestations, and -release artefacts. Public keys are pinned in repo -configuration; verification is mandatory on every manifest -read. +The Sigstore CLI used to sign release artefacts (keyless). +Backup manifests are signed separately with the agent's +Ed25519 keyring, and that signature is verified on every +manifest read. #### Cross-account replication @@ -215,7 +219,7 @@ from the BDEK via HKDF. See #### Deployment A logical PostgreSQL service we back up — one Patroni -cluster, one RDS instance, one CNPG `Cluster`. Replaces +cluster, one standalone primary, one CNPG `Cluster`. Replaces the word *stanza* from pgBackRest. Bound to ≥ 1 agents for HA. @@ -496,8 +500,8 @@ against single-key corruption. Recoverable via The PG protocol (`replication=database` connection parameter) used for `BASE_BACKUP`, `START_REPLICATION`, and `IDENTIFY_SYSTEM`. pg_hardstorage's entire data plane -runs over this — the reason it works on managed PG -without OS access. See +runs over this — the reason the agent needs no SSH or OS +access to the database host. See [wal pipeline](explanation/wal-pipeline.md). #### Replication slot diff --git a/docs/how-to/kubernetes/helm-sidecar-chart.md b/docs/how-to/kubernetes/helm-sidecar-chart.md index c10f36b..2c6711b 100644 --- a/docs/how-to/kubernetes/helm-sidecar-chart.md +++ b/docs/how-to/kubernetes/helm-sidecar-chart.md @@ -13,7 +13,7 @@ tags: > Install `charts/pg-hardstorage-sidecar` to run the host > agent as a StatefulSet that backs up an external -> PostgreSQL cluster (managed RDS / CloudSQL, bare metal, +> self-managed PostgreSQL cluster (bare metal or > VMs reachable from the cluster network). One replica; > persistent state PVC; the agent's same posture you'd run > as a `systemd` service on a VM, packaged for K8s. diff --git a/docs/how-to/kubernetes/index.md b/docs/how-to/kubernetes/index.md index ffdc939..21ec14a 100644 --- a/docs/how-to/kubernetes/index.md +++ b/docs/how-to/kubernetes/index.md @@ -49,7 +49,7 @@ shapes: | You run | Use | |------------------------------------------|--------------------------------------| -| External PG (managed RDS / CloudSQL / VM) | [Sidecar chart](helm-sidecar-chart.md) | +| External self-managed PG (VM / bare metal / non-operator) | [Sidecar chart](helm-sidecar-chart.md) | | CloudNativePG (CNPG) | [CNPG-I provider](cnpg-i-provider.md) (v0.5) | | Zalando postgres-operator | [WAL-G shim](walg-shim.md) | | Crunchy PGO | [pgBackRest shim](pgbackrest-shim.md) | diff --git a/docs/how-to/operating/schedule-backups.md b/docs/how-to/operating/schedule-backups.md index 07d2938..df25c9e 100644 --- a/docs/how-to/operating/schedule-backups.md +++ b/docs/how-to/operating/schedule-backups.md @@ -148,11 +148,10 @@ restarts don't reset the cadence. ## Schedules and the LLM helper -`pg_hardstorage llm propose backup-schedule …` is the -LLM-assisted variant: given a constraint set ("RPO ≤ 1h, off- -peak window 02-05"), it proposes a schedule, runs it through -the same parser, and stages the change. The schedule writes -themselves still go through `pg_hardstorage schedule`. See +`pg_hardstorage llm ask "..."` is the LLM-assisted variant: +given a constraint set ("RPO ≤ 1h, off-peak window 02-05"), it +can suggest a schedule and explain the trade-offs. The schedule +writes themselves still go through `pg_hardstorage schedule`. See the [`llm` CLI reference](../../reference/cli/pg_hardstorage_llm.md). ## Troubleshooting diff --git a/docs/how-to/operating/slot-disk-safety.md b/docs/how-to/operating/slot-disk-safety.md index ca20c68..6aa284f 100644 --- a/docs/how-to/operating/slot-disk-safety.md +++ b/docs/how-to/operating/slot-disk-safety.md @@ -53,8 +53,8 @@ the failure before it becomes an incident. ### Most deployments — bound the slot, alert on lag -For the typical case — a managed PG or a single primary where -losing a few MB of WAL after a sustained outage is acceptable — +For the typical case — a single primary where losing a few MB +of WAL after a sustained outage is acceptable — configure `max_slot_wal_keep_size` to a value larger than the worst-case streamer downtime you can tolerate, and alert before the cap is reached. diff --git a/docs/how-to/packaging/build-from-source.md b/docs/how-to/packaging/build-from-source.md index 9239b71..2d244b4 100644 --- a/docs/how-to/packaging/build-from-source.md +++ b/docs/how-to/packaging/build-from-source.md @@ -71,7 +71,7 @@ make all-binaries ``` ```console -pg_hardstorage v0.9.x +pg_hardstorage v1.0.x commit: abcdef1 date: 2026-05-04T08:13:42Z fips: false @@ -79,7 +79,7 @@ pg_hardstorage v0.9.x ``` A clean checkout that's not on a tagged commit produces -`v0.9.x-N-gabcdef1` — which is what you want, because that's +`v1.0.x-N-gabcdef1` — which is what you want, because that's the unique identifier of the build that's running. ## What just happened @@ -160,7 +160,7 @@ rather than calling `go build` directly, or pass the flag yourself: ```bash -go build -ldflags '-X github.com/cybertec-postgresql/pg_hardstorage/internal/version.Version=v0.9.0-dev' ... +go build -ldflags '-X github.com/cybertec-postgresql/pg_hardstorage/internal/version.Version=v1.0.0-dev' ... ``` ### Binary works on the build host but not on a target diff --git a/docs/how-to/packaging/fips-variant.md b/docs/how-to/packaging/fips-variant.md index b65a2e8..7ed25dd 100644 --- a/docs/how-to/packaging/fips-variant.md +++ b/docs/how-to/packaging/fips-variant.md @@ -90,7 +90,7 @@ subcommand also surfaces the flag: ``` ```console -pg_hardstorage v0.9.x +pg_hardstorage v1.0.x commit: abcdef1 ... fips: true diff --git a/docs/how-to/packaging/pkcs11-variant.md b/docs/how-to/packaging/pkcs11-variant.md index 461af09..1916c45 100644 --- a/docs/how-to/packaging/pkcs11-variant.md +++ b/docs/how-to/packaging/pkcs11-variant.md @@ -111,7 +111,7 @@ The system section lists every configured KEK and reports ## What just happened -The `pkcs11` build tag activates `internal/plugin/encryption/pkcs11`, +The `pkcs11` build tag activates `internal/plugin/kms/pkcs11`, which holds the `EncryptionPlugin` implementation backed by `miekg/pkcs11`. At runtime the provider: diff --git a/docs/how-to/verify/firecracker-sandbox.md b/docs/how-to/verify/firecracker-sandbox.md index e503187..8bc816f 100644 --- a/docs/how-to/verify/firecracker-sandbox.md +++ b/docs/how-to/verify/firecracker-sandbox.md @@ -46,11 +46,9 @@ __PG_HARDSTORAGE_VERIFY__:FAIL __PG_HARDSTORAGE_VERIFY__:SKIPPED ``` -A reference rootfs build script ships under -`scripts/firecracker-rootfs.sh`. Operators with custom -security baselines roll their own rootfs and keep the magic -prefix exact — the prefix is part of the v1.0 schema- -compatibility commitment. +No rootfs image is bundled: operators build their own +Firecracker rootfs and keep the magic prefix exact — the +prefix is part of the v1.0 schema-compatibility commitment. The init script must halt after printing (`reboot -f` is fine; the kernel cmdline carries `panic=1 reboot=k` so any @@ -76,10 +74,9 @@ The Firecracker backend refuses with a clear remediation if ```bash pg_hardstorage-firecracker verify db1 latest \ --repo s3://acme-pg-backups \ - --full \ - --sandbox-backend firecracker \ - --firecracker-kernel /var/lib/pg_hardstorage/firecracker/vmlinux \ - --firecracker-rootfs /var/lib/pg_hardstorage/firecracker/rootfs.ext4 + --full +# The sandbox backend and the kernel / rootfs paths are configured in +# pg_hardstorage.yaml (see step 3), not via CLI flags. ``` (For agent-driven verify the same fields go in diff --git a/docs/index.md b/docs/index.md index c698dcd..e1eaf52 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,8 +27,8 @@ description: >- `pg_hardstorage` is a single Go binary that doubles as a long-running host agent and an interactive CLI. It ferries base backups and WAL via the **PostgreSQL replication -protocol** (so it works against managed databases like RDS, -Cloud SQL, and Aiven without OS-level access), deduplicates +protocol** (so it needs no SSH, OS access, or `archive_command` +on the database host), deduplicates content-addressed chunks, AES-256-GCM-encrypts under a per-backup DEK wrapped by a configurable KEK, and signs every manifest with Ed25519. diff --git a/docs/operations/incident-response.md b/docs/operations/incident-response.md index d13f17b..6863323 100644 --- a/docs/operations/incident-response.md +++ b/docs/operations/incident-response.md @@ -47,7 +47,7 @@ after acknowledging each. | --- | --- | --- | --- | | `HSWALSilence`, `wal.slot_missing` | Replication slot dropped | `pg_hardstorage wal repair ` | [R6](../reference/runbooks/R6-slot-dropped-gap.md) | | `HSWALSilence`, slot present, lag rising | Network partition or PG primary stuck | Check `pg_replication_slots`; verify libpq reachability | [R6](../reference/runbooks/R6-slot-dropped-gap.md) | -| `HSBackupOverdue`, RPO target exceeded | Schedule paused / agent down | `pg_hardstorage agent status`; rerun `backup` manually | [R3](../reference/runbooks/R3-cold-start-from-backups.md) | +| `HSBackupOverdue`, RPO target exceeded | Schedule paused / agent down | `pg_hardstorage status `; rerun `backup` manually | [R3](../reference/runbooks/R3-cold-start-from-backups.md) | | `HSKEKUnreachable`, `kms.unreachable` | KMS provider degraded or IAM revoked | Check provider console; restore IAM grants | [R2](../reference/runbooks/R2-kms-key-destroyed.md) | | `HSScrubFindings{kind="bit-rot"}` | Chunk corruption at rest | Quarantine prefix; heal from replica region | [R4](../reference/runbooks/R4-repo-corruption-at-rest.md) | | `HSScrubFindings{kind="missing"}` | Manifest references absent chunk | `repo check` to confirm; restore from replica | [R4](../reference/runbooks/R4-repo-corruption-at-rest.md) | @@ -151,7 +151,7 @@ Once the incident is resolved: --since "2026-04-28T03:00:00Z" \ --until "2026-04-28T09:00:00Z" \ --include-anchors \ - -o ./incident-2026-04-28.tar.gz + --out ./incident-2026-04-28.tar.gz ``` See [audit evidence bundles](../compliance/audit-evidence-bundles.md) diff --git a/docs/operations/monitoring.md b/docs/operations/monitoring.md index b3f2746..29dc8d1 100644 --- a/docs/operations/monitoring.md +++ b/docs/operations/monitoring.md @@ -355,7 +355,7 @@ The agent's HTTP listener exposes: | --- | --- | | `/healthz` | Liveness — the process is alive. Always 200 if reachable. | | `/readyz` | Readiness — KMS reachable, repo reachable, leader-elected. | -| `/doctor` | Full doctor report as JSON; the same content as `pg_hardstorage doctor --json`. | +| `/doctor` | Full doctor report as JSON; the same content as `pg_hardstorage doctor -o json`. | | `/metrics` | Prometheus scrape. Served by the control plane unconditionally; served by the agent only when started with `--metrics-listen`. | Wire `/healthz` to your container liveness probe and `/readyz` diff --git a/docs/operations/operator-guide.md b/docs/operations/operator-guide.md index ef0d1d1..0ef6098 100644 --- a/docs/operations/operator-guide.md +++ b/docs/operations/operator-guide.md @@ -33,7 +33,7 @@ The pipeline is `BASE_BACKUP` over libpq → tar parser → FastCDC chunker → CAS PUTs → signed manifest. Default compression is zstd `SpeedBetterCompression`; default encryption is on if a KEK is present at the keyring path. Force one or the other with -`--encrypt` / `--no-encrypt`, `--no-compression`. Backup IDs follow +`--encrypt` / `--no-encrypt`. Backup IDs follow the shape `db1.full.20260427T093017Z`. For NDJSON progress (one event per line) suitable for piping to `jq`: @@ -117,7 +117,7 @@ pg_hardstorage restore db1 latest \ Prints what would happen — source backup, WAL replay range, RTO estimate, target tablespace mapping, verification gate — and exits -without touching disk. Pair with `--confirm` to run the same +without touching disk. Pair with `--force` to run the same operation non-interactively after operator review. ### Refusals (pre-flight, exit 4) diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index e898c4e..221ab63 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -290,9 +290,9 @@ restore won't overwrite arbitrary files. - For a clean restore: pick an empty target directory. - For a deliberate overwrite: pass `--force`. The body lists every - top-level entry that will be deleted, and the operation prompts - again before doing anything. With `--force --confirm` the second - prompt is suppressed. + top-level entry that will be deleted before it proceeds. Overwriting + a target that is a *different* cluster additionally requires + `--force-foreign`. `--force` is destructive: it removes the existing contents of the target directory before unpacking. Use it on data dirs, not on `/`. diff --git a/docs/operations/upgrade-procedures.md b/docs/operations/upgrade-procedures.md index b505bf3..3d4e3f3 100644 --- a/docs/operations/upgrade-procedures.md +++ b/docs/operations/upgrade-procedures.md @@ -176,8 +176,8 @@ Do it host-by-host inside the one-minor compat window: `doctor` is clean. 2. Repeat for every non-leader. 3. For the leader (or active control plane), trigger a leader - election (`pg_hardstorage agent --step-down` in v0.5+, or - `systemctl restart` today) so a freshly-upgraded agent takes + election (`systemctl restart` today; a dedicated agent + step-down flag is planned) so a freshly-upgraded agent takes over. 4. Replace the old leader's binary, restart. diff --git a/docs/reference/cli/pg_hardstorage_kms.md b/docs/reference/cli/pg_hardstorage_kms.md index 3b4f477..1db36eb 100644 --- a/docs/reference/cli/pg_hardstorage_kms.md +++ b/docs/reference/cli/pg_hardstorage_kms.md @@ -16,7 +16,7 @@ Manage encryption keys ### Synopsis -v0.1.1 ships the read-only `kms inspect` surface. Mutating +The read-only `kms inspect` surface. Mutating verbs (rotate, shred, hsm-status) land alongside the KMS plugin tier (gcp-kms, azure-key-vault, vault-transit) and the PKCS#11 / TPM 2.0 integrations. diff --git a/docs/reference/cli/pg_hardstorage_wal.md b/docs/reference/cli/pg_hardstorage_wal.md index 6c71537..1d6df70 100644 --- a/docs/reference/cli/pg_hardstorage_wal.md +++ b/docs/reference/cli/pg_hardstorage_wal.md @@ -19,10 +19,11 @@ WAL transport: continuous streaming, push/fetch, list, repair WAL transport plumbing. The streaming subcommand connects via the PostgreSQL replication -protocol over a libpq connection — no host-level access needed, so -this works against managed databases (RDS, Cloud SQL, ...). +protocol over a libpq connection — no host-level access to the database +is needed. It targets PostgreSQL you run yourself; fully-managed DBaaS +(RDS, Cloud SQL, ...) do not expose BASE_BACKUP / physical replication +and are not supported. -The push / fetch / list / repair subcommands are scaffold for now. ### Options diff --git a/docs/reference/index.md b/docs/reference/index.md index ff65026..bf45290 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -20,7 +20,7 @@ committed page. - [CLI reference](cli/index.md) — one page per `pg_hardstorage` subcommand, auto-generated from the - Cobra command tree. 202 pages today. + Cobra command tree. 216 pages today. ## REST API diff --git a/docs/reference/plugins/index.md b/docs/reference/plugins/index.md index 73518cf..1c55c40 100644 --- a/docs/reference/plugins/index.md +++ b/docs/reference/plugins/index.md @@ -80,8 +80,7 @@ func init() { } ``` -The `cmd/pg_hardstorage/main.go` (or a build-flavour- -specific `cmd/pg_hardstorage_fips/main.go`) imports the +The `cmd/pg_hardstorage/main.go` imports the concrete plugin packages with `_ "…/internal/plugin/sink/slack"` to trigger that side-effect. Drop the import; lose the plugin. diff --git a/docs/reference/plugins/llm-provider-contract.md b/docs/reference/plugins/llm-provider-contract.md index a4afe2c..dfec21b 100644 --- a/docs/reference/plugins/llm-provider-contract.md +++ b/docs/reference/plugins/llm-provider-contract.md @@ -16,7 +16,7 @@ layered on top via `SupportsTools()`; the orchestrator in `internal/llm/chat` handles prompt construction, message history, and tool dispatch. -v0.9+ ships exactly two providers in-tree: +v1.0 ships exactly two providers in-tree: - **`mock`** — for tests; canned responses. - **`openai`** — speaks the OpenAI Chat Completions wire @@ -27,7 +27,7 @@ v0.9+ ships exactly two providers in-tree: llama.cpp server. Older Anthropic-native and Ollama-native providers were -removed in v0.9+ (audit-driven simplification): one wire +removed before v1.0 (audit-driven simplification): one wire format, one set of tests, one deployment story including the air-gapped path. diff --git a/docs/reference/plugins/source-contract.md b/docs/reference/plugins/source-contract.md index dedbcfb..cc70868 100644 --- a/docs/reference/plugins/source-contract.md +++ b/docs/reference/plugins/source-contract.md @@ -18,7 +18,7 @@ managed-disk). !!! warning "Forward-looking contract" The Source tier is **not yet refactored into a - pluggable interface**. v0.9+ ships a single + pluggable interface**. v1.0 ships a single streaming-backup implementation in `internal/backup/runner/` whose orchestration is hard- coded. The interface signature below is the SPEC's diff --git a/docs/reference/plugins/tier2-go-plugin-protocol.md b/docs/reference/plugins/tier2-go-plugin-protocol.md index 51e135f..19c7eed 100644 --- a/docs/reference/plugins/tier2-go-plugin-protocol.md +++ b/docs/reference/plugins/tier2-go-plugin-protocol.md @@ -22,7 +22,7 @@ plugin-side dispatcher both); the gRPC-shaped contract that v1.1 will move to is at `proto/plugin/v1/plugin.proto`. !!! note "Two protocol shapes" - pg_hardstorage v0.9 ships a **stdio JSON-RPC** + pg_hardstorage v1.0 ships a **stdio JSON-RPC** protocol (`pg_hardstorage.plugin.v1`) for Tier-2 plugins. The SPEC and the in-repo `.proto` file describe the gRPC-over-`hashicorp/go-plugin` contract diff --git a/docs/reference/runbooks/control-plane-setup.md b/docs/reference/runbooks/control-plane-setup.md index 6b5ce4e..ea9d9b5 100644 --- a/docs/reference/runbooks/control-plane-setup.md +++ b/docs/reference/runbooks/control-plane-setup.md @@ -244,7 +244,7 @@ curl --cacert /etc/pg_hardstorage/server/cert.pem \ # { # "id": "db1.example.com", # "host": "db1.example.com", -# "version": "v0.4.0", +# "version": "v1.0.3", # "deployments": ["db1", "db2"], # "registered_at": "...", # "last_heartbeat": "..." diff --git a/docs/reference/storage-url-schemes.md b/docs/reference/storage-url-schemes.md index b1d07a3..2f0f65f 100644 --- a/docs/reference/storage-url-schemes.md +++ b/docs/reference/storage-url-schemes.md @@ -1,7 +1,7 @@ --- title: Storage URL schemes -description: The five repo-URL schemes — URL form, query parameters, auth chain, and capability matrix. +description: The six repo-URL schemes — URL form, query parameters, auth chain, and capability matrix. tags: - reference - storage diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index ac83cc5..cdc315f 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -20,7 +20,7 @@ operator deciding whether to upgrade. compatible — no migration. - **[v1.0](v1.0.md)** — the first stable release. Five Tier-1 KMS providers (AWS / GCP / Azure / Vault / HSM), - five Tier-1 storage backends (fs / s3 / gcs / azblob / - sftp), Patroni-aware WAL streaming, LLM-assisted + six Tier-1 storage backends (fs / s3 / gcs / azblob / + sftp / scp), Patroni-aware WAL streaming, LLM-assisted operations, two verifier sandboxes, full compliance surface. 24-month schema-compatibility commitment. diff --git a/docs/release-notes/v1.0.md b/docs/release-notes/v1.0.md index e49dfb8..8cd6f5d 100644 --- a/docs/release-notes/v1.0.md +++ b/docs/release-notes/v1.0.md @@ -15,17 +15,18 @@ batches and 27 audit cycles since v0.9. The headline: **you can pick any of the three major clouds + on-prem HSM** and get a code-change-free deployment, with a backup data plane that talks the PostgreSQL replication protocol -(so it works against managed databases like RDS, Cloud SQL, -and Aiven without OS-level access to the database host). +(so it needs no SSH or OS-level access to the database host — +fully-managed DBaaS that withhold `BASE_BACKUP`, such as RDS +or Cloud SQL, are out of scope). The v1.0 commitment: **24-month backward compatibility on every on-disk and on-the-wire schema** — manifests, config, output JSON, on-disk chunk envelope. Operators who write scripts against v1.0 keep working through 2028. -Apache 2.0. PostgreSQL 15+, with PG 17 as the default -sandbox version and PG 18 supported with allow-failure CI -guardrails. +Apache 2.0. PostgreSQL 15+, with PG 18 as the default +sandbox/demo version and PG 18 carried in CI under +allow-failure guardrails. --- @@ -56,7 +57,7 @@ WrapDEK / UnwrapDEK round-trips through it. The documents per-provider URL forms, auth chains, FIPS posture, and Shred semantics. -### 2. Five Tier-1 storage backends +### 2. Six Tier-1 storage backends Three major hyperscalers + on-prem fs + SFTP. Every backend supports the same `StoragePlugin` contract: native @@ -242,9 +243,10 @@ Plus governance primitives: ## Plugin model - **Tier-1** — in-tree Go interfaces compiled into the - main binary. All five KMS providers, five storage - backends, eleven renderers, fourteen sinks, eight LLM - providers ship as Tier-1. + main binary. Five KMS providers (PKCS#11/HSM in the + `pkcs11`/FIPS build), six storage backends, eleven + renderers, fourteen sinks, and one LLM provider (the + OpenAI-compatible client) ship as Tier-1. - **Tier-2** — out-of-tree plugins discovered via `$HSPLUGIN_PATH` and the stdio JSON-RPC `--probe` handshake. An operator can write a `mem://` storage @@ -261,7 +263,7 @@ walks through a working Tier-2 plugin in 30 minutes. The site you are reading lands with v1.0: - - **~110 hand-written pages** + 202 auto-generated CLI + - **~110 hand-written pages** + 216 auto-generated CLI reference pages + 202 manpages. - **Diátaxis-organised IA** — tutorials / how-to / reference / explanation, plus operations + compliance diff --git a/docs/tutorials/getting-started-simple.md b/docs/tutorials/getting-started-simple.md index c35ea8f..ed0c401 100644 --- a/docs/tutorials/getting-started-simple.md +++ b/docs/tutorials/getting-started-simple.md @@ -29,7 +29,7 @@ just wants to *back this database up*. ## What you need -- `pg_hardstorage` ≥ v0.9 installed on `$PATH` +- `pg_hardstorage` ≥ v1.0 installed on `$PATH` - `pg_hardstorage_simple` next to it (same release, same `make`) - A reachable PostgreSQL 15+ instance with a connection string that includes the REPLICATION attribute on the user diff --git a/docs/tutorials/getting-started.md b/docs/tutorials/getting-started.md index e6308e6..d29f085 100644 --- a/docs/tutorials/getting-started.md +++ b/docs/tutorials/getting-started.md @@ -9,9 +9,15 @@ backup on a schedule (e.g. nightly) that the stream rolls forward from. Daily backup + always-on stream = PITR to any segment-aligned point. -It works against managed PG (RDS, Cloud SQL, Azure DB) the same as -bare metal, deduplicates and encrypts content-addressed chunks, and -restores with PITR. PG 15+, Apache 2.0. +It backs up PostgreSQL you run yourself — bare metal, VMs, containers, +Patroni clusters, and operators like CloudNativePG — over the physical +replication protocol, deduplicates and encrypts content-addressed +chunks, and restores with PITR. PG 15+, Apache 2.0. + +Managed DBaaS (Amazon RDS, Aurora, Cloud SQL, Azure Database, Neon, +Supabase, and similar) do **not** expose the `BASE_BACKUP` replication +command to customers, so pg_hardstorage cannot take a physical base +backup of them — see the [FAQ](../faq.md). This page gets you from zero to a running streamer, a first base backup, and a restored data dir in five minutes. After that, see @@ -29,8 +35,9 @@ tarballs (Windows is CLI-only). Grab the matching one from verify the cosign signature, and drop the binary on your `$PATH`: ```sh -curl -LO https://github.com/cybertec-postgresql/pg_hardstorage/releases/download/v0.1.1/pg_hardstorage_linux_amd64.tar.gz -tar xzf pg_hardstorage_linux_amd64.tar.gz +VERSION=1.0.3 # latest release: https://github.com/cybertec-postgresql/pg_hardstorage/releases/latest +curl -LO "https://github.com/cybertec-postgresql/pg_hardstorage/releases/download/v${VERSION}/pg_hardstorage_${VERSION}_linux_amd64.tar.gz" +tar xzf "pg_hardstorage_${VERSION}_linux_amd64.tar.gz" sudo install -m 0755 pg_hardstorage /usr/local/bin/ pg_hardstorage version ``` @@ -38,7 +45,8 @@ pg_hardstorage version ### `.deb` (Debian / Ubuntu) ```sh -sudo dpkg -i pg-hardstorage_0.1.1_amd64.deb +VERSION=1.0.3 # the release you downloaded +sudo dpkg -i "pg-hardstorage_${VERSION}_amd64.deb" ``` The package installs the binary at `/usr/bin/pg_hardstorage`, drops a @@ -49,7 +57,8 @@ creates `/etc/pg_hardstorage/`, `/var/lib/pg_hardstorage/`, ### `.rpm` (Fedora / RHEL / Rocky / Alma) ```sh -sudo rpm -i pg-hardstorage-0.1.1-1.x86_64.rpm +VERSION=1.0.3 # the release you downloaded +sudo rpm -i "pg-hardstorage-${VERSION}-1.x86_64.rpm" ``` Same layout as the `.deb`. @@ -57,8 +66,9 @@ Same layout as the `.deb`. ### Container image ```sh -docker pull ghcr.io/cybertec-postgresql/pg_hardstorage:v0.1.1 -docker run --rm ghcr.io/cybertec-postgresql/pg_hardstorage:v0.1.1 version +VERSION=1.0.3 # latest release tag +docker pull "ghcr.io/cybertec-postgresql/pg_hardstorage:v${VERSION}" +docker run --rm "ghcr.io/cybertec-postgresql/pg_hardstorage:v${VERSION}" version ``` The image is distroless. Mount a config dir at `/etc/pg_hardstorage` @@ -247,7 +257,7 @@ you are doing — exit 9 means the verifier said no. ```sh $ pg_hardstorage version -pg_hardstorage v0.1.1 (abc1234, built 2026-04-29T12:00:00Z) +pg_hardstorage v1.0.3 (abc1234, built 2026-04-29T12:00:00Z) ``` `doctor` is the single-command "is anything wrong" check: diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 49f530a..46415a1 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -16,7 +16,7 @@ take the full getting-started instead. ## Getting started - [Getting started (simple)](getting-started-simple.md) — the - interactive `pg_hardstorage simple` helper; a working backup in + interactive `pg_hardstorage_simple` helper; a working backup in five minutes, no config file to write. - [Getting started (full CLI)](getting-started.md) — the complete `init` → `backup` → `status` → `restore` walkthrough against a diff --git a/docs/tutorials/kubernetes-cnpg.md b/docs/tutorials/kubernetes-cnpg.md index 130bdcb..d09c8e3 100644 --- a/docs/tutorials/kubernetes-cnpg.md +++ b/docs/tutorials/kubernetes-cnpg.md @@ -16,6 +16,13 @@ tags: > MinIO repo running in-cluster. About 30 minutes from a clean > `kind` node. +!!! warning "Roadmap — not yet shipped" + The CNPG-I provider this tutorial uses lands with **v0.5**. The + `kubectl apply` of the provider manifest shown below does not work + against a released build yet. Until then, the verified path is the + [sidecar Helm chart](../how-to/kubernetes/helm-sidecar-chart.md) + plus a CronJob. This page documents the target shape. + CloudNativePG (CNPG) ships its own backup story; this tutorial wires the cluster to `pg_hardstorage` via the **CNPG-I provider** so the operator manages backups through the familiar `Cluster.spec.backup` diff --git a/internal/cli/kms.go b/internal/cli/kms.go index f688297..d09ed34 100644 --- a/internal/cli/kms.go +++ b/internal/cli/kms.go @@ -18,7 +18,7 @@ import ( ) // newKmsCmd implements `pg_hardstorage kms` — the operator-facing -// surface of the keyring. v0.1.1 ships only the read-only `inspect` +// surface of the keyring. Ships only the read-only `inspect` // verb; the mutating verbs (rotate, shred, hsm-status) require // substantial new infrastructure (rewrap walker; PKCS#11 binding) // and remain stubs in the SPEC's+ tree. @@ -33,7 +33,7 @@ func newKmsCmd() *cobra.Command { c := &cobra.Command{ Use: "kms", Short: "Manage encryption keys", - Long: `v0.1.1 ships the read-only ` + "`kms inspect`" + ` surface. Mutating + Long: `The read-only ` + "`kms inspect`" + ` surface. Mutating verbs (rotate, shred, hsm-status) land alongside the KMS plugin tier (gcp-kms, azure-key-vault, vault-transit) and the PKCS#11 / TPM 2.0 integrations.`, diff --git a/internal/cli/wal.go b/internal/cli/wal.go index 9be067a..0d75b53 100644 --- a/internal/cli/wal.go +++ b/internal/cli/wal.go @@ -55,10 +55,11 @@ func newWalCmd() *cobra.Command { Long: `WAL transport plumbing. The streaming subcommand connects via the PostgreSQL replication -protocol over a libpq connection — no host-level access needed, so -this works against managed databases (RDS, Cloud SQL, ...). - -The push / fetch / list / repair subcommands are scaffold for now.`, +protocol over a libpq connection — no host-level access to the database +is needed. It targets PostgreSQL you run yourself; fully-managed DBaaS +(RDS, Cloud SQL, ...) do not expose BASE_BACKUP / physical replication +and are not supported. +`, } c.AddCommand(newWalStreamCmd()) c.AddCommand(newWalPreflightCmd()) diff --git a/man/man1/pg_hardstorage-kms.1 b/man/man1/pg_hardstorage-kms.1 index 85a5381..ebe35a3 100644 --- a/man/man1/pg_hardstorage-kms.1 +++ b/man/man1/pg_hardstorage-kms.1 @@ -13,7 +13,7 @@ pg_hardstorage-kms - Manage encryption keys .SH DESCRIPTION .PP -v0.1.1 ships the read-only \fBkms inspect\fR surface. Mutating +The read-only \fBkms inspect\fR surface. Mutating verbs (rotate, shred, hsm-status) land alongside the KMS plugin tier (gcp-kms, azure-key-vault, vault-transit) and the PKCS#11 / TPM 2.0 integrations. diff --git a/man/man1/pg_hardstorage-wal.1 b/man/man1/pg_hardstorage-wal.1 index 934f03a..d6195c5 100644 --- a/man/man1/pg_hardstorage-wal.1 +++ b/man/man1/pg_hardstorage-wal.1 @@ -17,11 +17,10 @@ WAL transport plumbing. .PP The streaming subcommand connects via the PostgreSQL replication -protocol over a libpq connection — no host-level access needed, so -this works against managed databases (RDS, Cloud SQL, ...). - -.PP -The push / fetch / list / repair subcommands are scaffold for now. +protocol over a libpq connection — no host-level access to the database +is needed. It targets PostgreSQL you run yourself; fully-managed DBaaS +(RDS, Cloud SQL, ...) do not expose BASE_BACKUP / physical replication +and are not supported. .SH OPTIONS