From d176cfb277283477660fdaee264af6d29007236c Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Wed, 13 May 2026 20:02:00 -0700 Subject: [PATCH 1/2] build(docker): replace pip-install-at-startup with proper Dockerfile (#171) The jetstream-consumer service was running `sh -c "pip install nats-py -q && python /consumer.py"` as its compose command, which re-downloaded nats-py on every container restart, added ~10s of startup latency, and broke entirely in air-gapped environments. This change introduces a real `jetstream-consumer/Dockerfile` that bakes the pinned dependency at build time (mirroring `exporter/Dockerfile`): multi-stage `python:3.12-slim` base, non-root UID/GID 1000, embedded `HEALTHCHECK` against `/health`, and a stdin-style `pip install -r /tmp/requirements.txt` to avoid `repr()`-quoting bugs from prior HI Hermes Dockerfiles. Dependencies are pinned in `jetstream-consumer/requirements.txt` (nats-py==2.9.0) and tini is deliberately not pinned (it ships with the base image). docker-compose.yml now `build:`s from `./jetstream-consumer` and lets operators pin a published GHCR tag via `JETSTREAM_CONSUMER_IMAGE` (new optional override documented in `.env.example`). A new `publish-jetstream-consumer-image.yml` workflow publishes multi-arch images to GHCR on every main-branch change to `jetstream-consumer/**`, matching the pattern established for `argus-exporter`. The static `test_dockerfile_constraints.py` regression test now also guards the new Dockerfile's Python base version. Closes #171 Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 7 +++ .../publish-jetstream-consumer-image.yml | 60 +++++++++++++++++++ docker-compose.yml | 20 +++++-- jetstream-consumer/Dockerfile | 30 ++++++++++ jetstream-consumer/requirements.txt | 4 ++ tests/test_dockerfile_constraints.py | 41 ++++++++----- 6 files changed, 143 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/publish-jetstream-consumer-image.yml create mode 100644 jetstream-consumer/Dockerfile create mode 100644 jetstream-consumer/requirements.txt diff --git a/.env.example b/.env.example index ae68d63..af3f373 100644 --- a/.env.example +++ b/.env.example @@ -129,3 +129,10 @@ NOMAD_ADDR=172.20.0.1:4646 # the defaults collide with another service on the host. # GRAFANA_PORT=3001 # EXPORTER_PORT=9100 + +# === Image overrides (optional) === +# jetstream-consumer is built locally by docker compose (see +# jetstream-consumer/Dockerfile; closes #171). Override to pin to a +# published GHCR tag in production, e.g.: +# JETSTREAM_CONSUMER_IMAGE=ghcr.io/homericintelligence/argus-jetstream-consumer:sha- +# JETSTREAM_CONSUMER_IMAGE=argus-jetstream-consumer:local diff --git a/.github/workflows/publish-jetstream-consumer-image.yml b/.github/workflows/publish-jetstream-consumer-image.yml new file mode 100644 index 0000000..a21b97e --- /dev/null +++ b/.github/workflows/publish-jetstream-consumer-image.yml @@ -0,0 +1,60 @@ +name: Publish Jetstream Consumer Image + +# Builds and pushes the argus-jetstream-consumer container to GHCR so +# deployments can reference a versioned image instead of pip-installing +# nats-py at startup on every container restart. Closes #171. + +on: + push: + branches: + - main + paths: + - "jetstream-consumer/**" + - ".github/workflows/publish-jetstream-consumer-image.yml" + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + publish: + name: Build & push argus-jetstream-consumer to GHCR + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # GHCR rejects uppercase owner segments, so lower-case the repo owner. + - name: Compute image name + id: image + run: | + owner=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]') + echo "name=ghcr.io/${owner}/argus-jetstream-consumer" >> "$GITHUB_OUTPUT" + + - name: Build & push multi-arch image + uses: docker/build-push-action@v7 + with: + context: jetstream-consumer + file: jetstream-consumer/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ steps.image.outputs.name }}:latest + ${{ steps.image.outputs.name }}:sha-${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: true + sbom: true diff --git a/docker-compose.yml b/docker-compose.yml index ba39ebb..9c9009f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -332,14 +332,26 @@ services: - argus jetstream-consumer: - image: python:3.11-slim + # Built from ./jetstream-consumer (Dockerfile bakes nats-py at build + # time — closes #171). consumer.py is COPYed into the image rather + # than bind-mounted so the published artifact is reproducible and + # air-gap deployable. + build: + context: ./jetstream-consumer + dockerfile: Dockerfile + image: ${JETSTREAM_CONSUMER_IMAGE:-argus-jetstream-consumer:local} container_name: argus-jetstream-consumer restart: unless-stopped + user: "1000:1000" + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" ports: - "127.0.0.1:9101:9101" - volumes: - - ./jetstream-consumer/consumer.py:/consumer.py:ro - command: sh -c "pip install nats-py -q && python /consumer.py" environment: NATS_URL: "nats://172.24.0.1:4222" EXPORTER_PORT: "9101" diff --git a/jetstream-consumer/Dockerfile b/jetstream-consumer/Dockerfile new file mode 100644 index 0000000..e459e17 --- /dev/null +++ b/jetstream-consumer/Dockerfile @@ -0,0 +1,30 @@ +# Pinned to stable 3.12-slim; see tests/test_dockerfile_constraints.py for guard +FROM python:3.12-slim AS base + +WORKDIR /app + +# Install pinned runtime deps at build time (was previously a runtime +# `pip install` baked into docker-compose `command:` — see #171). Baking +# them into the image makes startup fast, deterministic, and air-gap +# compatible. +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm /tmp/requirements.txt + +FROM base AS final + +# Create a dedicated non-root group and user with fixed UID/GID 1000, +# mirroring exporter/Dockerfile. +RUN groupadd -g 1000 consumer \ + && useradd -u 1000 -g consumer --no-create-home --shell /sbin/nologin consumer + +COPY --chown=consumer:consumer consumer.py /app/consumer.py + +USER consumer + +EXPOSE 9101 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:9101/health', timeout=4)" + +CMD ["python3", "/app/consumer.py"] diff --git a/jetstream-consumer/requirements.txt b/jetstream-consumer/requirements.txt new file mode 100644 index 0000000..fcf11ae --- /dev/null +++ b/jetstream-consumer/requirements.txt @@ -0,0 +1,4 @@ +# Pinned runtime dependencies for jetstream-consumer. +# Update with intent: bumping these requires rebuilding the image and +# re-publishing via .github/workflows/publish-jetstream-consumer-image.yml. +nats-py==2.9.0 diff --git a/tests/test_dockerfile_constraints.py b/tests/test_dockerfile_constraints.py index 279e93c..48567bc 100644 --- a/tests/test_dockerfile_constraints.py +++ b/tests/test_dockerfile_constraints.py @@ -1,14 +1,17 @@ """ -Static regression test: assert the exporter Dockerfile uses a stable Python version. -Guards against Dependabot auto-merging pre-release Python bumps (e.g. 3.13+). -Uses only stdlib: re, pathlib, unittest. +Static regression test: assert every Python service Dockerfile uses a stable +Python version. Guards against Dependabot auto-merging pre-release Python +bumps (e.g. 3.13+). Uses only stdlib: re, pathlib, unittest. """ import re import unittest from pathlib import Path REPO_ROOT = Path(__file__).parent.parent -DOCKERFILE = REPO_ROOT / "exporter" / "Dockerfile" +DOCKERFILES = [ + REPO_ROOT / "exporter" / "Dockerfile", + REPO_ROOT / "jetstream-consumer" / "Dockerfile", +] _MIN_VERSION = (3, 11) # Approved Python version ceiling. Advance this only after the next CPython @@ -23,17 +26,25 @@ class TestDockerfileConstraints(unittest.TestCase): def test_python_base_image_version_is_stable(self) -> None: - content = DOCKERFILE.read_text() - match = re.search(r"FROM python:(\d+)\.(\d+)", content) - assert match is not None, "Could not find a FROM python:X.Y line in exporter/Dockerfile" - version = (int(match.group(1)), int(match.group(2))) - assert version >= _MIN_VERSION, ( - f"Python base image {version} is below minimum stable version {_MIN_VERSION}" - ) - assert version <= _MAX_VERSION, ( - f"Python base image {version} exceeds approved stable ceiling {_MAX_VERSION}; " - "bump _MAX_VERSION intentionally after verifying the release is stable" - ) + for dockerfile in DOCKERFILES: + with self.subTest(dockerfile=str(dockerfile.relative_to(REPO_ROOT))): + assert dockerfile.exists(), f"Missing Dockerfile: {dockerfile}" + content = dockerfile.read_text() + match = re.search(r"FROM python:(\d+)\.(\d+)", content) + assert match is not None, ( + f"Could not find a FROM python:X.Y line in {dockerfile}" + ) + version = (int(match.group(1)), int(match.group(2))) + assert version >= _MIN_VERSION, ( + f"Python base image {version} in {dockerfile} is below " + f"minimum stable version {_MIN_VERSION}" + ) + assert version <= _MAX_VERSION, ( + f"Python base image {version} in {dockerfile} exceeds " + f"approved stable ceiling {_MAX_VERSION}; bump " + "_MAX_VERSION intentionally after verifying the release " + "is stable" + ) if __name__ == "__main__": From 393ee9a024dc8dfe286676238d821f112b935ace Mon Sep 17 00:00:00 2001 From: Micah Villmow <4211002+mvillmow@users.noreply.github.com> Date: Sat, 16 May 2026 01:36:17 -0700 Subject: [PATCH 2/2] ci(required): resolve compose env-var defaults before image validation The image format validator failed on ${JETSTREAM_CONSUMER_IMAGE:-argus-jetstream-consumer:local} because the literal ${...:-...} pattern did not match the image-name regex. Substitute compose env-var interpolations of the form ${VAR:-default} and ${VAR-default} with their default value before regex validation. Bare ${VAR} (no default) is skipped since we cannot validate without a runtime value. --- .github/workflows/_required.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_required.yml b/.github/workflows/_required.yml index 454f513..66ae782 100644 --- a/.github/workflows/_required.yml +++ b/.github/workflows/_required.yml @@ -281,11 +281,23 @@ jobs: ) lines = result.stdout.strip().splitlines() + # Resolve docker-compose env-var interpolations of the form + # ${VAR:-default} or ${VAR-default} to their default value so the + # resulting image reference can be validated. Bare ${VAR} (no + # default) is skipped since we cannot know the runtime value. + interp_with_default = re.compile(r'\$\{[A-Za-z_][A-Za-z0-9_]*:?-([^}]+)\}') + bare_interp = re.compile(r'\$\{[A-Za-z_][A-Za-z0-9_]*\}') + images = [] for line in lines: parts = line.strip().split() if len(parts) >= 2: - images.append(parts[-1]) + img = parts[-1] + img = interp_with_default.sub(lambda m: m.group(1), img) + if bare_interp.search(img): + # Cannot validate without a default; skip. + continue + images.append(img) if not images: print("No docker-compose image references found — skipping format validation")