Tests/dockerd version compat

.github/workflows/pr_ci.yml

@@ -5,36 +5,38 @@ on:
     branches:
       - "main"
       - "develop"
+  workflow_dispatch:
+
+# Restrict the default GITHUB_TOKEN to read-only (CodeQL hardening).
+permissions:
+  contents: read
 
 jobs:
-    testing:
-        runs-on:
-            group: organization/netbox-docker-agent
-            labels:
-                - self-hosted
-                - Linux
-                - X64
-        steps:
-            - name: Start test env
-              run: |
-                  /usr/local/bin/server.sh start github_ci_netbox_docker_agent
-            - name: executing remote ssh commands using password
-              uses: appleboy/ssh-action@v1.0.3
-              env:
-                  HEAD_REF: ${{ github.head_ref }}
-              with:
-                  host: agent-1.netbox-docker-agent.github-ci.saashup.com
-                  key: ${{ secrets.KEY }}
-                  username: ${{ secrets.USER }}
-                  envs: HEAD_REF
-                  script: |
-                      git clone https://github.com/SaaShup/netbox-docker-agent.git -b $HEAD_REF
-                      cd netbox-docker-agent; docker build -t saashup/netbox-docker-agent .; cd ../
-                      docker network create netbox-docker-agent
-                      docker run -d -p 1880:1880 -v /var/run/docker.sock:/var/run/docker.sock:rw -v netbox-docker-agent:/data --name netbox-docker-agent --network netbox-docker-agent saashup/netbox-docker-agent
-                      sleep 30
-                      cat ./netbox-docker-agent/tests/hurl/tests.hurl | docker run --rm --network netbox-docker-agent -i ghcr.io/orange-opensource/hurl:latest --test --color --variable host=http://netbox-docker-agent:1880 -u admin:saashup
-            - name: Stop test env
-              if: ${{ always() }}
-              run: |
-                  /usr/local/bin/server.sh stop github_ci_netbox_docker_agent
+  # Read the supported dockerd versions from versions.txt into a matrix.
+  versions:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: set
+        name: Read versions.txt into a matrix
+        run: |
+          matrix=$(grep -vE '^[[:space:]]*(#|$)' tests/compat/versions.txt | jq -R . | jq -cs .)
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          echo "Testing versions: $matrix"
+
+  # Run the full test suite (smoke + reads + version + lifecycle + netbox +
+  # websocket exec) against each pinned dockerd version, in parallel.
+  test:
+    needs: versions
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        docker_version: ${{ fromJson(needs.versions.outputs.matrix) }}
+    name: dockerd ${{ matrix.docker_version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run test suite
+        run: tests/compat/run.sh "${{ matrix.docker_version }}"

tests/compat/README.md

@@ -0,0 +1,72 @@
+# Docker version compatibility tests
+
+These tests verify that the agent works against a controlled, pinned range of
+`dockerd` versions — independent of whatever Docker happens to be installed on
+the host.
+
+## How it works
+
+The agent talks to the Docker Engine API over a hardcoded
+`/var/run/docker.sock`, with no API-version prefix, so it always uses the
+daemon's *default* API version. To pin the version under test we use
+**Docker-in-Docker**:
+
+1. `dind` runs the chosen `docker:<version>-dind` and serves its socket on a
+   shared volume.
+2. `agent` mounts that same volume at `/var/run`, so its
+   `/var/run/docker.sock` *is* the dind daemon — no agent change needed.
+3. `tester` (hurl) runs [`../hurl/tests.hurl`](../hurl/tests.hurl) against the
+   agent.
+
+See [docker-compose.yml](docker-compose.yml). The stack also includes a
+`netbox` service (a [wiremock](https://wiremock.org/) stand-in) so the
+agent->netbox callbacks can be exercised; the agent's `netbox_url` is pointed
+at it via [fixtures/config.netbox.js](fixtures/config.netbox.js).
+
+## What gets tested
+
+For each version `run.sh` runs three things against the standing stack:
+
+1. **The hurl suite** ([../hurl](../hurl)):
+   - `tests.hurl` — the original HTTP smoke tests.
+   - `read.hurl` — field-level assertions on the synchronous read endpoints
+     (`/api/networks`, `/api/containers`, `/api/images`, `/api/volumes`,
+     `/system/usage`). Where dockerd field drift shows up first.
+   - `version.hurl` — asserts `/info` reports the exact dockerd version
+     under test (requires the `docker_version` variable).
+   - `lifecycle.hurl` — a real container lifecycle against the daemon:
+     pull -> create -> start -> logs -> stats -> exec -> stop -> delete,
+     polling the read endpoints for each async side effect.
+2. **The websocket-exec test** ([ws-exec-test.mjs](ws-exec-test.mjs)) — drives
+   the interactive `/ws` exec channel using the agent's own bundled client lib
+   (can't be expressed in hurl). Runs inside the agent container.
+3. **The netbox-contract test** (`netbox.hurl`) — asserts the agent actually
+   sends the expected callbacks to netbox after a write, by inspecting the
+   wiremock request journal. Run against a freshly restarted agent so the
+   agent's own config-persistence during the suite can't interfere.
+
+## Supported versions
+
+The list of tested versions lives in [versions.txt](versions.txt), one per
+line. It is the single source of truth, consumed by both `run.sh` and CI.
+
+## Running locally
+
+Requires Docker with the Compose plugin.
+
+```sh
+# Test every version in versions.txt
+./run.sh
+
+# Test a single version
+./run.sh 29.5.2
+```
+
+The script builds the agent image, spins up dind + agent for each version,
+runs the hurl suite, tears the stack down, and prints a pass/fail summary.
+
+## Adding a version
+
+Add the version to [versions.txt](versions.txt) and open a PR. CI picks it up
+automatically via the matrix in
+[`../../.github/workflows/compat_ci.yml`](../../.github/workflows/compat_ci.yml).

tests/compat/docker-compose.yml

@@ -0,0 +1,98 @@
+# Compatibility test stack: pins the dockerd version the agent talks to.
+#
+#   dind   -> runs the chosen dockerd version, exposes its unix socket on a
+#             shared volume at /var/run/docker.sock
+#   agent  -> the netbox-docker-agent image, with that same volume mounted at
+#             /var/run, so its hardcoded /var/run/docker.sock points at dind
+#   tester -> hurl, runs the existing suite against the agent
+#
+# Driven by run.sh; set DOCKER_VERSION to pick the dockerd version.
+name: nda-compat
+
+services:
+  dind:
+    image: docker:${DOCKER_VERSION:?set DOCKER_VERSION (e.g. 29.5.2)}-dind
+    privileged: true
+    environment:
+      # Empty cert dir disables TLS, so dockerd serves a plain unix socket.
+      DOCKER_TLS_CERTDIR: ""
+    volumes:
+      - docker-run:/var/run
+    healthcheck:
+      test: ["CMD", "docker", "version"]
+      interval: 3s
+      timeout: 5s
+      retries: 30
+      start_period: 5s
+
+  # Stand-in for netbox: records the callbacks the agent makes after write ops
+  # (catch-all returns empty results so flows that read netbox first don't choke).
+  netbox:
+    image: wiremock/wiremock:3.9.1
+    command: ["--port", "8080", "--disable-banner"]
+    volumes:
+      - ./fixtures/wiremock/mappings:/home/wiremock/mappings:ro
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://localhost:8080/__admin/health"]
+      interval: 3s
+      timeout: 5s
+      retries: 20
+      start_period: 3s
+
+  agent:
+    build:
+      context: ../..
+    image: saashup/netbox-docker-agent:compat
+    # The image runs as the unprivileged node-red user, but the dind socket is
+    # owned by root. Override to root for tests so curl can reach the socket.
+    user: root
+    depends_on:
+      dind:
+        condition: service_healthy
+      netbox:
+        condition: service_healthy
+    volumes:
+      # Shares dind's /var/run, so /var/run/docker.sock is the dind daemon.
+      - docker-run:/var/run
+      # Seed (read-only) the netbox config; the entrypoint copies it to a
+      # writable /data/config.js so the agent can rewrite it (it persists
+      # config after netbox interactions; a :ro mount there would EROFS).
+      - ./fixtures/config.netbox.js:/seed/config.js:ro
+    entrypoint: ["sh", "-c", "cp /seed/config.js /data/config.js && exec ./entrypoint.sh"]
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "-u", "admin:saashup", "http://localhost:1880/"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+      start_period: 20s
+
+  # Runs the hurl suite. run.sh overrides the command to inject the per-version
+  # docker_version variable; the default below mirrors it for a manual
+  # `docker compose run --rm tester`.
+  tester:
+    image: ghcr.io/orange-opensource/hurl:latest
+    profiles: ["test"]
+    depends_on:
+      agent:
+        condition: service_healthy
+      netbox:
+        condition: service_healthy
+    volumes:
+      - ../hurl:/tests:ro
+    entrypoint: ["hurl"]
+    command:
+      - "--test"
+      - "--color"
+      - "--variable"
+      - "host=http://agent:1880"
+      - "--variable"
+      - "netbox=http://netbox:8080"
+      - "-u"
+      - "admin:saashup"
+      - "/tests/tests.hurl"
+      - "/tests/read.hurl"
+      - "/tests/lifecycle.hurl"
+      - "/tests/netbox.hurl"
+
+volumes:
+  docker-run:

tests/compat/fixtures/config.netbox.js

@@ -0,0 +1 @@
+{"netbox_url":"http://netbox:8080","netbox_token":"compat-test-token","ui":1,"id":"1","endpoint":"http://netbox:8080","flows":"0"}

tests/compat/fixtures/wiremock/mappings/catch-all.json

@@ -0,0 +1,13 @@
+{
+  "mappings": [
+    {
+      "priority": 10,
+      "request": { "method": "ANY", "urlPattern": ".*" },
+      "response": {
+        "status": 200,
+        "headers": { "Content-Type": "application/json" },
+        "jsonBody": { "results": [], "count": 0, "id": 1 }
+      }
+    }
+  ]
+}

tests/compat/run.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+#
+# Run the compatibility suite against the agent for one or more dockerd versions.
+#
+#   ./run.sh                 # test every version listed in versions.txt
+#   ./run.sh 29.5.2          # test a single version
+#   ./run.sh 29.5.2 28.3.1   # test a specific set of versions
+#
+# For each version it stands up dind(<version>) + a netbox mock + the agent,
+# then runs:
+#   1. the hurl suite (read/version/lifecycle + the original smoke tests)
+#   2. the websocket-exec test (not expressible in hurl)
+#   3. the netbox-contract test, against a freshly restarted agent so the
+#      agent's own config-persistence during step 1 can't interfere
+# ...before tearing everything down.
+#
+# Exits non-zero if any version fails.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+COMPOSE=(docker compose -p nda-compat)
+
+# Main suite (run together against the same agent).
+HURL_MAIN=(
+  /tests/tests.hurl
+  /tests/read.hurl
+  /tests/version.hurl
+  /tests/lifecycle.hurl
+)
+# Agent->netbox contract: run in isolation (see below).
+HURL_NETBOX=(/tests/netbox.hurl)
+
+if [ "$#" -gt 0 ]; then
+  versions=("$@")
+else
+  mapfile -t versions < <(grep -vE '^[[:space:]]*(#|$)' versions.txt)
+fi
+
+if [ "${#versions[@]}" -eq 0 ]; then
+  echo "No versions to test (check versions.txt)." >&2
+  exit 1
+fi
+
+declare -A results
+overall=0
+
+cleanup() { DOCKER_VERSION="${1:-x}" "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true; }
+
+wait_healthy() { # $1 = container name
+  for _ in $(seq 1 45); do
+    [ "$("${COMPOSE[@]}" ps -q "$1" | xargs -r docker inspect -f '{{.State.Health.Status}}' 2>/dev/null)" = "healthy" ] && return 0
+    sleep 2
+  done
+  return 1
+}
+
+hurl() { # remaining args: hurl files
+  "${COMPOSE[@]}" run --rm tester \
+    --test --color \
+    --variable host=http://agent:1880 \
+    --variable netbox=http://netbox:8080 \
+    --variable docker_version="$DOCKER_VERSION" \
+    -u admin:saashup "$@"
+}
+
+for v in "${versions[@]}"; do
+  echo "==================================================================="
+  echo "=== Testing agent against dockerd $v"
+  echo "==================================================================="
+  export DOCKER_VERSION="$v"
+  cleanup "$v"
+
+  if ! "${COMPOSE[@]}" up -d --build dind netbox agent; then
+    results["$v"]="SETUP-FAIL"; overall=1
+    "${COMPOSE[@]}" logs || true
+    cleanup "$v"
+    continue
+  fi
+
+  step_fail=0
+
+  # --- 1. hurl main suite ---------------------------------------------------
+  hurl "${HURL_MAIN[@]}" || { echo "!!! hurl suite failed for $v"; step_fail=1; }
+
+  # --- 2. websocket exec (F): not expressible in hurl -----------------------
+  # Self-contained: it pulls nginx:alpine through the agent itself.
+  "${COMPOSE[@]}" exec -T agent node --input-type=module - < ws-exec-test.mjs \
+    || { echo "!!! websocket-exec test failed for $v"; step_fail=1; }
+
+  # --- 3. netbox contract (E), isolated -------------------------------------
+  # The agent rewrites /data/config.js from netbox responses during the suite
+  # above, which can clear netbox_url. Restarting re-seeds a clean config (the
+  # entrypoint copies it from /seed), giving this test a known-good agent.
+  "${COMPOSE[@]}" restart agent >/dev/null 2>&1 || true
+  if wait_healthy agent; then
+    hurl "${HURL_NETBOX[@]}" || { echo "!!! netbox-contract test failed for $v"; step_fail=1; }
+  else
+    echo "!!! agent did not become healthy after restart for $v"; step_fail=1
+  fi
+
+  if [ "$step_fail" -eq 0 ]; then
+    results["$v"]="PASS"
+  else
+    results["$v"]="FAIL"; overall=1
+    echo "--- agent logs ($v) ---"; "${COMPOSE[@]}" logs agent | tail -40 || true
+  fi
+
+  cleanup "$v"
+done
+
+echo
+echo "===== Compatibility summary ====="
+for v in "${versions[@]}"; do
+  printf '  dockerd %-12s %s\n' "$v" "${results[$v]:-UNKNOWN}"
+done
+
+exit "$overall"

tests/compat/versions.txt

@@ -0,0 +1,8 @@
+# Supported dockerd versions, one per line.
+# Each version is tested by spinning up `docker:<version>-dind` and running
+# the full hurl suite against the agent connected to that daemon.
+# Lines starting with `#` and blank lines are ignored.
+# To support a new version: add it here and open a PR.
+29.5.2
+28.5.2
+27.5.1