From bd40854478b09932fb6ff3d2820bf2cecfab9f86 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 30 Jun 2026 15:17:07 +0200 Subject: [PATCH 1/8] Add forge Keystone deploy script and ControlPlane CR Stand up a kind cluster with a Keystone-only forge ControlPlane and emit an ephemeral clouds.yaml for the admin cloud. forge is pinned to a known-good commit and its own installer provides the pinned kind/kubectl toolchain. create.py indexes its role cache directly, so it aborts on a bare Keystone that lacks load-balancer_member (Octavia normally provisions it). Pre-create the roles create.py expects, keeping create.py itself untouched. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- test/integration/controlplane.yaml | 17 ++++ test/integration/deploy_keystone.sh | 136 ++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 test/integration/controlplane.yaml create mode 100755 test/integration/deploy_keystone.sh diff --git a/test/integration/controlplane.yaml b/test/integration/controlplane.yaml new file mode 100644 index 0000000..31b762a --- /dev/null +++ b/test/integration/controlplane.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: c5c3.io/v1alpha1 +kind: ControlPlane +metadata: + name: controlplane + namespace: openstack +spec: + openStackRelease: "2025.2" + services: + keystone: + replicas: 1 + publicEndpoint: https://keystone.127-0-0-1.nip.io:8443/v3 + gateway: + parentRef: + name: openstack-gw + hostname: keystone.127-0-0-1.nip.io + path: / diff --git a/test/integration/deploy_keystone.sh b/test/integration/deploy_keystone.sh new file mode 100755 index 0000000..f05601e --- /dev/null +++ b/test/integration/deploy_keystone.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Deploy a Keystone-only forge ControlPlane on a kind cluster and emit a +# clouds.yaml the project-manager can authenticate against. +# +# Safe to run standalone (see `make integration-up`). A kind cluster of the +# same name that already exists is reused; this script does not delete it. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Pin forge to a known-good commit; forge is a fast-moving prototyping repo. +FORGE_REF="${FORGE_REF:-35ef73ae48d6814270af1674370d60ae410efdc7}" +FORGE_DIR="${FORGE_DIR:-${TMPDIR:-/tmp}/opm-forge}" + +NAMESPACE="${NAMESPACE:-openstack}" +KIND_HOST_PORT="${KIND_HOST_PORT:-8443}" +CLOUDS_FILE="${OS_CLIENT_CONFIG_FILE:-${HERE}/clouds.yaml}" + +# Refuse to clobber an existing file outside our dedicated in-tree path. An +# operator who exports OS_CLIENT_CONFIG_FILE to point at their real +# ~/.config/openstack/clouds.yaml would otherwise have it truncated and +# replaced with throwaway test admin credentials by the write in step 3. +if [[ -e "${CLOUDS_FILE}" && "${CLOUDS_FILE}" != "${HERE}/clouds.yaml" ]]; then + echo "Refusing to overwrite existing ${CLOUDS_FILE}." >&2 + echo "Unset OS_CLIENT_CONFIG_FILE or point it at a throwaway path." >&2 + exit 1 +fi + +# 1. forge toolchain + infra ------------------------------------------------- +# forge's own installer pins kind/kubectl into ${HOME}/.local/bin. +if [[ ! -d "${FORGE_DIR}/.git" ]]; then + git clone https://github.com/c5c3/forge.git "${FORGE_DIR}" +else + git -C "${FORGE_DIR}" fetch --quiet origin +fi +git -C "${FORGE_DIR}" checkout --quiet "${FORGE_REF}" + +make -C "${FORGE_DIR}" install-test-deps +export PATH="${HOME}/.local/bin:${PATH}" + +# Pin the optional forge stacks off: this Keystone-only integration test needs +# neither Chaos Mesh nor the Prometheus stack, and they only add provisioning +# time and load. These are forge's defaults today; set them explicitly so a +# future forge default flip cannot silently pull them in. +KIND_HOST_PORT="${KIND_HOST_PORT}" WITH_CONTROLPLANE=true \ + WITH_CHAOS_MESH=false WITH_PROMETHEUS=false \ + make -C "${FORGE_DIR}" deploy-infra + +# 2. Keystone-only ControlPlane --------------------------------------------- +kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 \ + || kubectl create namespace "${NAMESPACE}" +kubectl apply -f "${HERE}/controlplane.yaml" + +# Wait for the ControlPlane's aggregate Ready condition. A bare +# `kubectl wait --for=condition=Ready --timeout=15m` is silent for the whole +# window, so a stuck sub-condition (the operator reconciles them in order: +# InfrastructureReady -> KeystoneReady -> KORCReady -> AdminCredentialReady -> +# CatalogReady) is indistinguishable from a dead hang in the CI log. Poll +# instead and print the first not-yet-True condition each tick, so the log +# shows which stage is pending; on timeout dump the full CR before failing +# (the verbose pod/event dump is left to run.sh's diagnostics trap). +cp_timeout="${CONTROLPLANE_TIMEOUT:-900}" +cp_deadline=$(($(date +%s) + cp_timeout)) +while true; do + if [[ "$(kubectl get controlplane/controlplane -n "${NAMESPACE}" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' \ + 2>/dev/null)" == "True" ]]; then + echo "ControlPlane/controlplane is Ready." + break + fi + pending="$(kubectl get controlplane/controlplane -n "${NAMESPACE}" -o json 2>/dev/null \ + | jq -r 'first(.status.conditions[]? | select(.type != "Ready" and .status != "True") + | "\(.type)=\(.status) (\(.reason)): \(.message)") // "status not reported yet"')" + echo " ControlPlane/controlplane not Ready yet: ${pending}" + if [[ "$(date +%s)" -ge "${cp_deadline}" ]]; then + echo "ERROR: ControlPlane/controlplane not Ready after ${cp_timeout}s." >&2 + kubectl get controlplane/controlplane -n "${NAMESPACE}" -o yaml || true + exit 1 + fi + sleep 15 +done + +# 3. clouds.yaml from the operator-projected admin credentials -------------- +admin_password="$(kubectl get secret controlplane-keystone-admin-credentials \ + -n "${NAMESPACE}" -o jsonpath='{.data.password}' | base64 -d)" + +# Serialize with a YAML library rather than a heredoc: the minted admin +# password is arbitrary bytes, and a `"` or `\` interpolated into a quoted YAML +# scalar would corrupt the file or silently change the value via escape +# processing. +umask 077 +ADMIN_PASSWORD="${admin_password}" KIND_HOST_PORT="${KIND_HOST_PORT}" \ + CLOUDS_FILE="${CLOUDS_FILE}" python3 - <<'PY' +import os + +import yaml + +data = { + "clouds": { + "admin": { + "auth": { + "auth_url": f"https://keystone.127-0-0-1.nip.io:{os.environ['KIND_HOST_PORT']}/v3", + "username": "admin", + "password": os.environ["ADMIN_PASSWORD"], + "project_name": "admin", + "user_domain_name": "Default", + "project_domain_name": "Default", + }, + "identity_api_version": 3, + "verify": False, + } + } +} +with open(os.environ["CLOUDS_FILE"], "w") as f: + yaml.safe_dump(data, f, default_flow_style=False) +PY +echo "Wrote clouds.yaml to ${CLOUDS_FILE}" + +# 4. Ensure the roles create.py expects exist on the bare Keystone ---------- +# create.py indexes its role cache directly (CACHE_ROLES[role_name]) when it +# assigns DEFAULT_ROLES = member, load-balancer_member, so a missing role +# aborts the run. A bare forge Keystone ships member but not +# load-balancer_member (Octavia provisions that one), so pre-create it here. +# Idempotent. +export OS_CLIENT_CONFIG_FILE="${CLOUDS_FILE}" +python3 - <<'PY' +import openstack + +cloud = openstack.connect(cloud="admin") +for name in ("member", "load-balancer_member"): + if cloud.identity.find_role(name) is None: + cloud.identity.create_role(name=name) + print(f"Created role: {name}") + else: + print(f"Role already present: {name}") +PY From 07785efc79f83e6af994e44a794923cc9860ff99 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 30 Jun 2026 15:19:52 +0200 Subject: [PATCH 2/8] Add integration run.sh orchestration and SDK verifier run.sh deploys Keystone, runs the real `tox -e create` path as admin with --nodomain-name-prefix so the project is named exactly opm-integration-test, then asserts it with verify.py. A trap tears down the kind cluster on every exit path but only when this run created it, dumping diagnostics first on failure. verify.py is a standalone openstacksdk script -- not named test_*.py -- so `tox -e test` does not collect it. tox passes OS_CLIENT_CONFIG_FILE through to the create env, and the generated clouds.yaml is gitignored. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- .gitignore | 1 + test/integration/run.sh | 97 ++++++++++++++++++++++++++++++++++++++ test/integration/verify.py | 38 +++++++++++++++ tox.ini | 1 + 4 files changed, 137 insertions(+) create mode 100755 test/integration/run.sh create mode 100644 test/integration/verify.py diff --git a/.gitignore b/.gitignore index bdc65d1..f760d59 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ clouds.yml secure.yml settings.toml +test/integration/clouds.yaml __pycache__ diff --git a/test/integration/run.sh b/test/integration/run.sh new file mode 100755 index 0000000..e6070a1 --- /dev/null +++ b/test/integration/run.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Orchestrate the Keystone integration test: deploy Keystone on kind via forge, +# create one project as admin through the real `tox -e create` path, and assert +# it via the OpenStack SDK. +# +# The kind cluster is always torn down on exit (success or failure), but only +# when this run actually created it -- a pre-existing debug cluster of the same +# name is reused and left in place. On failure a diagnostics dump is emitted +# before teardown. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${HERE}/../.." && pwd)" + +CLUSTER_NAME="${CLUSTER_NAME:-forge}" +NAMESPACE="${NAMESPACE:-openstack}" +PROJECT_NAME="${PROJECT_NAME:-opm-integration-test}" + +export PATH="${HOME}/.local/bin:${PATH}" +export OS_CLIENT_CONFIG_FILE="${HERE}/clouds.yaml" + +# Record cluster ownership before we deploy so teardown only removes a cluster +# this run created. +CREATED_CLUSTER=0 +if ! kind get clusters 2>/dev/null | grep -Fqx "${CLUSTER_NAME}"; then + CREATED_CLUSTER=1 +fi + +dump_diagnostics() { + echo "=== Diagnostics for kind cluster ${CLUSTER_NAME} ===" + kubectl get nodes -o wide || true + kubectl get pods -A -o wide || true + kubectl get controlplane -n "${NAMESPACE}" -o yaml || true + kubectl get events -A --sort-by=.lastTimestamp || true + + # Describe and dump logs for every Pod in the namespace, current AND + # previous instance. A CrashLoopBackOff Pod (e.g. the projected + # controlplane-keystone) keeps the failure only in its *previous* container + # log, and `describe` carries the Last State (OOMKilled, exit code). We + # iterate every Pod instead of a label selector on purpose: the c5c3- + # projected Keystone Pods do not carry a stable app label we can rely on, so + # an `-l application=keystone` selector silently matched nothing. + echo "=== Pod descriptions and logs in namespace ${NAMESPACE} ===" + for pod in $(kubectl get pods -n "${NAMESPACE}" \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do + echo "--- describe pod/${pod} ---" + kubectl describe pod/"${pod}" -n "${NAMESPACE}" || true + echo "--- logs pod/${pod} (current) ---" + kubectl logs pod/"${pod}" -n "${NAMESPACE}" \ + --all-containers --tail=200 || true + echo "--- logs pod/${pod} (previous) ---" + kubectl logs pod/"${pod}" -n "${NAMESPACE}" \ + --all-containers --previous --tail=200 || true + done +} + +cleanup() { + rc=$? + if [[ "${rc}" -ne 0 ]]; then + dump_diagnostics + fi + if [[ "${CREATED_CLUSTER}" -eq 1 ]]; then + echo "Deleting kind cluster ${CLUSTER_NAME}" + kind delete cluster --name "${CLUSTER_NAME}" || true + else + echo "Leaving pre-existing kind cluster ${CLUSTER_NAME} in place" + fi + rm -f "${OS_CLIENT_CONFIG_FILE}" + exit "${rc}" +} +trap cleanup EXIT + +"${HERE}/deploy_keystone.sh" + +cd "${ROOT}" +tox -e create -- --cloud admin --domain default --name "${PROJECT_NAME}" \ + --nodomain-name-prefix --quota-class basic --nocreate-user + +python "${HERE}/verify.py" + +# Keep reruns idempotent by removing the project unless asked to keep it. The +# EXIT trap destroys the whole cluster anyway, so this is best-effort cleanup. +if [[ "${KEEP_PROJECT:-0}" != "1" ]]; then + python3 - "${PROJECT_NAME}" <<'PY' || true +import sys + +import openstack + +name = sys.argv[1] +cloud = openstack.connect(cloud="admin") +domain = cloud.identity.find_domain("default") +project = cloud.identity.find_project(name, domain_id=domain.id) +if project is not None: + cloud.identity.delete_project(project.id) + print(f"Deleted project: {name}") +PY +fi diff --git a/test/integration/verify.py b/test/integration/verify.py new file mode 100644 index 0000000..079484c --- /dev/null +++ b/test/integration/verify.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Standalone OpenStack SDK assertions for the integration test. + +This script is intentionally NOT a unittest/pytest test and is deliberately +not named ``test_*.py`` so that ``tox -e test`` +(``python -m unittest discover ./test``) never collects it. It is invoked +directly by ``run.sh`` after the project has been created and exits non-zero +on any failed assertion. +""" + +import sys + +import openstack + +PROJECT_NAME = "opm-integration-test" +DOMAIN_NAME = "default" + + +def main() -> int: + """Assert the integration project and its group exist via the SDK.""" + cloud = openstack.connect(cloud="admin") + + domain = cloud.identity.find_domain(DOMAIN_NAME) + assert domain is not None, f"domain {DOMAIN_NAME!r} not found" + + project = cloud.identity.find_project(PROJECT_NAME, domain_id=domain.id) + assert project is not None, f"project {PROJECT_NAME!r} not found" + + group = cloud.identity.find_group(PROJECT_NAME, domain_id=domain.id) + assert group is not None, f"group {PROJECT_NAME!r} not found" + + print(f"OK: project {PROJECT_NAME!r} and group exist in domain {DOMAIN_NAME!r}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tox.ini b/tox.ini index b7e7535..f0cf5e0 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ envlist = manage [testenv] whitelist_externals = echo list_dependencies_command = echo +passenv = OS_CLIENT_CONFIG_FILE deps = -rrequirements.txt From 72829805346aa3c7b00558ab6e06e9227455be76 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 30 Jun 2026 15:20:33 +0200 Subject: [PATCH 3/8] Add Makefile integration targets integration runs the full provision/create/verify/teardown cycle; integration-up deploys Keystone and leaves the cluster running for local debugging; integration-down deletes the kind cluster. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- Makefile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c362f46 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +CLUSTER_NAME ?= forge +export CLUSTER_NAME + +.PHONY: integration integration-up integration-down + +# Full cycle: provision kind + Keystone, create a project, verify it, then +# tear the cluster down (always, including on failure). +integration: + test/integration/run.sh + +# Deploy Keystone and leave the cluster running for local debugging. +integration-up: + test/integration/deploy_keystone.sh + +# Tear down the cluster created by integration-up. +integration-down: + kind delete cluster --name $(CLUSTER_NAME) From 27f881d64d8c0aa723cab211b37b908cb81653b0 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 30 Jun 2026 15:21:22 +0200 Subject: [PATCH 4/8] Add Ansible pre/test playbooks for integration job pre-integration.yml prepares an IPv6-only Zuul node: the ensure-pip / ensure-pipenv / ensure-docker roles, the libldap/libsasl build deps the create env needs, and the accept_ra=2 sysctl fix that keeps RAs honoured once kind enables IPv6 forwarding. The pinned kind/kubectl toolchain comes from forge's install-test-deps, invoked by deploy_keystone.sh. test-integration.yml installs tox and openstacksdk into a venv and runs make integration from the checkout. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- playbooks/pre-integration.yml | 59 ++++++++++++++++++++++++++++++++++ playbooks/test-integration.yml | 25 ++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 playbooks/pre-integration.yml create mode 100644 playbooks/test-integration.yml diff --git a/playbooks/pre-integration.yml b/playbooks/pre-integration.yml new file mode 100644 index 0000000..23f3549 --- /dev/null +++ b/playbooks/pre-integration.yml @@ -0,0 +1,59 @@ +--- +- name: Pre integration play + hosts: all + + roles: + - ensure-pip + - ensure-pipenv + - ensure-docker + + tasks: + - name: Ensure required APT packages are installed + ansible.builtin.apt: + name: + - jq + - libldap2-dev + - libsasl2-dev + update_cache: true + become: true # run as root + + - name: Install mikefarah/yq + # forge's deploy-infra.sh needs mikefarah/yq (Go) for its `select(...)`, + # `strenv(...)` and `-i` filters. The Ubuntu `yq` APT package is the + # incompatible python-yq (a jq wrapper), so fetch the upstream binary. + ansible.builtin.get_url: + url: "https://github.com/mikefarah/yq/releases/download/{{ yq_version }}/yq_linux_amd64" + dest: /usr/local/bin/yq + owner: root + group: root + mode: "0755" + become: true + vars: + yq_version: v4.53.3 + + - name: Keep router advertisements honoured while IPv6 forwarding is on + # The OSISM Zuul nodes are IPv6-only and learn their default route via + # SLAAC/RA. When kind makes Docker create a dual-stack network it enables + # net.ipv6.conf.all.forwarding, after which the kernel (default + # accept_ra=1) stops honouring RAs and the node drops off the network + # mid-run. accept_ra=2 keeps RAs honoured while forwarding is on. The + # all/default keys do not retroactively apply to an already-up interface, + # so the live default interface is set explicitly too -- that per-interface + # key is what actually preserves the running node's default route. + ansible.builtin.copy: + dest: /etc/sysctl.d/99-opm-integration-accept-ra.conf + content: | + net.ipv6.conf.all.accept_ra = 2 + net.ipv6.conf.default.accept_ra = 2 + {% if ansible_default_ipv6.interface is defined %} + net.ipv6.conf.{{ ansible_default_ipv6.interface }}.accept_ra = 2 + {% endif %} + owner: root + group: root + mode: "0644" + become: true + + - name: Apply the sysctl settings + ansible.builtin.command: sysctl --system + become: true + changed_when: false diff --git a/playbooks/test-integration.yml b/playbooks/test-integration.yml new file mode 100644 index 0000000..7982ecc --- /dev/null +++ b/playbooks/test-integration.yml @@ -0,0 +1,25 @@ +--- +- name: Run integration play + hosts: all + + vars: + python_venv_dir: /tmp/venv + + tasks: + - name: Install Python dependencies into the venv + ansible.builtin.pip: + name: + - tox + - openstacksdk + virtualenv: "{{ python_venv_dir }}" + virtualenv_command: python3 -m venv + + - name: Run the integration test + ansible.builtin.shell: | + set -e -o pipefail -x + export PATH="{{ python_venv_dir }}/bin:${HOME}/.local/bin:${PATH}" + make integration + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + changed_when: true From 080f015fc13eed6d58d03c409049610db863046e Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 30 Jun 2026 15:21:58 +0200 Subject: [PATCH 5/8] Add Zuul integration job to the check pipeline openstack-project-manager-integration runs on ubuntu-noble with a 2400s timeout. It is added to the check pipeline so the kind+Keystone job runs on every change. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- .zuul.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index c1a75a5..b9d99b5 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -11,6 +11,18 @@ vars: tox_envlist: test +- job: + name: openstack-project-manager-integration + # The full forge ControlPlane stack (cert-manager, MariaDB/Memcached/ + # external-secrets/keystone/c5c3 operators, k-orc, OpenBao, flux) saturates + # the default node's CPU before the Envoy data-plane Pod can schedule, so + # Gateway/openstack-gw never gets a backing Pod and stays NoResources. The + # larger flavour has the CPU headroom the single-node kind cluster needs. + nodeset: ubuntu-noble-large + pre-run: playbooks/pre-integration.yml + run: playbooks/test-integration.yml + timeout: 2400 + - project: merge-mode: squash-merge default-branch: main @@ -19,6 +31,7 @@ - flake8 - openstack-project-manager-mypy - openstack-project-manager-tox + - openstack-project-manager-integration - yamllint - python-black From 7fa4baf527ff37383b6a906a35a7e36c5809acaa Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Tue, 30 Jun 2026 15:22:36 +0200 Subject: [PATCH 6/8] Document the integration test harness Cover prerequisites, the make targets, the environment overrides, what verify.py asserts, and the ephemeral-credentials handling. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- test/integration/README.md | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 test/integration/README.md diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 0000000..3f02b2d --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,74 @@ +# Integration test + +This integration test exercises the real project-creation path of +`openstack-project-manager` against a **live Keystone** instead of mocks. +Keystone is deployed locally on a [kind](https://kind.sigs.k8s.io/) cluster via +[forge](https://github.com/c5c3/forge) (its ControlPlane Quick Start), and the +test then creates one project using the admin account and verifies it through +the OpenStack SDK. + +It is deliberately a scaffold: one happy-path test. The unit suite under +`test/unit/` keeps everything mocked; this harness is the first thing that +talks to a real `openstack.connect()` / `keystone.projects.update()` path. + +## Prerequisites + +- A Docker-capable host with the Docker daemon running. +- Outbound network access (to clone forge, pull operator images, and resolve + `keystone.127-0-0-1.nip.io` via [nip.io](https://nip.io/) to `127.0.0.1`). +- For local runs, `openstacksdk` and `tox` must be importable in the active + environment (the CI job installs them into a throwaway venv). `kind` and + `kubectl` are installed by forge's own `make install-test-deps` into + `~/.local/bin`; the scripts prepend that directory to `PATH`. + +## Usage + +```bash +# Full cycle: provision kind + Keystone, create a project, verify it, and tear +# the cluster down on every exit path (including failure). +make integration + +# Deploy Keystone and leave the cluster running for local debugging. +make integration-up + +# Tear down the cluster left running by `make integration-up`. +make integration-down +``` + +`make integration` only deletes a cluster it created itself. If a kind cluster +named `forge` already exists (for example one you started with +`make integration-up`), it is reused and left in place on exit. + +## Environment overrides + +| Variable | Default | Purpose | +|---|---|---| +| `FORGE_REF` | `main` | forge git ref to check out (tip is fetched on every run; not pinned). | +| `FORGE_DIR` | `${TMPDIR:-/tmp}/opm-forge` | Where forge is cloned. | +| `CLUSTER_NAME` | `forge` | kind cluster name. | +| `KIND_HOST_PORT` | `8443` | Host port the Envoy Gateway (and Keystone) is exposed on. | +| `NAMESPACE` | `openstack` | Namespace the ControlPlane CR is applied to. | +| `OS_CLIENT_CONFIG_FILE` | `test/integration/clouds.yaml` | Generated clouds.yaml path. | +| `KEEP_PROJECT` | `0` | Set to `1` to keep the created project after `verify.py` (the cluster is torn down regardless). | + +## What is asserted + +After `tox -e create` creates the project as admin, `verify.py` connects with +the OpenStack SDK and asserts that: + +- the `default` domain resolves, +- the `opm-integration-test` project exists in it, and +- the per-project group of the same name exists. + +`verify.py` exits non-zero on any failed assertion. It is a standalone script, +not a `unittest`/`pytest` test, and is not named `test_*.py`, so the fast unit +gate (`tox -e test`) never collects it. + +## Credentials + +No real credentials are committed. `deploy_keystone.sh` reads the admin +password from the operator-projected +`controlplane-keystone-admin-credentials` Secret and writes an ephemeral +`clouds.yaml` (gitignored) with `verify: false`, because the Envoy Gateway +serves a self-signed cert. `run.sh` removes the generated `clouds.yaml` on +exit. From 3ffd3c110bdb1c276458a494748dbf186cf30182 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Fri, 3 Jul 2026 20:05:09 +0200 Subject: [PATCH 7/8] Fit the integration ControlPlane onto single-node kind The Keystone integration job OOM-killed its Keystone pod (exit 137, CrashLoopBackOff). Two causes: the CR left infrastructure.{database,cache}.replicas unset, so the CRD default of 3 brought up a 3-node Galera MariaDB plus three Memcached pods; and Docker Desktop's containerd ships LimitNOFILE=infinity, so uWSGI pre-allocated multiple GiB per worker from the inherited ~1e9 fd limit and died within seconds of spawning (more memory does not help). Pin database.replicas and cache.replicas to 1 -- a single-instance non-Galera MariaDB and one Memcached pod -- and bump FORGE_REF to a828d6a, whose deploy-infra caps each kind node's containerd RLIMIT_NOFILE to 1048576 before any workload is scheduled. Together they let Keystone come up at its default 512Mi/2-worker footprint. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Christian Berendt --- test/integration/controlplane.yaml | 14 ++++++++++++++ test/integration/deploy_keystone.sh | 21 ++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/test/integration/controlplane.yaml b/test/integration/controlplane.yaml index 31b762a..5d6c882 100644 --- a/test/integration/controlplane.yaml +++ b/test/integration/controlplane.yaml @@ -6,6 +6,20 @@ metadata: namespace: openstack spec: openStackRelease: "2025.2" + # Pin the operator-projected backing services to a single instance for the + # single-node kind cluster. The CRD default for both is 3 (a 3-node Galera + # MariaDB plus three Memcached pods, matching the production baseline), which + # OOM-kills a single-node kind. database.replicas: 1 yields a single-instance + # non-Galera MariaDB (the c5c3-operator derives Galera from replicas > 1); + # cache.replicas: 1 a single Memcached pod. Only replicas is set here; the + # operator defaulting webhook fills the clusterRef/database/secretRef/backend + # references. database.replicas is immutable after creation, but CI recreates + # the cluster on every run so that never bites here. + infrastructure: + database: + replicas: 1 + cache: + replicas: 1 services: keystone: replicas: 1 diff --git a/test/integration/deploy_keystone.sh b/test/integration/deploy_keystone.sh index f05601e..d154b48 100755 --- a/test/integration/deploy_keystone.sh +++ b/test/integration/deploy_keystone.sh @@ -8,8 +8,17 @@ set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Pin forge to a known-good commit; forge is a fast-moving prototyping repo. -FORGE_REF="${FORGE_REF:-35ef73ae48d6814270af1674370d60ae410efdc7}" +# Track the latest forge main; forge is a fast-moving prototyping repo and we +# deliberately do not pin to a commit. main caps each kind node's containerd +# RLIMIT_NOFILE (default 1048576) inside deploy-infra before any workload is +# scheduled. Docker Desktop's containerd ships LimitNOFILE=infinity, so pods +# otherwise inherit an ~1e9 open-file limit and uWSGI (the Keystone API server) +# pre-allocates multiple GiB per worker and is OOM-killed within seconds of +# spawning; raising the memory limit does not help. The cap lets Keystone come +# up at its default 512Mi/2-worker footprint. main also carries the single-node +# backing-service defaults our controlplane.yaml relies on +# (infrastructure.{database,cache}.replicas). +FORGE_REF="${FORGE_REF:-main}" FORGE_DIR="${FORGE_DIR:-${TMPDIR:-/tmp}/opm-forge}" NAMESPACE="${NAMESPACE:-openstack}" @@ -30,10 +39,12 @@ fi # forge's own installer pins kind/kubectl into ${HOME}/.local/bin. if [[ ! -d "${FORGE_DIR}/.git" ]]; then git clone https://github.com/c5c3/forge.git "${FORGE_DIR}" -else - git -C "${FORGE_DIR}" fetch --quiet origin fi -git -C "${FORGE_DIR}" checkout --quiet "${FORGE_REF}" +# Fetch and check out the tip of ${FORGE_REF} (default: main) as a detached +# HEAD, so a reused clone always advances to the latest commit instead of +# staying on a stale local branch. +git -C "${FORGE_DIR}" fetch --quiet origin "${FORGE_REF}" +git -C "${FORGE_DIR}" checkout --quiet FETCH_HEAD make -C "${FORGE_DIR}" install-test-deps export PATH="${HOME}/.local/bin:${PATH}" From d56c61c23de4694e2ef755c46846af824eabcc0b Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 5 Jul 2026 12:20:26 +0200 Subject: [PATCH 8/8] Align the integration ControlPlane with current forge main forge made two changes the harness has to follow: - database.storageSize is now a spec field (forge 1468088); pin it to 512Mi like forge's kind Quick Start CR so the projected MariaDB stops requesting the 100Gi CRD default that a CI run never fills. - Managed-mode ControlPlanes now default to Dynamic (engine-issued) DB credentials and the static per-ControlPlane seed was retired (forge 49fb87c..e3b90b5). The engine path only works once setup-database-tenant.sh has onboarded the per-tenant OpenBao database-engine connection and role, and deploy-infra runs that hook only when it applies the bundled CR itself (WITH_CONTROLPLANE_CR=true). This harness applies its own CR, so deploy_keystone.sh now mirrors the hook: wait for the projected MariaDB to be Ready, then run the idempotent tenant script with the OpenBao root token. Without it, DBCredentialsReady never turns True and the Ready wait times out. Assisted-by: Claude:claude-fable-5 Signed-off-by: Christian Berendt --- test/integration/README.md | 7 ++++++ test/integration/controlplane.yaml | 19 +++++++++++---- test/integration/deploy_keystone.sh | 37 +++++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/test/integration/README.md b/test/integration/README.md index 3f02b2d..adb26ab 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -72,3 +72,10 @@ password from the operator-projected `clouds.yaml` (gitignored) with `verify: false`, because the Envoy Gateway serves a self-signed cert. `run.sh` removes the generated `clouds.yaml` on exit. + +Keystone's own database credentials are engine-issued (forge's Dynamic +credentials mode, the managed-mode default): after applying the ControlPlane +CR, `deploy_keystone.sh` waits for the projected MariaDB and runs forge's +`setup-database-tenant.sh` to onboard the OpenBao database-engine tenant, the +same step forge's `deploy-infra` performs when it applies its bundled CR. No +long-lived DB password is seeded anywhere. diff --git a/test/integration/controlplane.yaml b/test/integration/controlplane.yaml index 5d6c882..6a0736b 100644 --- a/test/integration/controlplane.yaml +++ b/test/integration/controlplane.yaml @@ -11,13 +11,24 @@ spec: # MariaDB plus three Memcached pods, matching the production baseline), which # OOM-kills a single-node kind. database.replicas: 1 yields a single-instance # non-Galera MariaDB (the c5c3-operator derives Galera from replicas > 1); - # cache.replicas: 1 a single Memcached pod. Only replicas is set here; the - # operator defaulting webhook fills the clusterRef/database/secretRef/backend - # references. database.replicas is immutable after creation, but CI recreates - # the cluster on every run so that never bites here. + # cache.replicas: 1 a single Memcached pod. database.storageSize is pinned to + # 512Mi (mirroring forge's kind Quick Start CR) so the projected MariaDB + # requests a test-sized volume instead of the 100Gi CRD default (the + # production baseline) that a CI run never fills. Only these fields are set + # here; the operator defaulting webhook fills the clusterRef/database/ + # secretRef/backend references. database.replicas and database.storageSize + # are immutable after creation, but CI recreates the cluster on every run so + # that never bites here. + # + # database.credentialsMode is deliberately left unset: forge defaults managed + # mode to Dynamic (engine-issued short-lived DB credentials from the OpenBao + # database engine; the static per-ControlPlane seed was retired). That mode + # needs the per-tenant database-engine onboarding, which deploy_keystone.sh + # runs after applying this CR. infrastructure: database: replicas: 1 + storageSize: 512Mi cache: replicas: 1 services: diff --git a/test/integration/deploy_keystone.sh b/test/integration/deploy_keystone.sh index d154b48..a7b296a 100755 --- a/test/integration/deploy_keystone.sh +++ b/test/integration/deploy_keystone.sh @@ -62,6 +62,39 @@ kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 \ || kubectl create namespace "${NAMESPACE}" kubectl apply -f "${HERE}/controlplane.yaml" +# 3. OpenBao database-engine tenant onboarding ------------------------------- +# forge defaults a managed-mode ControlPlane to Dynamic (engine-issued) DB +# credentials; the static per-ControlPlane credential seed was retired. The +# engine-issued creds path database/mariadb/creds/keystone- only +# exists once setup-database-tenant.sh has provisioned the per-tenant +# connection and role, and deploy-infra runs that hook only when it applies the +# bundled CR itself (WITH_CONTROLPLANE_CR=true). This harness applies its own +# CR, so mirror deploy-infra's openbao_onboard_database_tenant here: wait for +# the projected MariaDB to be Ready, then run the (idempotent) tenant script +# with the OpenBao root token. Without this, DBCredentialsReady never turns +# True and the Ready wait below times out. +echo "Waiting for MariaDB openstack-db to be projected..." +for _ in $(seq 1 30); do + if kubectl get mariadb/openstack-db -n "${NAMESPACE}" >/dev/null 2>&1; then + break + fi + sleep 10 +done +kubectl wait mariadb/openstack-db -n "${NAMESPACE}" \ + --for=condition=Ready --timeout=600s + +BAO_TOKEN="$(kubectl get secret openbao-init-keys -n openbao-system \ + -o jsonpath='{.data.init-output}' | base64 -d | jq -r '.root_token')" +if [[ -z "${BAO_TOKEN}" || "${BAO_TOKEN}" == "null" ]]; then + echo "ERROR: could not extract the OpenBao root token from" \ + "openbao-system/openbao-init-keys." >&2 + exit 1 +fi +export BAO_TOKEN +bash "${FORGE_DIR}/deploy/openbao/bootstrap/setup-database-tenant.sh" \ + "${NAMESPACE}" controlplane +unset BAO_TOKEN + # Wait for the ControlPlane's aggregate Ready condition. A bare # `kubectl wait --for=condition=Ready --timeout=15m` is silent for the whole # window, so a stuck sub-condition (the operator reconciles them in order: @@ -91,7 +124,7 @@ while true; do sleep 15 done -# 3. clouds.yaml from the operator-projected admin credentials -------------- +# 4. clouds.yaml from the operator-projected admin credentials -------------- admin_password="$(kubectl get secret controlplane-keystone-admin-credentials \ -n "${NAMESPACE}" -o jsonpath='{.data.password}' | base64 -d)" @@ -127,7 +160,7 @@ with open(os.environ["CLOUDS_FILE"], "w") as f: PY echo "Wrote clouds.yaml to ${CLOUDS_FILE}" -# 4. Ensure the roles create.py expects exist on the bare Keystone ---------- +# 5. Ensure the roles create.py expects exist on the bare Keystone ---------- # create.py indexes its role cache directly (CACHE_ROLES[role_name]) when it # assigns DEFAULT_ROLES = member, load-balancer_member, so a missing role # aborts the run. A bare forge Keystone ships member but not