Add Keystone-on-kind integration test (forge)

.gitignore

@@ -2,5 +2,6 @@
 clouds.yml
 secure.yml
 settings.toml
+test/integration/clouds.yaml
 
 __pycache__

.zuul.yaml

@@ -11,6 +11,18 @@
     vars:
       tox_envlist: test
 
+- job:
+    name: openstack-project-manager-integration
+    # The full forge ControlPlane stack (cert-manager, MariaDB/Memcached/
+    # external-secrets/keystone/c5c3 operators, k-orc, OpenBao, flux) saturates
+    # the default node's CPU before the Envoy data-plane Pod can schedule, so
+    # Gateway/openstack-gw never gets a backing Pod and stays NoResources. The
+    # larger flavour has the CPU headroom the single-node kind cluster needs.
+    nodeset: ubuntu-noble-large
+    pre-run: playbooks/pre-integration.yml
+    run: playbooks/test-integration.yml
+    timeout: 2400
+
 - project:
     merge-mode: squash-merge
     default-branch: main
@@ -19,6 +31,7 @@
         - flake8
         - openstack-project-manager-mypy
         - openstack-project-manager-tox
+        - openstack-project-manager-integration
         - yamllint
         - python-black
 

Makefile

@@ -0,0 +1,17 @@
+CLUSTER_NAME ?= forge
+export CLUSTER_NAME
+
+.PHONY: integration integration-up integration-down
+
+# Full cycle: provision kind + Keystone, create a project, verify it, then
+# tear the cluster down (always, including on failure).
+integration:
+	test/integration/run.sh
+
+# Deploy Keystone and leave the cluster running for local debugging.
+integration-up:
+	test/integration/deploy_keystone.sh
+
+# Tear down the cluster created by integration-up.
+integration-down:
+	kind delete cluster --name $(CLUSTER_NAME)

playbooks/pre-integration.yml

@@ -0,0 +1,59 @@
+---
+- name: Pre integration play
+  hosts: all
+
+  roles:
+    - ensure-pip
+    - ensure-pipenv
+    - ensure-docker
+
+  tasks:
+    - name: Ensure required APT packages are installed
+      ansible.builtin.apt:
+        name:
+          - jq
+          - libldap2-dev
+          - libsasl2-dev
+        update_cache: true
+      become: true  # run as root
+
+    - name: Install mikefarah/yq
+      # forge's deploy-infra.sh needs mikefarah/yq (Go) for its `select(...)`,
+      # `strenv(...)` and `-i` filters. The Ubuntu `yq` APT package is the
+      # incompatible python-yq (a jq wrapper), so fetch the upstream binary.
+      ansible.builtin.get_url:
+        url: "https://github.com/mikefarah/yq/releases/download/{{ yq_version }}/yq_linux_amd64"
+        dest: /usr/local/bin/yq
+        owner: root
+        group: root
+        mode: "0755"
+      become: true
+      vars:
+        yq_version: v4.53.3
+
+    - name: Keep router advertisements honoured while IPv6 forwarding is on
+      # The OSISM Zuul nodes are IPv6-only and learn their default route via
+      # SLAAC/RA. When kind makes Docker create a dual-stack network it enables
+      # net.ipv6.conf.all.forwarding, after which the kernel (default
+      # accept_ra=1) stops honouring RAs and the node drops off the network
+      # mid-run. accept_ra=2 keeps RAs honoured while forwarding is on. The
+      # all/default keys do not retroactively apply to an already-up interface,
+      # so the live default interface is set explicitly too -- that per-interface
+      # key is what actually preserves the running node's default route.
+      ansible.builtin.copy:
+        dest: /etc/sysctl.d/99-opm-integration-accept-ra.conf
+        content: |
+          net.ipv6.conf.all.accept_ra = 2
+          net.ipv6.conf.default.accept_ra = 2
+          {% if ansible_default_ipv6.interface is defined %}
+          net.ipv6.conf.{{ ansible_default_ipv6.interface }}.accept_ra = 2
+          {% endif %}
+        owner: root
+        group: root
+        mode: "0644"
+      become: true
+
+    - name: Apply the sysctl settings
+      ansible.builtin.command: sysctl --system
+      become: true
+      changed_when: false

playbooks/test-integration.yml

@@ -0,0 +1,25 @@
+---
+- name: Run integration play
+  hosts: all
+
+  vars:
+    python_venv_dir: /tmp/venv
+
+  tasks:
+    - name: Install Python dependencies into the venv
+      ansible.builtin.pip:
+        name:
+          - tox
+          - openstacksdk
+        virtualenv: "{{ python_venv_dir }}"
+        virtualenv_command: python3 -m venv
+
+    - name: Run the integration test
+      ansible.builtin.shell: |
+        set -e -o pipefail -x
+        export PATH="{{ python_venv_dir }}/bin:${HOME}/.local/bin:${PATH}"
+        make integration
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+        executable: /bin/bash
+      changed_when: true

test/integration/README.md

@@ -0,0 +1,81 @@
+# Integration test
+
+This integration test exercises the real project-creation path of
+`openstack-project-manager` against a **live Keystone** instead of mocks.
+Keystone is deployed locally on a [kind](https://kind.sigs.k8s.io/) cluster via
+[forge](https://github.com/c5c3/forge) (its ControlPlane Quick Start), and the
+test then creates one project using the admin account and verifies it through
+the OpenStack SDK.
+
+It is deliberately a scaffold: one happy-path test. The unit suite under
+`test/unit/` keeps everything mocked; this harness is the first thing that
+talks to a real `openstack.connect()` / `keystone.projects.update()` path.
+
+## Prerequisites
+
+- A Docker-capable host with the Docker daemon running.
+- Outbound network access (to clone forge, pull operator images, and resolve
+  `keystone.127-0-0-1.nip.io` via [nip.io](https://nip.io/) to `127.0.0.1`).
+- For local runs, `openstacksdk` and `tox` must be importable in the active
+  environment (the CI job installs them into a throwaway venv). `kind` and
+  `kubectl` are installed by forge's own `make install-test-deps` into
+  `~/.local/bin`; the scripts prepend that directory to `PATH`.
+
+## Usage
+
+```bash
+# Full cycle: provision kind + Keystone, create a project, verify it, and tear
+# the cluster down on every exit path (including failure).
+make integration
+
+# Deploy Keystone and leave the cluster running for local debugging.
+make integration-up
+
+# Tear down the cluster left running by `make integration-up`.
+make integration-down
+```
+
+`make integration` only deletes a cluster it created itself. If a kind cluster
+named `forge` already exists (for example one you started with
+`make integration-up`), it is reused and left in place on exit.
+
+## Environment overrides
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `FORGE_REF` | `main` | forge git ref to check out (tip is fetched on every run; not pinned). |
+| `FORGE_DIR` | `${TMPDIR:-/tmp}/opm-forge` | Where forge is cloned. |
+| `CLUSTER_NAME` | `forge` | kind cluster name. |
+| `KIND_HOST_PORT` | `8443` | Host port the Envoy Gateway (and Keystone) is exposed on. |
+| `NAMESPACE` | `openstack` | Namespace the ControlPlane CR is applied to. |
+| `OS_CLIENT_CONFIG_FILE` | `test/integration/clouds.yaml` | Generated clouds.yaml path. |
+| `KEEP_PROJECT` | `0` | Set to `1` to keep the created project after `verify.py` (the cluster is torn down regardless). |
+
+## What is asserted
+
+After `tox -e create` creates the project as admin, `verify.py` connects with
+the OpenStack SDK and asserts that:
+
+- the `default` domain resolves,
+- the `opm-integration-test` project exists in it, and
+- the per-project group of the same name exists.
+
+`verify.py` exits non-zero on any failed assertion. It is a standalone script,
+not a `unittest`/`pytest` test, and is not named `test_*.py`, so the fast unit
+gate (`tox -e test`) never collects it.
+
+## Credentials
+
+No real credentials are committed. `deploy_keystone.sh` reads the admin
+password from the operator-projected
+`controlplane-keystone-admin-credentials` Secret and writes an ephemeral
+`clouds.yaml` (gitignored) with `verify: false`, because the Envoy Gateway
+serves a self-signed cert. `run.sh` removes the generated `clouds.yaml` on
+exit.
+
+Keystone's own database credentials are engine-issued (forge's Dynamic
+credentials mode, the managed-mode default): after applying the ControlPlane
+CR, `deploy_keystone.sh` waits for the projected MariaDB and runs forge's
+`setup-database-tenant.sh` to onboard the OpenBao database-engine tenant, the
+same step forge's `deploy-infra` performs when it applies its bundled CR. No
+long-lived DB password is seeded anywhere.

test/integration/controlplane.yaml

@@ -0,0 +1,42 @@
+---
+apiVersion: c5c3.io/v1alpha1
+kind: ControlPlane
+metadata:
+  name: controlplane
+  namespace: openstack
+spec:
+  openStackRelease: "2025.2"
+  # Pin the operator-projected backing services to a single instance for the
+  # single-node kind cluster. The CRD default for both is 3 (a 3-node Galera
+  # MariaDB plus three Memcached pods, matching the production baseline), which
+  # OOM-kills a single-node kind. database.replicas: 1 yields a single-instance
+  # non-Galera MariaDB (the c5c3-operator derives Galera from replicas > 1);
+  # cache.replicas: 1 a single Memcached pod. database.storageSize is pinned to
+  # 512Mi (mirroring forge's kind Quick Start CR) so the projected MariaDB
+  # requests a test-sized volume instead of the 100Gi CRD default (the
+  # production baseline) that a CI run never fills. Only these fields are set
+  # here; the operator defaulting webhook fills the clusterRef/database/
+  # secretRef/backend references. database.replicas and database.storageSize
+  # are immutable after creation, but CI recreates the cluster on every run so
+  # that never bites here.
+  #
+  # database.credentialsMode is deliberately left unset: forge defaults managed
+  # mode to Dynamic (engine-issued short-lived DB credentials from the OpenBao
+  # database engine; the static per-ControlPlane seed was retired). That mode
+  # needs the per-tenant database-engine onboarding, which deploy_keystone.sh
+  # runs after applying this CR.
+  infrastructure:
+    database:
+      replicas: 1
+      storageSize: 512Mi
+    cache:
+      replicas: 1
+  services:
+    keystone:
+      replicas: 1
+      publicEndpoint: https://keystone.127-0-0-1.nip.io:8443/v3
+      gateway:
+        parentRef:
+          name: openstack-gw
+        hostname: keystone.127-0-0-1.nip.io
+        path: /