From 0bd9b72843b85c2858907171b75b9f6716cee7b9 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Wed, 11 Feb 2026 18:47:59 -0800 Subject: [PATCH 01/11] add tls test fixtures --- tests/fixtures/10-tls.yaml | 95 ++++++++++++++++++++++++++++++ tests/fixtures/tls/dhparam.pem | 13 ++++ tests/fixtures/tls/public-cert.cnf | 24 ++++++++ tests/fixtures/tls/server.crt | 20 +++++++ tests/fixtures/tls/test-server.key | 28 +++++++++ tests/scenarios/smoke.py | 1 + 6 files changed, 181 insertions(+) create mode 100644 tests/fixtures/10-tls.yaml create mode 100644 tests/fixtures/tls/dhparam.pem create mode 100644 tests/fixtures/tls/public-cert.cnf create mode 100644 tests/fixtures/tls/server.crt create mode 100644 tests/fixtures/tls/test-server.key diff --git a/tests/fixtures/10-tls.yaml b/tests/fixtures/10-tls.yaml new file mode 100644 index 00000000..8574883c --- /dev/null +++ b/tests/fixtures/10-tls.yaml @@ -0,0 +1,95 @@ +--- +# Single-node deployment with TLS and load-balancer service +# Tests: Basic deployment with TLS using inline cert + secret refs, no keeper, minimal config +# Expected pods: 1 ClickHouse +clickhouse: + replicasCount: 1 + shardsCount: 1 + + defaultUser: + password: "TestTLSPassword123" + allowExternalAccess: true + + persistence: + enabled: true + size: 2Gi + accessMode: ReadWriteOnce + + service: + type: ClusterIP + + lbService: + enabled: true + + configurationFiles: + # To regenerate the public certificate: + # cd tests/fixtures/tls/ # this directory + # openssl req -x509 -key test-server.key -days 10950 \ + # -out server.crt -config public-cert.cnf -extensions v3_req + # + # Update inlineFileContent with the new certificate content for + # the public certificate to be used in the TLS test fixture. + # + # To verify the public certificate: + # openssl x509 -in server.crt -text -noout + # + # To verify that the private key & public certificate moduli match: + # openssl x509 -noout -modulus -in server.crt | openssl md5 + # openssl rsa -noout -modulus -in test-server.key | openssl md5 + config.d/foo.crt: | + -----BEGIN CERTIFICATE----- + MIIDQjCCAiqgAwIBAgIUXWU1ixzpK9OTqFfA+ZhK/EOtLIYwDQYJKoZIhvcNAQEL + BQAwHjEcMBoGA1UEAwwTKi5zdmMuY2x1c3Rlci5sb2NhbDAgFw0yNjAyMTQwMTIz + NTNaGA8yMDU2MDIwNzAxMjM1M1owHjEcMBoGA1UEAwwTKi5zdmMuY2x1c3Rlci5s + b2NhbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIvGupZ9FR3DLC1R + /+7hbUySKYSIYOnfZQBnfkMVyLBDdp1WX9aspJLEaazO0rU4l0YjqnLsnckuBxmr + OOzzeNA+8ExBkPEANR/mROMIcwXhrdFO3sWH2amVncHFUxspwgDhbZJ0zfVNtQo0 + Q/JthTWGqYW+4HbDnzOWWkUo23oZcClyELTbhQitxgrsOUyDIcR2ZNae3yueVAoK + F12fH4Sms75FLvwwlUuWU3F1lJKr/U7nPxBdl6CY/sPXITov2LcmwlQLebCchjVB + 3kEvPKJRBPmW0Dyrr9IRyyExU3qfYdzsJdZOHY/qOB0Dw42qXolkf2L5m6GtW3EF + hnqM5SsCAwEAAaN2MHQwCwYDVR0PBAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMB + MDEGA1UdEQQqMCiCFSouKi5zdmMuY2x1c3Rlci5sb2NhbIIJbG9jYWxob3N0hwR/ + AAABMB0GA1UdDgQWBBQzPiDb07CH5dKWb5xYa1PoGtz45jANBgkqhkiG9w0BAQsF + AAOCAQEActxS3ySYtlVmykOlaqTaTj8wyZjQ+gFBQ9APEuKwT4+F2HcuIRWWxJIT + Pt+CigSMG0XGFQi/+ZRfksXSmfcmErMgDUEr7jsxwvqn6esbAvHX02Vk82oAGkKu + t3JVxRAz9Dsl1Hm0W+IAGO336QFwd7iTo367TKSz+jcyHRfnftEZnxtbIvqAkR7V + 8z1NRYhRR5ZeLpwjiqwdAU4AgXKQifQWeWjPeJJr+4pEvy2ivpzhhalX7/tB1TuP + 4hpQdRi/c+0h5pDtrBpW61gUb0xVWySIkMifLLXfLNjynLKBJmWG91M34fTK1O8+ + LYd+XhJEhXtFt8+3kktghDcehcuK7g== + -----END CERTIFICATE----- + + bar.key: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.key + + dhparam.pem: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: dhparam.pem + + config.d/openssl.xml: | + + + + /etc/clickhouse-server/config.d/foo.crt + /etc/clickhouse-server/secrets.d/bar.key/clickhouse-certs/server.key + /etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem + relaxed + true + true + sslv2,sslv3 + true + + + + +keeper: + enabled: false + +operator: + enabled: true + + diff --git a/tests/fixtures/tls/dhparam.pem b/tests/fixtures/tls/dhparam.pem new file mode 100644 index 00000000..55ab3873 --- /dev/null +++ b/tests/fixtures/tls/dhparam.pem @@ -0,0 +1,13 @@ +-----BEGIN DH PARAMETERS----- +MIICDAKCAgEA7QR588LscfM+0JHrq22Xc13COYt5l2p+SqWKAijcdN6n7TWpaSVQ +3N1Lj+EFt2sXZ3NpNtfv8YAMZRdbAd1oe+kquGuugtxmUUUGUXiXqeIS6Gp7bGAj +5WWobY7WtkD/HsGrvI6kxhTk3nXIEYolUHMJGb0yVLcvYR73j5F6K3ONfd107T49 +/8PoVOr3ZcoBymfQK/a/mNVADKPPQ/ALAHjpIZEkQlCEj9Jw4Osaro0BEySoKJhK +5lIybQ5TJO023r9rpbKNxILaRy5esq4Vir3tlPb9eKumte6X4HFvTU36aTp5ZX/m +Ef25jhGRxnkH/N/WDEHXZPOToqyNJzdlmhvZjLj+Ru2SknS+pAZ9ZbbovzG1qPhW +BxFwotZLmTaD1+Xhm374HEY8PGeMytnrRq5W5oMpzY9PbDL+MhwxwChvWMpfPmbE +YN+InQjWNrw+C/VGLwyiOsQRhKnsCJSNckDv4cDOKuaIajhInnjGrQn7c51X2qT/ +8ScJ18FLrokEw/n+61xo4TFq7L9RSddiWbaTLvXrX6ZJvE/G0APA7eDeSN/p83TV +/pYgtiHOsgaSQ8qMFAIa03hdzfw/XqA8DTu5gf6JbV9BcPJ/381Kv3oC53I7XjQP +ZfpINvFBEs1Ss4fULqnQ3V65DktpS2HhC2gDVlw+084dSy6cnXX7pv8CAQICAgFF +-----END DH PARAMETERS----- diff --git a/tests/fixtures/tls/public-cert.cnf b/tests/fixtures/tls/public-cert.cnf new file mode 100644 index 00000000..0c11169a --- /dev/null +++ b/tests/fixtures/tls/public-cert.cnf @@ -0,0 +1,24 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req +prompt = no + +[req_distinguished_name] +CN = *.svc.cluster.local + +[v3_req] +keyUsage = digitalSignature, keyEncipherment, dataEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names + +[alt_names] +# Wildcard pattern covers all Kubernetes service DNS names: +# - t10-tls-service.t10-tls.svc.cluster.local +# - t01-tls-clickhouse.t01-tls.svc.cluster.local +# - chi----..svc.cluster.local +# This makes the certificate resilient to test fixture renames. +DNS.1 = *.*.svc.cluster.local + +# Localhost for local testing +DNS.2 = localhost +IP.1 = 127.0.0.1 diff --git a/tests/fixtures/tls/server.crt b/tests/fixtures/tls/server.crt new file mode 100644 index 00000000..aa2a9679 --- /dev/null +++ b/tests/fixtures/tls/server.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQjCCAiqgAwIBAgIUXWU1ixzpK9OTqFfA+ZhK/EOtLIYwDQYJKoZIhvcNAQEL +BQAwHjEcMBoGA1UEAwwTKi5zdmMuY2x1c3Rlci5sb2NhbDAgFw0yNjAyMTQwMTIz +NTNaGA8yMDU2MDIwNzAxMjM1M1owHjEcMBoGA1UEAwwTKi5zdmMuY2x1c3Rlci5s +b2NhbDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIvGupZ9FR3DLC1R +/+7hbUySKYSIYOnfZQBnfkMVyLBDdp1WX9aspJLEaazO0rU4l0YjqnLsnckuBxmr +OOzzeNA+8ExBkPEANR/mROMIcwXhrdFO3sWH2amVncHFUxspwgDhbZJ0zfVNtQo0 +Q/JthTWGqYW+4HbDnzOWWkUo23oZcClyELTbhQitxgrsOUyDIcR2ZNae3yueVAoK +F12fH4Sms75FLvwwlUuWU3F1lJKr/U7nPxBdl6CY/sPXITov2LcmwlQLebCchjVB +3kEvPKJRBPmW0Dyrr9IRyyExU3qfYdzsJdZOHY/qOB0Dw42qXolkf2L5m6GtW3EF +hnqM5SsCAwEAAaN2MHQwCwYDVR0PBAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMB +MDEGA1UdEQQqMCiCFSouKi5zdmMuY2x1c3Rlci5sb2NhbIIJbG9jYWxob3N0hwR/ +AAABMB0GA1UdDgQWBBQzPiDb07CH5dKWb5xYa1PoGtz45jANBgkqhkiG9w0BAQsF +AAOCAQEActxS3ySYtlVmykOlaqTaTj8wyZjQ+gFBQ9APEuKwT4+F2HcuIRWWxJIT +Pt+CigSMG0XGFQi/+ZRfksXSmfcmErMgDUEr7jsxwvqn6esbAvHX02Vk82oAGkKu +t3JVxRAz9Dsl1Hm0W+IAGO336QFwd7iTo367TKSz+jcyHRfnftEZnxtbIvqAkR7V +8z1NRYhRR5ZeLpwjiqwdAU4AgXKQifQWeWjPeJJr+4pEvy2ivpzhhalX7/tB1TuP +4hpQdRi/c+0h5pDtrBpW61gUb0xVWySIkMifLLXfLNjynLKBJmWG91M34fTK1O8+ +LYd+XhJEhXtFt8+3kktghDcehcuK7g== +-----END CERTIFICATE----- diff --git a/tests/fixtures/tls/test-server.key b/tests/fixtures/tls/test-server.key new file mode 100644 index 00000000..2681b86d --- /dev/null +++ b/tests/fixtures/tls/test-server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCLxrqWfRUdwywt +Uf/u4W1MkimEiGDp32UAZ35DFciwQ3adVl/WrKSSxGmsztK1OJdGI6py7J3JLgcZ +qzjs83jQPvBMQZDxADUf5kTjCHMF4a3RTt7Fh9mplZ3BxVMbKcIA4W2SdM31TbUK +NEPybYU1hqmFvuB2w58zllpFKNt6GXApchC024UIrcYK7DlMgyHEdmTWnt8rnlQK +Chddnx+EprO+RS78MJVLllNxdZSSq/1O5z8QXZegmP7D1yE6L9i3JsJUC3mwnIY1 +Qd5BLzyiUQT5ltA8q6/SEcshMVN6n2Hc7CXWTh2P6jgdA8ONql6JZH9i+ZuhrVtx +BYZ6jOUrAgMBAAECggEAA/WLassnjPdD9L4CMixVIeUbTfNlo1o+NyZh+i56HRwG +wRV66M7CaUcs+HBx9st1Os8J0JregUikz4JSaMvLW+1cdccpqYSS/KX+GjGEEzeS +6n8sajXwjBApLsg9vrg5FDijwjvoFY8E62wS50wDiOzujP5HcgsVIwCa6qM/TD8K +KVFBwLe9mL9yrGlSPCoj0vvt1R5VbEJh7PeYGfjUl30r0lWmP8ZlN6u2Q+aafEdy +p3gIHPDRRv6vPSiR24tVQgv0FaTnXRAjYtwhNqlXPMhtHnBEtNA1kuGE9IZimPw/ +6P3L3SkbGKL5TRZL6wRfrYada6TeFX9ueE45ES2UuQKBgQDBK/2nCvQr6wlF45kb +cQ7iunAcu5/DAlqQEXsOoef+WNNzIPxHCTH7YW4GhAUUSYyorLWi20tknrKJrITN +IotSVgItzpfia7HB3EIuSBysMOS81W+R3HYRoigQigT8YwxiRSACf6x1O5J0J/jC +tq2Galhwz8ILWaJ5SdACv+sBnQKBgQC5POFBz/GaDFooMFXU4kBw5e1YjkNkEQUh +bFQL9wZBiP2Xv1QiRV+kbrHwlW1p9gUjOogWz05L2z7qPbtdaIV6MPwEgr2xBCt5 +mkuSWhUDtzQ2pVFsSY75SOg8CE0lHnHSyUEaPhac06qn3KShUcakpm5A684aUjjy +1IH0huuLZwKBgAdBvc+uq6mStNB5UmEjiCmgU2Hg8omC5yAOaA8OqgZ2E8t5a8DH +aadF67o273HpqW0Uv+YUUuq+w3pEjuCd8ZnwPTi3UCFjZlQgECRo9RrK42zsn7pd +C9pxuwuUA8fveKGgcylk3new+zl93uyBrFcmW5gxVdrTTTU9PqE70HpJAoGAAaDH +Wgy50uDI6hGCr5xNdLCQpXaaoQaFRQXutyw0od7SW8MSujph3NAcQEEP9R50bRrW +l1y7E2+Z3fUs8GU6xxgnHuMHR8cBmtAAWgjwple13cUWMh1zZD1/zQdFpk3eMjwS +lmh1SmuR1GfcCo7tcAUGcwufhBu05G15tux4pYECgYEAkzun1vyNNoC2CrGR3wIO +SMwDy5JxZLIDAeAkidHsDrGzYzOekHLrHR2r47F4mJPUyiGVNCIz69fiVkcngshr +sEeYFP5cN6Ip7chEp683xVBbKM+jsQtMmhadaL5tsulqgScPquCap+dPvaq/ogJ1 +2jbKCT447cR8wtviH/OnOgY= +-----END PRIVATE KEY----- diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index e6b328f5..69e80df7 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -13,6 +13,7 @@ "fixtures/02-replicated-with-users.yaml", "fixtures/08-extracontainer-data-mount.yaml", "fixtures/09-usersprofiles-settings.yaml", + "fixtures/10-tls.yaml", # "fixtures/03-sharded-advanced.yaml", # "fixtures/04-external-keeper.yaml", # "fixtures/05-persistence-disabled.yaml", From 759803a55be04fe28e07d8a2a586abecfb5b5894 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Wed, 11 Feb 2026 18:48:39 -0800 Subject: [PATCH 02/11] update python ver in test README --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index a2117d84..81798d62 100644 --- a/tests/README.md +++ b/tests/README.md @@ -273,7 +273,7 @@ Areas that **need additional testing or are not fully covered**: ## 🌍 Supported Environment - **Operating System**: [Ubuntu](https://ubuntu.com/) 22.04 / 24.04 -- **Python**: >= 3.10.12 +- **Python**: >= 3.10.12, <= 3.12 (3.13+ has `lzma` package that is incompatible with test framework) - **Kubernetes**: >= 1.24 - **Helm**: >= 3.8.0 - **Minikube**: >= 1.28.0 (for local testing) From a7ec45b179f7834b27044e62c5b5bc02efb51108 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Wed, 11 Feb 2026 18:52:03 -0800 Subject: [PATCH 03/11] refactor support for OrbStack as k8s cluster option --- tests/README.md | 36 ++++++++++++++++++++++++--- tests/scenarios/smoke.py | 12 ++++----- tests/steps/kubernetes.py | 2 +- tests/steps/local_cluster.py | 32 ++++++++++++++++++++++++ tests/steps/minikube.py | 3 +++ tests/steps/orbstack.py | 47 ++++++++++++++++++++++++++++++++++++ 6 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 tests/steps/local_cluster.py create mode 100644 tests/steps/orbstack.py diff --git a/tests/README.md b/tests/README.md index 81798d62..eb581284 100644 --- a/tests/README.md +++ b/tests/README.md @@ -273,11 +273,12 @@ Areas that **need additional testing or are not fully covered**: ## 🌍 Supported Environment - **Operating System**: [Ubuntu](https://ubuntu.com/) 22.04 / 24.04 -- **Python**: >= 3.10.12, <= 3.12 (3.13+ has `lzma` package that is incompatible with test framework) +- **Python**: >= 3.10.12, <= 3.12 (3.13+ has `lzma` package that is incompatible with the test framework) - **Kubernetes**: >= 1.24 - **Helm**: >= 3.8.0 - **Minikube**: >= 1.28.0 (for local testing) - **Docker**: Required as Minikube driver + - (alternatively) **OrbStack**: >= 2.0 - **kubectl**: Latest stable version --- @@ -286,7 +287,9 @@ Areas that **need additional testing or are not fully covered**: ### Kubernetes Cluster -You need access to a Kubernetes cluster. For **local testing**, use Minikube: +You need access to a Kubernetes cluster. For **local testing**, two providers are supported: + +#### Option 1: Minikube (default) ```bash # Install Minikube @@ -297,6 +300,27 @@ sudo install minikube-linux-amd64 /usr/local/bin/minikube minikube version ``` +#### Option 2: OrbStack (account required, may need license) + +1. Install OrbStack by following the [_Quick start_ guide \(docs.orbstack.dev\)]( + https://docs.orbstack.dev/quick-start#installation). +2. Enable Kubernetes in OrbStack: + 1. Open the OrbStack app + 2. Go to Settings... (`Cmd ⌘ + ,`) → Kubernetes + 3. Toggle the `Enable Kubernetes cluster` option + 4. Click the `Apply and Restart` button +3. Verify that OrbStack is running. + ```sh + $ orb status + # Running + ``` +4. Verify the Kubernetes context. + ```sh + $ kubectl config get-contexts orbstack + # CURRENT NAME CLUSTER AUTHINFO NAMESPACE + # * orbstack orbstack orbstack + ``` + ### Helm Install Helm 3: @@ -334,14 +358,18 @@ To run the complete test suite (all active fixtures + upgrades): ```bash # From the repository root +# With Minikube (default) python3 ./tests/run/smoke.py + +# With OrbStack +LOCAL_K8S_PROVIDER=orbstack python3 ./tests/run/smoke.py ``` This will: -1. Start/restart Minikube with 4 CPUs and 6GB memory +1. \[Minikube only\] Start/restart Minikube with 4 CPUs and 6GB memory 2. Run all enabled fixture deployments 3. Run upgrade scenarios -4. Clean up and delete Minikube +4. \[Minikube only\] Clean up and delete Minikube **Expected Duration**: 10 minutes diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index 69e80df7..47bc5726 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -2,7 +2,7 @@ import os import tests.steps.kubernetes as kubernetes -import tests.steps.minikube as minikube +import tests.steps.local_cluster as local_cluster import tests.steps.helm as helm import tests.steps.clickhouse as clickhouse from tests.steps.deployment import HelmState @@ -52,7 +52,7 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): return with When("install ClickHouse with fixture configuration"): - kubernetes.use_context(context_name="minikube") + kubernetes.use_context(context_name=local_cluster.get_context_name()) helm.install( namespace=namespace, release_name=release_name, values_file=fixture_file ) @@ -102,7 +102,7 @@ def check_upgrade(self, initial_fixture, upgrade_fixture): note(f"Upgraded pods: {upgrade_state.get_expected_pod_count()}") with When("install ClickHouse with initial configuration"): - kubernetes.use_context(context_name="minikube") + kubernetes.use_context(context_name=local_cluster.get_context_name()) helm.install( namespace=namespace, release_name=release_name, values_file=initial_fixture ) @@ -189,9 +189,9 @@ def check_all_upgrades(self): def feature(self): """Run all comprehensive smoke tests.""" - with Given("minikube environment"): - minikube.setup_minikube_environment() - kubernetes.use_context(context_name="minikube") + with Given("local Kubernetes environment"): + local_cluster.setup_local_cluster() + kubernetes.use_context(context_name=local_cluster.get_context_name()) Feature(run=check_all_fixtures) diff --git a/tests/steps/kubernetes.py b/tests/steps/kubernetes.py index 35634062..f660913b 100644 --- a/tests/steps/kubernetes.py +++ b/tests/steps/kubernetes.py @@ -7,7 +7,7 @@ def get_pods(self, namespace): """Get the list of pods in the specified namespace and return in a list.""" - pods = run(cmd=f"minikube kubectl -- get pods -n {namespace} -o json") + pods = run(cmd=f"kubectl get pods -n {namespace} -o json") pods = json.loads(pods.stdout)["items"] return [p["metadata"]["name"] for p in pods] diff --git a/tests/steps/local_cluster.py b/tests/steps/local_cluster.py new file mode 100644 index 00000000..4e813d3d --- /dev/null +++ b/tests/steps/local_cluster.py @@ -0,0 +1,32 @@ +from tests.steps.system import * +import tests.steps.orbstack as orbstack +import tests.steps.minikube as minikube +import os + + +def resolve_provider(): + LOCAL_K8S_PROVIDER = os.environ.get("LOCAL_K8S_PROVIDER", "minikube").lower() + if LOCAL_K8S_PROVIDER not in (orbstack.CONTEXT_NAME, minikube.CONTEXT_NAME): + raise ValueError(f"Unknown LOCAL_K8S_PROVIDER: {LOCAL_K8S_PROVIDER}. " + "Supported values: " + f"'{minikube.CONTEXT_NAME}', " + f"'{orbstack.CONTEXT_NAME}'") + + return LOCAL_K8S_PROVIDER + + +@TestStep(Given) +def setup_local_cluster(self): + """Set up a local Kubernetes cluster.""" + provider = resolve_provider() + note(f"Using local Kubernetes provider: {provider}") + + if provider == "minikube": + minikube.setup_minikube_environment() + elif provider == "orbstack": + orbstack.setup_orbstack_environment() + + +def get_context_name(): + # This is okay since the provider is tightly-coupled to the context name + return resolve_provider() diff --git a/tests/steps/minikube.py b/tests/steps/minikube.py index 82084c9f..bf51e5e7 100644 --- a/tests/steps/minikube.py +++ b/tests/steps/minikube.py @@ -2,6 +2,9 @@ from tests.steps.kubernetes import use_context +CONTEXT_NAME = "minikube" + + @TestStep(Given) def minikube_start(self, cpus, memory): """Start minikube.""" diff --git a/tests/steps/orbstack.py b/tests/steps/orbstack.py new file mode 100644 index 00000000..4800641f --- /dev/null +++ b/tests/steps/orbstack.py @@ -0,0 +1,47 @@ +from tests.steps.system import * +from tests.steps.kubernetes import use_context + + +CONTEXT_NAME = "orbstack" + + +@TestStep(Given) +def orbstack_start(self): + """Start OrbStack.""" + + if orbstack_status(): + return + + run(cmd="orbctl start") + + +@TestStep(When) +def orbstack_status(self): + """Check if OrbStack is running.""" + + try: + result = run(cmd="orbctl status", check=False) + return result.returncode == 0 and "Running" in result.stdout + except: + return False + + +@TestStep(Given) +def setup_orbstack_environment(self, clean_up=True): + """Set up OrbStack environment with context.""" + + orbstack_start() + + use_context(context_name=CONTEXT_NAME) + + yield + + if clean_up: + cleanup_orbstack_environment() + + +@TestStep(Finally) +def cleanup_orbstack_environment(self): + """Clean up OrbStack environment.""" + + note("OrbStack environment lifecycle is managed outside of this framework.") From 0d410c5052a6bb15b5e70c9bcc8d8cf04b81b455 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Wed, 11 Feb 2026 20:22:04 -0800 Subject: [PATCH 04/11] add tls smoke tests --- tests/README.md | 2 +- tests/scenarios/smoke.py | 25 +++++++++++ tests/steps/tls.py | 97 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 tests/steps/tls.py diff --git a/tests/README.md b/tests/README.md index eb581284..3d0986ff 100644 --- a/tests/README.md +++ b/tests/README.md @@ -246,7 +246,6 @@ Areas that **need additional testing or are not fully covered**: - ❌ **Backup and restore** - No automated backup/restore testing - ❌ **Disaster recovery** - No full cluster failure scenarios - ❌ **Network policies** - Limited testing of K8s network restrictions -- ❌ **TLS/SSL** - No certificate or encryption testing - ❌ **Monitoring integration** - Prometheus scraping tested only via annotations - ❌ **Logging integration** - No FluentD/ElasticSearch integration tests - ❌ **Multi-cluster** - No federation or distributed query tests @@ -266,6 +265,7 @@ Areas that **need additional testing or are not fully covered**: - ⚠️ **Configuration drift** - No testing of manual changes vs. Helm state - ⚠️ **Resource exhaustion** - No OOM or disk full scenarios - ⚠️ **Long-running stability** - Tests are short-lived (minutes, not hours/days) +- ⚠️ **TLS/SSL** - Only tests configuration setup, does not verify actual encryption --- diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index 47bc5726..aac6b060 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -5,6 +5,7 @@ import tests.steps.local_cluster as local_cluster import tests.steps.helm as helm import tests.steps.clickhouse as clickhouse +import tests.steps.tls as tls from tests.steps.deployment import HelmState @@ -51,6 +52,12 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): skip("Skipping external keeper test (requires pre-existing keeper)") return + # Create TLS secrets if this is a TLS fixture + if "tls" in fixture_name: + with And("create TLS secrets"): + kubernetes.use_context(context_name=local_cluster.get_context_name()) + tls.create_tls_secret(namespace=namespace) + with When("install ClickHouse with fixture configuration"): kubernetes.use_context(context_name=local_cluster.get_context_name()) helm.install( @@ -70,6 +77,24 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): namespace=namespace, admin_password=admin_password ) + # Add TLS configuration verification for TLS fixtures + if "tls" in fixture_name: + with And("verify TLS configuration in CHI"): + chi_name = f"{release_name}-clickhouse" + tls.verify_tls_files_in_chi( + namespace=namespace, + chi_name=chi_name, + ) + + tls.verify_tls_secret_references_in_chi( + namespace=namespace, + chi_name=chi_name, + ) + + tls.verify_openssl_config_on_pod( + namespace=namespace, + ) + # Verify metrics endpoint is accessible with And("verify metrics endpoint"): clickhouse.verify_metrics_endpoint(namespace=namespace) diff --git a/tests/steps/tls.py b/tests/steps/tls.py new file mode 100644 index 00000000..e9e06559 --- /dev/null +++ b/tests/steps/tls.py @@ -0,0 +1,97 @@ +import json +import os +import xml.etree.ElementTree as ET + +from cryptography.hazmat.primitives.serialization import load_pem_parameters, load_pem_private_key +from cryptography.x509 import load_pem_x509_certificate + +from tests.steps.system import * +from tests.steps.kubernetes import * +import tests.steps.clickhouse as clickhouse + + +@TestStep(Then) +def verify_tls_files_in_chi(self, namespace, chi_name): + """Verify TLS files are present in CHI spec.""" + chi_info = run(cmd=f"kubectl get chi {chi_name} -n {namespace} -o json") + chi_data = json.loads(chi_info.stdout) + + files = chi_data.get("spec", {}).get("configuration", {}).get("files", {}) + + for expected_file in ["config.d/foo.crt", "bar.key", "dhparam.pem", "config.d/openssl.xml"]: + assert expected_file in files, f"Expected TLS file '{expected_file}' not found in CHI" + note(f"✓ TLS file present: {expected_file}") + + +@TestStep(Then) +def verify_tls_secret_references_in_chi(self, namespace, chi_name): + """Verify secret references are correct in CHI spec.""" + chi_info = run(cmd=f"kubectl get chi {chi_name} -n {namespace} -o json") + chi_data = json.loads(chi_info.stdout) + + files = chi_data.get("spec", {}).get("configuration", {}).get("files", {}) + + expected_secrets = { + "bar.key": "clickhouse-certs", + "dhparam.pem": "clickhouse-certs", + } + + for file_key, expected_secret_name in expected_secrets.items(): + assert file_key in files, f"File '{file_key}' not found in CHI" + file_config = files[file_key] + + assert isinstance(file_config, dict), f"Expected dict for secret ref in '{file_key}'" + assert "valueFrom" in file_config, f"No valueFrom in '{file_key}'" + + secret_ref = file_config["valueFrom"]["secretKeyRef"] + actual_secret_name = secret_ref["name"] + + assert actual_secret_name == expected_secret_name, \ + f"Expected secret '{expected_secret_name}' for '{file_key}', got '{actual_secret_name}'" + + note(f"✓ Secret reference correct: {file_key} → {actual_secret_name}") + + +@TestStep(Then) +def verify_openssl_config_on_pod(self, namespace): + """Verify openssl.xml format on the ClickHouse pod.""" + pod_name = clickhouse.get_ready_clickhouse_pod(namespace=namespace) + + result = run( + cmd=f"kubectl exec -n {namespace} {pod_name} " + f"-- cat /etc/clickhouse-server/config.d/openssl.xml" + ) + content = result.stdout + + try: + root = ET.fromstring(content) + except ET.ParseError as e: + raise AssertionError(f"openssl.xml is not valid XML: {e}") + + server_node = root.find(".//openSSL/server") + assert server_node is not None, "openssl.xml missing node" + + note(f"✓ openssl.xml present and valid on pod at /etc/clickhouse-server/config.d/openssl.xml") + + +@TestStep(When) +def create_tls_secret(self, namespace): + """Create a Kubernetes secret with TLS files from ../fixtures/tls/.""" + + tests_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + certs_dir = os.path.join(tests_dir, "fixtures", "tls") + + cert_file = os.path.join(certs_dir, "server.crt") + key_file = os.path.join(certs_dir, "test-server.key") + dhparam_file = os.path.join(certs_dir, "dhparam.pem") + + # At time of secret creation, the namespace might not exist + run(cmd=f"kubectl create namespace {namespace}", check=False) + # Optimistically delete secret in case it already exists for idempotency + run(cmd=f"kubectl delete secret clickhouse-certs -n {namespace}", check=False) + run(cmd=f"kubectl create secret generic clickhouse-certs -n {namespace} " + f"--from-file=server.crt={cert_file} " + f"--from-file=server.key={key_file} " + f"--from-file=dhparam.pem={dhparam_file}") + + note(f"✓ Created TLS secret: clickhouse-certs") From 18592d8a9f6e83a0f271a9461d3ed04644e09877 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Thu, 12 Feb 2026 09:22:07 -0800 Subject: [PATCH 05/11] new k8s method get_file_contents_from_pod --- tests/steps/kubernetes.py | 13 +++++++++++++ tests/steps/tls.py | 8 ++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/steps/kubernetes.py b/tests/steps/kubernetes.py index f660913b..e5031b4d 100644 --- a/tests/steps/kubernetes.py +++ b/tests/steps/kubernetes.py @@ -501,3 +501,16 @@ def delete_pod(self, namespace, pod_name): """ run(cmd=f"kubectl delete pod {pod_name} -n {namespace}", check=True) note(f"✓ Pod {pod_name} deleted from namespace {namespace}") + + +@TestStep(When) +def get_file_contents_from_pod(self, namespace, pod_name, file_path): + """Read the contents of a file from a pod. + + Args: + namespace: Kubernetes namespace + pod_name: Name of the pod + file_path: Absolute path to the file inside the pod + """ + result = run(cmd=f"kubectl exec -n {namespace} {pod_name} -- cat {file_path}") + return result.stdout diff --git a/tests/steps/tls.py b/tests/steps/tls.py index e9e06559..62acd633 100644 --- a/tests/steps/tls.py +++ b/tests/steps/tls.py @@ -57,11 +57,11 @@ def verify_openssl_config_on_pod(self, namespace): """Verify openssl.xml format on the ClickHouse pod.""" pod_name = clickhouse.get_ready_clickhouse_pod(namespace=namespace) - result = run( - cmd=f"kubectl exec -n {namespace} {pod_name} " - f"-- cat /etc/clickhouse-server/config.d/openssl.xml" + content = get_file_contents_from_pod( + namespace=namespace, + pod_name=pod_name, + file_path="/etc/clickhouse-server/config.d/openssl.xml", ) - content = result.stdout try: root = ET.fromstring(content) From f9c84bd7a4bf3244ba79ec1a8d9753c08d7ce8d8 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Thu, 12 Feb 2026 09:58:51 -0800 Subject: [PATCH 06/11] add tls file content validation to smoke tests --- tests/README.md | 1 + tests/requirements.txt | 3 ++- tests/scenarios/smoke.py | 4 ++++ tests/steps/tls.py | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 3d0986ff..5ed07bf5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -347,6 +347,7 @@ pip3 install -r tests/requirements.txt - `testflows.texts==2.0.211217.1011222` - Text utilities - `PyYAML==6.0.1` - YAML parsing - `requests==2.32.3` - HTTP requests +- `cryptography==46.0.5` - TLS validation --- diff --git a/tests/requirements.txt b/tests/requirements.txt index fdd308a8..5c620785 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ requests==2.32.3 testflows==2.4.13 testflows.texts==2.0.211217.1011222 -PyYAML==6.0.1 \ No newline at end of file +PyYAML==6.0.1 +cryptography==46.0.5 diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index aac6b060..1b9733bb 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -94,6 +94,10 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): tls.verify_openssl_config_on_pod( namespace=namespace, ) + + tls.verify_tls_files_on_pod( + namespace=namespace, + ) # Verify metrics endpoint is accessible with And("verify metrics endpoint"): diff --git a/tests/steps/tls.py b/tests/steps/tls.py index 62acd633..717c17a2 100644 --- a/tests/steps/tls.py +++ b/tests/steps/tls.py @@ -74,6 +74,45 @@ def verify_openssl_config_on_pod(self, namespace): note(f"✓ openssl.xml present and valid on pod at /etc/clickhouse-server/config.d/openssl.xml") +@TestStep(Then) +def verify_tls_files_on_pod(self, namespace): + """Verify TLS file contents on the ClickHouse pod.""" + + pod_name = clickhouse.get_ready_clickhouse_pod(namespace=namespace) + + cert_pem = get_file_contents_from_pod( + namespace=namespace, + pod_name=pod_name, + file_path="/etc/clickhouse-server/config.d/foo.crt", + ) + + key_pem = get_file_contents_from_pod( + namespace=namespace, + pod_name=pod_name, + file_path="/etc/clickhouse-server/secrets.d/bar.key/clickhouse-certs/server.key", + ) + + cert = load_pem_x509_certificate(cert_pem.encode()) + key = load_pem_private_key(key_pem.encode(), password=None) + + cert_modulus = cert.public_key().public_numbers().n + key_modulus = key.public_key().public_numbers().n + + assert cert_modulus == key_modulus, "Certificate and private key moduli do not match" + note("✓ Certificate and private key moduli match") + + dh_pem = get_file_contents_from_pod( + namespace=namespace, + pod_name=pod_name, + file_path="/etc/clickhouse-server/secrets.d/dhparam.pem/clickhouse-certs/dhparam.pem", + ) + + dh_params = load_pem_parameters(dh_pem.encode()) + assert dh_params.parameter_numbers().g == 2, \ + f"Expected DH params generator g=2, got g={dh_params.parameter_numbers().g}" + note("✓ DH params valid (g=2)") + + @TestStep(When) def create_tls_secret(self, namespace): """Create a Kubernetes secret with TLS files from ../fixtures/tls/.""" From 4b01f1036a322bf8283810676a7ba0101e37e1f9 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Tue, 24 Feb 2026 16:31:13 -0800 Subject: [PATCH 07/11] fix indiscriminate quoting of clickhouse.settings values --- charts/clickhouse/templates/chi.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/clickhouse/templates/chi.yaml b/charts/clickhouse/templates/chi.yaml index 790a7428..e2971d7d 100644 --- a/charts/clickhouse/templates/chi.yaml +++ b/charts/clickhouse/templates/chi.yaml @@ -190,7 +190,8 @@ spec: {{- if .Values.clickhouse.settings }} settings: {{- range $key, $value := .Values.clickhouse.settings }} - {{ $key }}: "{{ $value }}" + {{- $valueIsNumeric := regexMatch "^[0-9]+$" ( $value | toString ) }} + {{ $key }}: {{ ternary $value ( $value | quote ) $valueIsNumeric }} {{- end }} {{- end }} clusters: From 4322f7d2279763aea9e62835c253c655b6f9928b Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Thu, 12 Feb 2026 17:08:35 -0800 Subject: [PATCH 08/11] expose secure ports --- charts/clickhouse/templates/chi.yaml | 26 ++++++++++++++++++++++++++ tests/fixtures/10-tls.yaml | 3 +++ tests/scenarios/smoke.py | 5 +++++ tests/steps/clickhouse.py | 4 ++-- tests/steps/tls.py | 15 +++++++++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/charts/clickhouse/templates/chi.yaml b/charts/clickhouse/templates/chi.yaml index e2971d7d..6d4184f1 100644 --- a/charts/clickhouse/templates/chi.yaml +++ b/charts/clickhouse/templates/chi.yaml @@ -1,4 +1,10 @@ {{- $service_name := tpl (include "clickhouse.serviceTemplateName" . ) . -}} +{{- $extraPortNames := list }} +{{- range .Values.clickhouse.extraPorts }} +{{- $extraPortNames = append $extraPortNames .name }} +{{- end }} +{{- $serviceHttpsPort := (((.Values.clickhouse).settings).https_port) | default "" -}} +{{- $serviceSecureTcpPort := (((.Values.clickhouse).settings).tcp_port_secure) | default "" -}} --- apiVersion: "clickhouse.altinity.com/v1" kind: ClickHouseInstallation @@ -62,6 +68,16 @@ spec: targetPort: {{ .containerPort }} {{- end }} {{- end }} + {{- if and $serviceHttpsPort (not (has "https" $extraPortNames)) }} + - name: https + port: {{ $serviceHttpsPort }} + targetPort: {{ $serviceHttpsPort }} + {{- end }} + {{- if and $serviceSecureTcpPort (not (has "tcp-secure" $extraPortNames)) }} + - name: tcp-secure + port: {{ $serviceSecureTcpPort }} + targetPort: {{ $serviceSecureTcpPort }} + {{- end }} selector: {{- include "clickhouse.selectorLabels" . | nindent 12 }} {{- if .Values.clickhouse.lbService.enabled }} @@ -96,6 +112,16 @@ spec: targetPort: {{ .containerPort }} {{- end }} {{- end }} + {{- if and $serviceHttpsPort (not (has "https" $extraPortNames)) }} + - name: https + port: {{ $serviceHttpsPort }} + targetPort: {{ $serviceHttpsPort }} + {{- end }} + {{- if and $serviceSecureTcpPort (not (has "tcp-secure" $extraPortNames)) }} + - name: tcp-secure + port: {{ $serviceSecureTcpPort }} + targetPort: {{ $serviceSecureTcpPort }} + {{- end }} selector: {{- include "clickhouse.selectorLabels" . | nindent 12 }} {{- end }} diff --git a/tests/fixtures/10-tls.yaml b/tests/fixtures/10-tls.yaml index 8574883c..a8cd7ecf 100644 --- a/tests/fixtures/10-tls.yaml +++ b/tests/fixtures/10-tls.yaml @@ -21,6 +21,9 @@ clickhouse: lbService: enabled: true + settings: + https_port: 8444 + configurationFiles: # To regenerate the public certificate: # cd tests/fixtures/tls/ # this directory diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index 1b9733bb..356cdda4 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -99,6 +99,11 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): namespace=namespace, ) + tls.verify_settings_ports_in_chi( + namespace=namespace, + chi_name=chi_name, + ) + # Verify metrics endpoint is accessible with And("verify metrics endpoint"): clickhouse.verify_metrics_endpoint(namespace=namespace) diff --git a/tests/steps/clickhouse.py b/tests/steps/clickhouse.py index ea565d05..433fcb36 100644 --- a/tests/steps/clickhouse.py +++ b/tests/steps/clickhouse.py @@ -478,8 +478,8 @@ def verify_profiles_and_user_settings( actual_value = settings_cfg.get(key) expected_value = str(value) assert ( - actual_value == expected_value - ), f"Expected setting {key}={expected_value}, got {actual_value}" + str(actual_value) == expected_value + ), f"Expected setting {key}={expected_value!r}, got {actual_value!r}" note("Users, profiles, and settings configuration verified") diff --git a/tests/steps/tls.py b/tests/steps/tls.py index 717c17a2..0ef62f1c 100644 --- a/tests/steps/tls.py +++ b/tests/steps/tls.py @@ -52,6 +52,21 @@ def verify_tls_secret_references_in_chi(self, namespace, chi_name): note(f"✓ Secret reference correct: {file_key} → {actual_secret_name}") +@TestStep(Then) +def verify_settings_ports_in_chi(self, namespace, chi_name): + """Verify settings block has correct port configuration in CHI spec.""" + chi_info = run(cmd=f"kubectl get chi {chi_name} -n {namespace} -o json") + chi_data = json.loads(chi_info.stdout) + + settings = chi_data.get("spec", {}).get("configuration", {}).get("settings", {}) + expected_https_port = 8444; + assert settings.get("https_port") == expected_https_port, \ + f"Expected https_port: {expected_https_port}, got: {settings.get('https_port')!r}" + assert "tcp_port_secure" not in settings, \ + f"Did not expect 'tcp_port_secure' in settings, but found: {settings.get('tcp_port_secure')!r}" + note(f"✓ Settings block only has https_port as explicitly set: {expected_https_port}") + + @TestStep(Then) def verify_openssl_config_on_pod(self, namespace): """Verify openssl.xml format on the ClickHouse pod.""" From c71d935d8a621fa64e8d9df781bc8e18f3dcc9ae Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Fri, 13 Feb 2026 10:23:20 -0800 Subject: [PATCH 09/11] refactor smokes to gracefully clean up dangling namespaces --- tests/scenarios/smoke.py | 2 ++ tests/steps/kubernetes.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index 356cdda4..ddad8c28 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -111,6 +111,7 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): with Finally("cleanup deployment"): helm.uninstall(namespace=namespace, release_name=release_name) kubernetes.delete_namespace(namespace=namespace) + kubernetes.remove_chi_finalizers(namespace=namespace) @TestScenario @@ -193,6 +194,7 @@ def check_upgrade(self, initial_fixture, upgrade_fixture): with Finally("cleanup deployment"): helm.uninstall(namespace=namespace, release_name=release_name) kubernetes.delete_namespace(namespace=namespace) + kubernetes.remove_chi_finalizers(namespace=namespace) @TestFeature diff --git a/tests/steps/kubernetes.py b/tests/steps/kubernetes.py index e5031b4d..7e54a7f4 100644 --- a/tests/steps/kubernetes.py +++ b/tests/steps/kubernetes.py @@ -472,6 +472,29 @@ def get_secrets(self, namespace): return [item["metadata"]["name"] for item in secrets_data.get("items", [])] +@TestStep(Finally) +def remove_chi_finalizers(self, namespace): + """Remove finalizers from CHI resources to unblock namespace deletion. + + Unless the operator is externally deployed, after a helm uninstall, the + operator disappears but CHI resources will still have finalizers that block + namespace deletion, causing the namespace to persist with 'Terminating' status. + + Args: + namespace: Kubernetes namespace to delete + """ + result = run( + cmd=f"kubectl get chi -n {namespace} -o name", + ) + chi_resource_names = result.stdout.strip().split() + for resource_name in chi_resource_names: + run( + cmd=f"kubectl patch {resource_name} -n {namespace} " + f"--type json -p '[{{\"op\": \"remove\", \"path\": \"/metadata/finalizers\"}}]'", + ) + note(f"✓ Removed finalizers from {resource_name}: {namespace}/{resource_name}") + + @TestStep(Finally) def delete_namespace(self, namespace): """Delete a Kubernetes namespace. From 5a5a667d826c87dd29e5fa0fb83442e9418312ba Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Thu, 26 Feb 2026 18:38:35 -0800 Subject: [PATCH 10/11] reuse component in tls smoke logic --- tests/scenarios/smoke.py | 4 ---- tests/steps/tls.py | 20 ++++++++------------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index ddad8c28..ef707a75 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -80,15 +80,12 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): # Add TLS configuration verification for TLS fixtures if "tls" in fixture_name: with And("verify TLS configuration in CHI"): - chi_name = f"{release_name}-clickhouse" tls.verify_tls_files_in_chi( namespace=namespace, - chi_name=chi_name, ) tls.verify_tls_secret_references_in_chi( namespace=namespace, - chi_name=chi_name, ) tls.verify_openssl_config_on_pod( @@ -101,7 +98,6 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): tls.verify_settings_ports_in_chi( namespace=namespace, - chi_name=chi_name, ) # Verify metrics endpoint is accessible diff --git a/tests/steps/tls.py b/tests/steps/tls.py index 0ef62f1c..7da10449 100644 --- a/tests/steps/tls.py +++ b/tests/steps/tls.py @@ -1,4 +1,3 @@ -import json import os import xml.etree.ElementTree as ET @@ -11,11 +10,10 @@ @TestStep(Then) -def verify_tls_files_in_chi(self, namespace, chi_name): +def verify_tls_files_in_chi(self, namespace): """Verify TLS files are present in CHI spec.""" - chi_info = run(cmd=f"kubectl get chi {chi_name} -n {namespace} -o json") - chi_data = json.loads(chi_info.stdout) - + chi_data = clickhouse.get_chi_info(namespace=namespace) + files = chi_data.get("spec", {}).get("configuration", {}).get("files", {}) for expected_file in ["config.d/foo.crt", "bar.key", "dhparam.pem", "config.d/openssl.xml"]: @@ -24,11 +22,10 @@ def verify_tls_files_in_chi(self, namespace, chi_name): @TestStep(Then) -def verify_tls_secret_references_in_chi(self, namespace, chi_name): +def verify_tls_secret_references_in_chi(self, namespace): """Verify secret references are correct in CHI spec.""" - chi_info = run(cmd=f"kubectl get chi {chi_name} -n {namespace} -o json") - chi_data = json.loads(chi_info.stdout) - + chi_data = clickhouse.get_chi_info(namespace=namespace) + files = chi_data.get("spec", {}).get("configuration", {}).get("files", {}) expected_secrets = { @@ -53,10 +50,9 @@ def verify_tls_secret_references_in_chi(self, namespace, chi_name): @TestStep(Then) -def verify_settings_ports_in_chi(self, namespace, chi_name): +def verify_settings_ports_in_chi(self, namespace): """Verify settings block has correct port configuration in CHI spec.""" - chi_info = run(cmd=f"kubectl get chi {chi_name} -n {namespace} -o json") - chi_data = json.loads(chi_info.stdout) + chi_data = clickhouse.get_chi_info(namespace=namespace) settings = chi_data.get("spec", {}).get("configuration", {}).get("settings", {}) expected_https_port = 8444; From db74055016d9d5544570c0f848a8a90000604d83 Mon Sep 17 00:00:00 2001 From: Felix Wong Date: Thu, 5 Mar 2026 21:17:22 -0800 Subject: [PATCH 11/11] expand tls smoke test verification --- tests/README.md | 2 +- tests/scenarios/smoke.py | 8 +++++ tests/steps/tls.py | 67 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 5ed07bf5..8a57f9cb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -173,6 +173,7 @@ The test suite provides comprehensive coverage across multiple dimensions: #### **2. ClickHouse Functionality** - ✅ Version verification - ✅ Connection testing +- ✅ Server-side TLS/SSL and HTTPS - ✅ Query execution - ✅ Cluster topology (system.clusters) - ✅ Replication health (system.replicas) @@ -265,7 +266,6 @@ Areas that **need additional testing or are not fully covered**: - ⚠️ **Configuration drift** - No testing of manual changes vs. Helm state - ⚠️ **Resource exhaustion** - No OOM or disk full scenarios - ⚠️ **Long-running stability** - Tests are short-lived (minutes, not hours/days) -- ⚠️ **TLS/SSL** - Only tests configuration setup, does not verify actual encryption --- diff --git a/tests/scenarios/smoke.py b/tests/scenarios/smoke.py index ef707a75..d97d7169 100644 --- a/tests/scenarios/smoke.py +++ b/tests/scenarios/smoke.py @@ -79,6 +79,7 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): # Add TLS configuration verification for TLS fixtures if "tls" in fixture_name: + https_port = 8444 with And("verify TLS configuration in CHI"): tls.verify_tls_files_in_chi( namespace=namespace, @@ -98,6 +99,13 @@ def check_deployment(self, fixture_file, skip_external_keeper=True): tls.verify_settings_ports_in_chi( namespace=namespace, + expected_https_port=https_port, + ) + + with And("verify HTTPS endpoint certificate"): + tls.verify_https_certificate( + namespace=namespace, + https_port=https_port, ) # Verify metrics endpoint is accessible diff --git a/tests/steps/tls.py b/tests/steps/tls.py index 7da10449..2c36f977 100644 --- a/tests/steps/tls.py +++ b/tests/steps/tls.py @@ -1,8 +1,10 @@ import os import xml.etree.ElementTree as ET +from datetime import datetime, timezone from cryptography.hazmat.primitives.serialization import load_pem_parameters, load_pem_private_key from cryptography.x509 import load_pem_x509_certificate +from cryptography.x509.oid import ExtensionOID, NameOID from tests.steps.system import * from tests.steps.kubernetes import * @@ -50,12 +52,11 @@ def verify_tls_secret_references_in_chi(self, namespace): @TestStep(Then) -def verify_settings_ports_in_chi(self, namespace): +def verify_settings_ports_in_chi(self, namespace, expected_https_port): """Verify settings block has correct port configuration in CHI spec.""" chi_data = clickhouse.get_chi_info(namespace=namespace) settings = chi_data.get("spec", {}).get("configuration", {}).get("settings", {}) - expected_https_port = 8444; assert settings.get("https_port") == expected_https_port, \ f"Expected https_port: {expected_https_port}, got: {settings.get('https_port')!r}" assert "tcp_port_secure" not in settings, \ @@ -124,6 +125,68 @@ def verify_tls_files_on_pod(self, namespace): note("✓ DH params valid (g=2)") +@TestStep(Then) +def verify_https_certificate(self, namespace, https_port): + """Verify the HTTPS endpoint serves TLS with the correct certificate. + + Performs a TLS handshake against the ClickHouse HTTPS port from within + the pod, then validates the served certificate against the configured one. + """ + pod_name = clickhouse.get_ready_clickhouse_pod(namespace=namespace) + + result = run( + cmd=f"kubectl exec -n {namespace} {pod_name} -- " + f"sh -c 'openssl s_client -connect localhost:{https_port} " + f"&1'", + check=False, + ) + + served_cert = load_pem_x509_certificate(result.stdout.encode()) + note(f"✓ TLS handshake successful on port {https_port}") + + now = datetime.now(timezone.utc) + assert served_cert.not_valid_before_utc <= now, \ + f"Certificate not yet valid (notBefore: {served_cert.not_valid_before_utc})" + assert served_cert.not_valid_after_utc > now, \ + f"Certificate has expired (notAfter: {served_cert.not_valid_after_utc})" + note( + f"✓ Certificate valid: " + f"{served_cert.not_valid_before_utc.date()} to " + f"{served_cert.not_valid_after_utc.date()}" + ) + + cn_attrs = served_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME) + assert cn_attrs, "Certificate has no CN" + cn = cn_attrs[0].value + note(f"✓ Certificate CN: {cn}") + + # The cryptography library has no get-by-name accessor for X.509 + # extensions, so we iterate all extensions and match by OID to find + # the Subject Alternative Name extension. + san_names = [] + for ext in served_cert.extensions: + if ext.oid == ExtensionOID.SUBJECT_ALTERNATIVE_NAME: + san_names = [str(name.value) for name in ext.value] + break + + assert san_names, "Certificate has no SANs" + assert any(cn in san for san in san_names), \ + f"CN '{cn}' not found as substring of any SAN: {san_names}" + note(f"✓ CN is substring of SAN (SANs: {san_names})") + + configured_cert_pem = get_file_contents_from_pod( + namespace=namespace, + pod_name=pod_name, + file_path="/etc/clickhouse-server/config.d/foo.crt", + ) + configured_cert = load_pem_x509_certificate(configured_cert_pem.encode()) + + assert served_cert == configured_cert, \ + f"Served cert (serial {served_cert.serial_number:#x}) " \ + f"!= configured cert (serial {configured_cert.serial_number:#x})" + note(f"✓ Served certificate matches configured (serial {served_cert.serial_number:#x})") + + @TestStep(When) def create_tls_secret(self, namespace): """Create a Kubernetes secret with TLS files from ../fixtures/tls/."""