test(e2e):Run controller in e2e tests process, dump namespace obejcts and pod logs on failure.

.github/workflows/ci.yaml

@@ -171,25 +171,6 @@ jobs:
       - name: Lint Helm Chart
         run: helm lint ./dist/chart
 
-      - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1
-        with:
-          cluster_name: helm-test
-          version: ${{ env.kind-version }}
-          config: ci/kind-cluster.config
-
-      - name: Run Helm tests
-        run: make test-helm
-
-      - name: Test Report
-        uses: dorny/test-reporter@v2
-        if: ${{ !cancelled() }}
-        with:
-          name: Helm tests
-          badge-title: Helm tests
-          path: "**/report/*.xml"
-          reporter: java-junit
-
   compat-e2e-test:
     runs-on: [ubuntu-latest]
     strategy:
@@ -225,8 +206,17 @@ jobs:
       - name: Go Mod
         run: go mod download
 
-      - name: Build image
-        run: make docker-build IMG="clickhouse.com/clickhouse-operator:v0.0.1" BUILD_TIME=e2e
+      - name: Install tools
+        run: make operator-sdk opm
+
+      - name: Pre-pull ClickHouse images into Kind
+        run: |
+          for version in $(echo "${{ matrix.clickhouse_version }}" | tr ',' ' '); do
+            for image in clickhouse/clickhouse-keeper clickhouse/clickhouse-server; do
+              docker pull "docker.io/${image}:${version}"
+              kind load docker-image "docker.io/${image}:${version}"
+            done
+          done
 
       - name: Run compatibility e2e tests
         run: make test-compat-e2e
@@ -238,7 +228,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: compat-e2e-report-${{ strategy.job-index }}
-          path: "**/report/*.xml"
+          path: "**/report/*"
           if-no-files-found: error
           overwrite: true
 
@@ -261,9 +251,6 @@ jobs:
       - name: Go Mod
         run: go mod download
 
-      - name: Build image
-        run: make docker-build IMG="clickhouse.com/clickhouse-operator:v0.0.1" BUILD_TIME=e2e
-
       - name: Create k8s Kind Cluster
         uses: helm/kind-action@v1
         with:
@@ -280,7 +267,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: e2e-report-${{ matrix.scope }}
-          path: "**/report/*.xml"
+          path: "**/report/*"
           if-no-files-found: error
           overwrite: true
 

.gitignore

@@ -9,6 +9,7 @@ Dockerfile.cross
 dist/chart/*.tgz
 dist/chart/index.yaml
 dist/chart/charts
+dist/install.yaml
 clickhouse-operator-helm-*.tgz
 clickhouse-operator_*.tar.gz
 bundle

.golangci.yml

@@ -103,6 +103,10 @@ linters:
               - "github.com/onsi/gomega"
         - name: package-comments
           disabled: true
+    staticcheck:
+      dot-import-whitelist:
+        - "github.com/onsi/ginkgo/v2"
+        - "github.com/onsi/gomega"
     gosec:
       excludes:
         - G204

Makefile

@@ -56,7 +56,7 @@ endif
 
 # Set the Operator SDK version to use. By default, what is installed on the system is used.
 # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit.
-OPERATOR_SDK_VERSION ?= v1.42.0
+OPERATOR_SDK_VERSION ?= v1.42.2
 OPERATOR_MANAGER_VERSION ?= v1.62.0
 # Image URL to use all building/pushing image targets
 IMG ?= ${IMAGE_TAG_BASE}:${FULL_VERSION}
@@ -124,12 +124,12 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /helm) \
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /deploy) \
 	--ginkgo.v
 
 .PHONY: test-ci
 test-ci: manifests generate fmt vet envtest ## Run tests in CI env.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /helm) \
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /deploy) \
 	-v -count=1 -race -coverprofile cover.out --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
 
 fuzz-keeper: generate # Run keeper spec fuzz tests
@@ -143,23 +143,19 @@ fuzz: fuzz-keeper fuzz-clickhouse ## Run all fuzz tests
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e: ## Run all e2e tests.
-	go test ./test/e2e/ -test.timeout 30m --ginkgo.v
+	go test ./test/e2e/ -test.timeout 30m -v --ginkgo.v
 
 .PHONY: test-keeper-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-keeper-e2e: ## Run keeper e2e tests.
-	go test ./test/e2e/ --ginkgo.label-filter keeper -test.timeout 30m --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
+	go test ./test/e2e/ --ginkgo.label-filter keeper -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
 
 .PHONY: test-clickhouse-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-clickhouse-e2e: ## Run clickhouse e2e tests.
-	go test ./test/e2e/ --ginkgo.label-filter clickhouse -test.timeout 30m --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
+	go test ./test/e2e/ --ginkgo.label-filter clickhouse -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
 
 .PHONY: test-compat-e2e  # Run compatibility smoke tests across ClickHouse versions.
-test-compat-e2e: ## Run compatibility e2e tests.
-	go test ./test/e2e/ --ginkgo.label-filter compatibility -test.timeout 30m --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
-
-.PHONY: test-helm  # Run helm chart tests against a Kind k8s instance that is spun up.
-test-helm: ginkgo ## Run helm chart tests
-	$(GINKGO) run --nodes 4 -p -v --junit-report=report/junit-report.xml test/helm
+test-compat-e2e: ## Run compatibility e2e tests (requires CLICKHOUSE_VERSION env var).
+	go test ./test/deploy/ -test.timeout 30m -v --ginkgo.v --ginkgo.junit-report=report/junit-report.xml
 
 .PHONY: lint
 lint: golangci-lint codespell actionlint ## Run golangci-lint linter, codespell, and actionlint
@@ -441,6 +437,9 @@ OPERATOR_SDK = $(shell which operator-sdk)
 endif
 endif
 
+operator-sdk-path: operator-sdk
+	@echo $(OPERATOR_SDK)
+
 .PHONY: bundle
 bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files.
 	$(OPERATOR_SDK) generate kustomize manifests -q
@@ -503,7 +502,8 @@ catalog-dockerfile: opm ## Generate catalog Dockerfile
 catalog-template: # Generate catalog template with all bundles from registry
 	./ci/generate-catalog-template.sh $(IMAGE_TAG_BASE)-bundle
 
-catalog-render: opm catalog-template ## Genermrate FBC catalog from template
+.PHONY: catalog-render
+catalog-render: opm catalog-template ## Generate FBC catalog from template
 	$(OPM) alpha render-template catalog/clickhouse-operator-template.yaml > catalog/catalog.yaml
 	$(OPM) validate catalog
 

Tiltfile

@@ -50,7 +50,7 @@ local_resource(
 docker_build_with_restart(
     image_repo,
     ".",
-    dockerfile = "./Dockerfile.tilt",
+    dockerfile = "./dev.Dockerfile",
     entrypoint = ["/manager"],
     only = [
         "bin/manager_linux",

bundle.Dockerfile

@@ -7,7 +7,7 @@ LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/
 LABEL operators.operatorframework.io.bundle.package.v1=clickhouse-operator
 LABEL operators.operatorframework.io.bundle.channels.v1=stable
 LABEL operators.operatorframework.io.bundle.channel.default.v1=stable
-LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.42.0
+LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.42.2
 LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1
 LABEL operators.operatorframework.io.metrics.project_layout=go.kubebuilder.io/v4
 

cmd/main.go

@@ -195,11 +195,11 @@ func run() error {
 		upgradeChecker = upgrade.NewChecker(updater)
 	}
 
-	if err = keeper.SetupWithManager(mgr, zapLogger, upgradeChecker); err != nil {
+	if err = keeper.SetupWithManager(mgr, zapLogger, upgradeChecker, nil); err != nil {
 		return fmt.Errorf("unable to setup KeeperCluster controller: %w", err)
 	}
 
-	if err = clickhouse.SetupWithManager(mgr, zapLogger, upgradeChecker); err != nil {
+	if err = clickhouse.SetupWithManager(mgr, zapLogger, upgradeChecker, nil); err != nil {
 		return fmt.Errorf("unable to setup ClickHouseCluster controller: %w", err)
 	}
 

config/manifests/bases/clickhouse-operator.clusterserviceversion.yaml

@@ -15,7 +15,7 @@ spec:
   apiservicedefinitions: {}
   customresourcedefinitions:
     owned:
-    - description: ClickHouseCluster is the Schema for the clickhouseclusters API.
+    - description: ClickHouseCluster is the Schema for the `clickhouseclusters` API.
       displayName: ClickHouse Cluster
       kind: ClickHouseCluster
       name: clickhouseclusters.clickhouse.com
@@ -42,15 +42,13 @@ spec:
         name: ""
         version: v1
       specDescriptors:
-      - description: Settings for the replicas storage.
+      - description: Specification of persistent storage for ClickHouse data.
         displayName: Data Volume Claim Spec
         path: dataVolumeClaimSpec
       - description: Reference to the KeeperCluster that is used for ClickHouse coordination.
         displayName: Keeper Cluster Reference
         path: keeperClusterRef
-      - description: |-
-          Number of replicas in the single shard
-          This is a pointer to distinguish between explicit zero and unspecified.
+      - description: Number of replicas in the single shard.
         displayName: Replica count in shard
         path: replicas
       statusDescriptors:
@@ -67,7 +65,7 @@ spec:
       - description: ObservedGeneration indicates latest generation observed by controller.
         displayName: Observed Generation
         path: observedGeneration
-      - description: ReadyReplicas Total number of replicas ready to server requests.
+      - description: ReadyReplicas Total number of replicas ready to serve requests.
         displayName: Ready Replicas
         path: readyReplicas
       - description: StatefulSetRevision indicates target StatefulSet revision for
@@ -78,8 +76,11 @@ spec:
           revision.
         displayName: Update Revision
         path: updateRevision
+      - description: Version indicates the version reported by the container image.
+        displayName: Version
+        path: version
       version: v1alpha1
-    - description: KeeperCluster is the Schema for the keeperclusters API.
+    - description: KeeperCluster is the Schema for the `keeperclusters` API.
       displayName: Keeper Cluster
       kind: KeeperCluster
       name: keeperclusters.clickhouse.com
@@ -103,12 +104,10 @@ spec:
         name: ""
         version: v1
       specDescriptors:
-      - description: Settings for the replicas storage.
+      - description: Specification of persistent storage for ClickHouse Keeper data.
         displayName: Data Volume Claim Spec
         path: dataVolumeClaimSpec
-      - description: |-
-          Number of replicas in the cluster
-          This is a pointer to distinguish between explicit zero and unspecified.
+      - description: Number of replicas in the cluster
         displayName: Replica count
         path: replicas
       statusDescriptors:
@@ -124,7 +123,7 @@ spec:
       - description: ObservedGeneration indicates latest generation observed by controller.
         displayName: Observed Generation
         path: observedGeneration
-      - description: ReadyReplicas Total number of replicas ready to server requests.
+      - description: ReadyReplicas Total number of replicas ready to serve requests.
         displayName: Ready Replicas
         path: readyReplicas
       - description: StatefulSetRevision indicates target StatefulSet revision for
@@ -135,6 +134,9 @@ spec:
           revision.
         displayName: Update Revision
         path: updateRevision
+      - description: Version indicates the version reported by the container image.
+        displayName: Version
+        path: version
       version: v1alpha1
   description: |
     The ClickHouse Operator is a Kubernetes operator that automates the deployment, configuration, and management of ClickHouse clusters and ClickHouse Keeper clusters on Kubernetes.

Dockerfile.tilt → dev.Dockerfile

@@ -2,4 +2,6 @@ FROM ubuntu:latest
 
 RUN apt-get update && apt-get install -y ca-certificates
 
+USER 65532:65532
+
 ADD bin/manager_linux /manager

go.mod

@@ -20,7 +20,6 @@ require (
 	github.com/sethvargo/go-envconfig v1.3.0
 	github.com/testcontainers/testcontainers-go v0.41.0
 	go.uber.org/zap v1.27.1
-	golang.org/x/net v0.51.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.35.2
 	k8s.io/apimachinery v0.35.2
@@ -78,7 +77,6 @@ require (
 	github.com/google/cel-go v0.26.0 // indirect
 	github.com/google/gnostic-models v0.7.1 // indirect
 	github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc // indirect
-	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -136,6 +134,7 @@ require (
 	golang.org/x/crypto v0.48.0 // indirect
 	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect
 	golang.org/x/mod v0.33.0 // indirect
+	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect

go.sum

@@ -153,8 +153,6 @@ github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc h1:VBbFa1lDYWEeV5FZKU
 github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
-github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

internal/controller/clickhouse/commands.go

@@ -5,6 +5,8 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"net"
+	"strconv"
 	"sync"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
@@ -53,16 +55,18 @@ type commander struct {
 	log     controllerutil.Logger
 	cluster *v1.ClickHouseCluster
 	auth    clickhouse.Auth
+	dialer  controllerutil.DialContextFunc
 
 	lock  sync.RWMutex
 	conns map[v1.ClickHouseReplicaID]clickhouse.Conn
 }
 
-func newCommander(log controllerutil.Logger, cluster *v1.ClickHouseCluster, secret *corev1.Secret) *commander {
+func newCommander(log controllerutil.Logger, cluster *v1.ClickHouseCluster, secret *corev1.Secret, dialer controllerutil.DialContextFunc) *commander {
 	return &commander{
 		log:     log.Named("commander"),
 		conns:   map[v1.ClickHouseReplicaID]clickhouse.Conn{},
 		cluster: cluster,
+		dialer:  dialer,
 		auth: clickhouse.Auth{
 			Username: OperatorManagementUsername,
 			Password: string(secret.Data[SecretKeyManagementPassword]),
@@ -409,8 +413,9 @@ func (cmd *commander) getConn(id v1.ClickHouseReplicaID) (clickhouse.Conn, error
 	cmd.log.Debug("creating new ClickHouse connection", "replica_id", id)
 
 	conn, err := clickhouse.Open(&clickhouse.Options{
-		Addr: []string{fmt.Sprintf("%s:%d", cmd.cluster.HostnameByID(id), PortManagement)},
-		Auth: cmd.auth,
+		Addr:        []string{net.JoinHostPort(cmd.cluster.HostnameByID(id), strconv.FormatInt(int64(PortManagement), 10))},
+		Auth:        cmd.auth,
+		DialContext: cmd.dialer,
 		Debugf: func(format string, args ...any) {
 			cmd.log.Debug(fmt.Sprintf(format, args...))
 		},

internal/controller/clickhouse/commands_test.go

@@ -219,9 +219,11 @@ var _ = Describe("commander", Ordered, Label("integration"), func() {
 
 	It("should get every node version", func(ctx context.Context) {
 		for id := range cmd.cluster.ReplicaIDs() {
-			v0, err := cmd.Version(ctx, id)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(v0).To(MatchRegexp(`^\d+\.\d+\.\d+\.\d+$`))
+			Eventually(func(g Gomega) {
+				v, err := cmd.Version(ctx, id)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(v).To(MatchRegexp(`^\d+\.\d+\.\d+\.\d+$`))
+			}, "1m", "100ms").Should(Succeed())
 		}
 	})