feat: clickhouse replicas

charts/portkey-app/templates/_helpers.tpl

@@ -552,6 +552,40 @@ Template containing common environment variables that are used by several servic
 {{- end -}}
 {{- end -}}
 
+{{- define "clickhouse.replicationEnabled" -}}
+{{- if and (not .Values.clickhouse.external.enabled) (or .Values.clickhouseKeeper.enabled .Values.clickhouseKeeper.external.enabled) -}}
+true
+{{- end -}}
+{{- end -}}
+
+{{- define "clickhouse.shardingEnabled" -}}
+{{- if and (include "clickhouse.replicationEnabled" .) .Values.clickhouse.sharding.enabled -}}
+true
+{{- end -}}
+{{- end -}}
+
+{{- define "clickhouse.headlessServiceName" -}}
+{{- printf "%s-%s-headless" (include "portkey.fullname" .) .Values.clickhouse.name -}}
+{{- end -}}
+
+{{- define "clickhouseKeeper.deployEnabled" -}}
+{{- if and .Values.clickhouseKeeper.enabled (not .Values.clickhouseKeeper.external.enabled) -}}
+true
+{{- end -}}
+{{- end -}}
+
+{{- define "clickhouseKeeper.headlessServiceName" -}}
+{{- printf "%s-%s-headless" (include "portkey.fullname" .) .Values.clickhouseKeeper.name -}}
+{{- end -}}
+
+{{- define "clickhouseKeeper.serviceAccountName" -}}
+{{- if .Values.clickhouseKeeper.serviceAccount.create -}}
+    {{ default (printf "%s-%s" (include "portkey.fullname" .) .Values.clickhouseKeeper.name) .Values.clickhouseKeeper.serviceAccount.name | trunc 63 | trimSuffix "-" }}
+{{- else -}}
+    {{ default "default" .Values.clickhouseKeeper.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
 {{- define "mysql.serviceAccountName" -}}
 {{- if .Values.mysql.serviceAccount.create -}}
     {{ default (printf "%s-%s" (include "portkey.fullname" .) .Values.mysql.name) .Values.mysql.serviceAccount.name | trunc 63 | trimSuffix "-" }}

charts/portkey-app/templates/backend/deployment.yaml

@@ -100,6 +100,16 @@ spec:
             - name: MCP_GATEWAY_URL
               value: {{ .Values.mcp.externalUrl | quote }}
             {{- end }}
+            {{- if include "clickhouse.replicationEnabled" . }}
+            - name: CLICKHOUSE_REPLICATED
+              value: "true"
+            - name: CLICKHOUSE_CLUSTER_NAME
+              value: {{ .Values.clickhouse.clusterName | quote }}
+            {{- end }}
+            {{- if include "clickhouse.shardingEnabled" . }}
+            - name: CLICKHOUSE_SHARDING_ENABLED
+              value: "true"
+            {{- end }}
           image: "{{ .Values.images.backendImage.repository }}:{{ .Values.images.backendImage.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.images.backendImage.pullPolicy }}
           command: ["/bin/sh", "-c"]

charts/portkey-app/templates/clickhouse-keeper/config-map.yaml

@@ -0,0 +1,43 @@
+{{- if include "clickhouseKeeper.deployEnabled" . }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "portkey.fullname" . }}-{{ .Values.clickhouseKeeper.name }}
+  labels:
+    {{- include "portkey.labels" . | nindent 4 }}
+  annotations:
+    {{- include "portkey.annotations" . | nindent 4 }}
+data:
+  keeper_config.xml: |
+    <clickhouse>
+      <logger>
+        <level>information</level>
+        <console>1</console>
+      </logger>
+      <keeper_server>
+        <tcp_port>{{ .Values.clickhouseKeeper.containerPort }}</tcp_port>
+        <server_id>${SERVER_ID}</server_id>
+        <log_storage_path>/var/lib/clickhouse-keeper/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse-keeper/snapshots</snapshot_storage_path>
+        <coordination_settings>
+          <operation_timeout_ms>10000</operation_timeout_ms>
+          <session_timeout_ms>30000</session_timeout_ms>
+          <raft_logs_level>warning</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+          {{- $keeperReplicas := int .Values.clickhouseKeeper.replicas }}
+          {{- $fullname := include "portkey.fullname" . }}
+          {{- $keeperName := .Values.clickhouseKeeper.name }}
+          {{- $headlessSvc := include "clickhouseKeeper.headlessServiceName" . }}
+          {{- $raftPort := .Values.clickhouseKeeper.containerRaftPort }}
+          {{- range $i := until $keeperReplicas }}
+          <server>
+            <id>{{ add $i 1 }}</id>
+            <hostname>{{ $fullname }}-{{ $keeperName }}-{{ $i }}.{{ $headlessSvc }}</hostname>
+            <port>{{ $raftPort }}</port>
+          </server>
+          {{- end }}
+        </raft_configuration>
+      </keeper_server>
+    </clickhouse>
+{{- end }}

charts/portkey-app/templates/clickhouse-keeper/headless-service.yaml

@@ -0,0 +1,32 @@
+{{- if include "clickhouseKeeper.deployEnabled" . }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "clickhouseKeeper.headlessServiceName" . }}
+  labels:
+    {{- include "portkey.labels" . | nindent 4 }}
+    {{- with .Values.clickhouseKeeper.service.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- include "portkey.annotations" . | nindent 4 }}
+    {{- with .Values.clickhouseKeeper.service.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+    - name: keeper-client
+      port: {{ .Values.clickhouseKeeper.containerPort }}
+      targetPort: keeper-client
+      protocol: TCP
+    - name: keeper-raft
+      port: {{ .Values.clickhouseKeeper.containerRaftPort }}
+      targetPort: keeper-raft
+      protocol: TCP
+  selector:
+    {{- include "portkey.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: {{ include "portkey.fullname" . }}-{{ .Values.clickhouseKeeper.name }}
+{{- end }}

charts/portkey-app/templates/clickhouse-keeper/service-account.yaml

@@ -0,0 +1,16 @@
+{{- if and (include "clickhouseKeeper.deployEnabled" .) .Values.clickhouseKeeper.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "clickhouseKeeper.serviceAccountName" . }}
+  labels:
+    {{- include "portkey.labels" . | nindent 4 }}
+    {{- with .Values.clickhouseKeeper.serviceAccount.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- include "portkey.annotations" . | nindent 4 }}
+    {{- with .Values.clickhouseKeeper.serviceAccount.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+{{- end }}

charts/portkey-app/templates/clickhouse-keeper/stateful-set.yaml

@@ -0,0 +1,140 @@
+{{- if include "clickhouseKeeper.deployEnabled" . }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "portkey.fullname" . }}-{{ .Values.clickhouseKeeper.name }}
+  labels:
+    {{- include "portkey.labels" . | nindent 4 }}
+    {{- with .Values.clickhouseKeeper.statefulSet.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- include "portkey.annotations" . | nindent 4 }}
+    {{- with .Values.clickhouseKeeper.statefulSet.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  serviceName: {{ include "clickhouseKeeper.headlessServiceName" . }}
+  replicas: {{ .Values.clickhouseKeeper.replicas }}
+  selector:
+    matchLabels:
+      {{- include "portkey.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: {{ include "portkey.fullname" . }}-{{ .Values.clickhouseKeeper.name }}
+  template:
+    metadata:
+      {{- with .Values.clickhouseKeeper.statefulSet.annotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- with .Values.clickhouseKeeper.statefulSet.labels }}
+            {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- include "portkey.labels" . | nindent 8 }}
+        app.kubernetes.io/component: {{ include "portkey.fullname" . }}-{{ .Values.clickhouseKeeper.name }}
+    spec:
+      imagePullSecrets:
+      {{- range .Values.imageCredentials }}
+      - name: {{ .name }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.clickhouseKeeper.statefulSet.podSecurityContext | nindent 8 }}
+      serviceAccountName: {{ include "clickhouseKeeper.serviceAccountName" . }}
+      initContainers:
+        - name: generate-keeper-config
+          image: busybox:1.36
+          command:
+            - sh
+            - -c
+            - |
+              HOSTNAME=$(hostname)
+              SERVER_ID=$((${HOSTNAME##*-} + 1))
+              sed "s/\${SERVER_ID}/${SERVER_ID}/g" /etc/clickhouse-keeper-template/keeper_config.xml > /etc/clickhouse-keeper/keeper_config.xml
+          volumeMounts:
+            - name: keeper-config-template
+              mountPath: /etc/clickhouse-keeper-template
+            - name: keeper-config
+              mountPath: /etc/clickhouse-keeper
+      containers:
+        - name: {{ .Values.clickhouseKeeper.name }}
+          image: "{{ .Values.images.clickhouseImage.repository }}:{{ .Values.images.clickhouseImage.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.images.clickhouseImage.pullPolicy }}
+          command:
+            - clickhouse-keeper
+            - --config-file=/etc/clickhouse-keeper/keeper_config.xml
+          ports:
+            - name: keeper-client
+              containerPort: {{ .Values.clickhouseKeeper.containerPort }}
+              protocol: TCP
+            - name: keeper-raft
+              containerPort: {{ .Values.clickhouseKeeper.containerRaftPort }}
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command:
+                - bash
+                - -c
+                - echo ruok | nc localhost {{ .Values.clickhouseKeeper.containerPort }} | grep -q imok
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            exec:
+              command:
+                - bash
+                - -c
+                - echo ruok | nc localhost {{ .Values.clickhouseKeeper.containerPort }} | grep -q imok
+            initialDelaySeconds: 15
+            periodSeconds: 20
+            timeoutSeconds: 5
+          resources:
+            {{- toYaml .Values.clickhouseKeeper.statefulSet.resources | nindent 12 }}
+          securityContext:
+            {{- toYaml .Values.clickhouseKeeper.statefulSet.securityContext | nindent 12 }}
+          volumeMounts:
+            - name: keeper-config
+              mountPath: /etc/clickhouse-keeper
+            - name: keeper-data
+              mountPath: /var/lib/clickhouse-keeper
+      {{- with .Values.clickhouseKeeper.statefulSet.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.clickhouseKeeper.statefulSet.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.clickhouseKeeper.statefulSet.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: keeper-config-template
+          configMap:
+            name: {{ include "portkey.fullname" . }}-{{ .Values.clickhouseKeeper.name }}
+        - name: keeper-config
+          emptyDir: {}
+        {{- if not .Values.clickhouseKeeper.statefulSet.persistence.enabled }}
+        - name: keeper-data
+          emptyDir: {}
+        {{- end }}
+  {{- if .Values.clickhouseKeeper.statefulSet.persistence.enabled }}
+  volumeClaimTemplates:
+    - apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
+        name: keeper-data
+        labels:
+          {{- include "portkey.selectorLabels" . | nindent 10 }}
+          app.kubernetes.io/component: {{ include "portkey.fullname" . }}-{{ .Values.clickhouseKeeper.name }}
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: {{ .Values.clickhouseKeeper.statefulSet.persistence.size }}
+        {{- if .Values.clickhouseKeeper.statefulSet.persistence.storageClassName }}
+        storageClassName: {{ .Values.clickhouseKeeper.statefulSet.persistence.storageClassName }}
+        {{- end }}
+  {{- end }}
+{{- end }}

charts/portkey-app/templates/clickhouse/config-map.yaml

@@ -34,6 +34,62 @@ data:
         <ttl_only_drop_parts>{{ .Values.clickhouse.retention.ttlOnlyDropParts }}</ttl_only_drop_parts>
       </merge_tree>
       {{- end }}
+      {{- if include "clickhouse.replicationEnabled" . }}
+      <interserver_http_port>{{ .Values.clickhouse.containerInterserverPort }}</interserver_http_port>
+      <remote_servers>
+        <{{ .Values.clickhouse.clusterName }}>
+          {{- if include "clickhouse.shardingEnabled" . }}
+          {{- $shardsCount := int .Values.clickhouse.sharding.shardsCount }}
+          {{- $replicasPerShard := int .Values.clickhouse.sharding.replicasPerShard }}
+          {{- $fullname := include "portkey.fullname" . }}
+          {{- $chName := .Values.clickhouse.name }}
+          {{- $headlessSvc := include "clickhouse.headlessServiceName" . }}
+          {{- $nativePort := .Values.clickhouse.containerNativePort }}
+          {{- range $s := until $shardsCount }}
+          <shard>
+            <internal_replication>true</internal_replication>
+            {{- range $r := until $replicasPerShard }}
+            {{- $ordinal := add (mul $s $replicasPerShard) $r }}
+            <replica>
+              <host>{{ $fullname }}-{{ $chName }}-{{ $ordinal }}.{{ $headlessSvc }}</host>
+              <port>{{ $nativePort }}</port>
+            </replica>
+            {{- end }}
+          </shard>
+          {{- end }}
+          {{- else }}
+          <shard>
+            <internal_replication>true</internal_replication>
+            <discovery>
+              <path>/clickhouse/discovery/{{ .Values.clickhouse.clusterName }}</path>
+            </discovery>
+          </shard>
+          {{- end }}
+        </{{ .Values.clickhouse.clusterName }}>
+      </remote_servers>
+      <zookeeper>
+        {{- if .Values.clickhouseKeeper.external.enabled }}
+        {{- range .Values.clickhouseKeeper.external.hosts }}
+        <node>
+          <host>{{ .host }}</host>
+          <port>{{ .port }}</port>
+        </node>
+        {{- end }}
+        {{- else }}
+        {{- $keeperReplicas := int .Values.clickhouseKeeper.replicas }}
+        {{- $keeperHeadlessSvc := include "clickhouseKeeper.headlessServiceName" . }}
+        {{- $keeperFullname := include "portkey.fullname" . }}
+        {{- $keeperName := .Values.clickhouseKeeper.name }}
+        {{- $keeperPort := .Values.clickhouseKeeper.containerPort }}
+        {{- range $i := until $keeperReplicas }}
+        <node>
+          <host>{{ $keeperFullname }}-{{ $keeperName }}-{{ $i }}.{{ $keeperHeadlessSvc }}</host>
+          <port>{{ $keeperPort }}</port>
+        </node>
+        {{- end }}
+        {{- end }}
+      </zookeeper>
+      {{- end }}
     </clickhouse>
   {{- if not .Values.clickhouse.external.existingSecretName }}
   users.xml: |

charts/portkey-app/templates/clickhouse/headless-service.yaml

@@ -0,0 +1,36 @@
+{{- if include "clickhouse.replicationEnabled" . }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "clickhouse.headlessServiceName" . }}
+  labels:
+    {{- include "portkey.labels" . | nindent 4 }}
+    {{- with .Values.clickhouse.service.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- include "portkey.annotations" . | nindent 4 }}
+    {{- with .Values.clickhouse.service.annotations }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+    - name: ch
+      port: {{ .Values.clickhouse.service.httpPort }}
+      targetPort: ch
+      protocol: TCP
+    - name: ch-native
+      port: {{ .Values.clickhouse.service.nativePort }}
+      targetPort: ch-native
+      protocol: TCP
+    - name: ch-interserver
+      port: {{ .Values.clickhouse.containerInterserverPort }}
+      targetPort: ch-interserver
+      protocol: TCP
+  selector:
+    {{- include "portkey.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: {{ include "portkey.fullname" . }}-{{ .Values.clickhouse.name }}
+{{- end }}