diff --git a/api/v1alpha1/clickhousecluster_types.go b/api/v1alpha1/clickhousecluster_types.go index 8ba4ddc..15f716f 100644 --- a/api/v1alpha1/clickhousecluster_types.go +++ b/api/v1alpha1/clickhousecluster_types.go @@ -77,6 +77,10 @@ type ClickHouseClusterSpec struct { // +optional // +kubebuilder:validation:Pattern=`^(lts|stable|\d+\.\d+)?$` UpgradeChannel string `json:"upgradeChannel,omitempty"` + + // VersionProbe defines configuration for the version detection Job. + // +optional + VersionProbe *VersionProbeSpec `json:"versionProbe,omitempty"` } // WithDefaults sets default values for ClickHouseClusterSpec fields. diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index 55bc7c6..dca7cec 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -210,6 +211,14 @@ type PodTemplateSpec struct { // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` + // PriorityClassName is the name of the PriorityClass to use for this pod. + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` + + // RuntimeClassName is the name of the RuntimeClass to use for this pod. + // +optional + RuntimeClassName string `json:"runtimeClassName,omitempty"` + // Volumes defines the list of volumes that can be mounted by containers belonging to the pod. // More info: https://kubernetes.io/docs/concepts/storage/volumes // +optional @@ -348,6 +357,56 @@ type DefaultPasswordSelector struct { ConfigMap *ConfigMapKeySelector `json:"configMap,omitempty"` } +const ( + // DefaultVersionProbeCPU is the default CPU request and limit for the version probe Job. + DefaultVersionProbeCPU = "100m" + // DefaultVersionProbeMemory is the default memory request and limit for the version probe Job. + DefaultVersionProbeMemory = "128Mi" +) + +// MetadataItem defines a single key-value pair for labels or annotations. +type MetadataItem struct { + // Key of the metadata item. + Key string `json:"key"` + // Value of the metadata item. + Value string `json:"value"` +} + +// VersionProbeSpec describes the configuration for the version probe task. +type VersionProbeSpec struct { + // Additional labels to apply specifically to the version probe Job and Pod. + // +optional + // +listType=map + // +listMapKey=key + Labels []MetadataItem `json:"labels,omitempty"` + + // Additional annotations to apply specifically to the version probe Job and Pod. + // +optional + // +listType=map + // +listMapKey=key + Annotations []MetadataItem `json:"annotations,omitempty"` + + // CPURequest defines the CPU request for the version probe container. + // +optional + // +kubebuilder:default:="100m" + CPURequest *resource.Quantity `json:"cpuRequest,omitempty"` + + // CPULimit defines the CPU limit for the version probe container. + // +optional + // +kubebuilder:default:="100m" + CPULimit *resource.Quantity `json:"cpuLimit,omitempty"` + + // MemoryRequest defines the memory request for the version probe container. + // +optional + // +kubebuilder:default:="128Mi" + MemoryRequest *resource.Quantity `json:"memoryRequest,omitempty"` + + // MemoryLimit defines the memory limit for the version probe container. + // +optional + // +kubebuilder:default:="128Mi" + MemoryLimit *resource.Quantity `json:"memoryLimit,omitempty"` +} + // Validate validates the DefaultPasswordSelector configuration. func (s *DefaultPasswordSelector) Validate() error { if s == nil { diff --git a/api/v1alpha1/keepercluster_types.go b/api/v1alpha1/keepercluster_types.go index 9c53096..de0105b 100644 --- a/api/v1alpha1/keepercluster_types.go +++ b/api/v1alpha1/keepercluster_types.go @@ -65,6 +65,10 @@ type KeeperClusterSpec struct { // +optional // +kubebuilder:validation:Pattern=`^(lts|stable|\d+\.\d+)?$` UpgradeChannel string `json:"upgradeChannel,omitempty"` + + // VersionProbe defines configuration for the version detection Job. + // +optional + VersionProbe *VersionProbeSpec `json:"versionProbe,omitempty"` } // WithDefaults sets default values for KeeperClusterSpec fields. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 65306d7..2e5fe79 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -132,6 +132,11 @@ func (in *ClickHouseClusterSpec) DeepCopyInto(out *ClickHouseClusterSpec) { (*in).DeepCopyInto(*out) } in.Settings.DeepCopyInto(&out.Settings) + if in.VersionProbe != nil { + in, out := &in.VersionProbe, &out.VersionProbe + *out = new(VersionProbeSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickHouseClusterSpec. @@ -400,6 +405,11 @@ func (in *KeeperClusterSpec) DeepCopyInto(out *KeeperClusterSpec) { (*in).DeepCopyInto(*out) } in.Settings.DeepCopyInto(&out.Settings) + if in.VersionProbe != nil { + in, out := &in.VersionProbe, &out.VersionProbe + *out = new(VersionProbeSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeeperClusterSpec. @@ -472,6 +482,21 @@ func (in *LoggerConfig) DeepCopy() *LoggerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetadataItem) DeepCopyInto(out *MetadataItem) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetadataItem. +func (in *MetadataItem) DeepCopy() *MetadataItem { + if in == nil { + return nil + } + out := new(MetadataItem) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodDisruptionBudgetSpec) DeepCopyInto(out *PodDisruptionBudgetSpec) { *out = *in @@ -589,3 +614,48 @@ func (in *SecretKeySelector) DeepCopy() *SecretKeySelector { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VersionProbeSpec) DeepCopyInto(out *VersionProbeSpec) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make([]MetadataItem, len(*in)) + copy(*out, *in) + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make([]MetadataItem, len(*in)) + copy(*out, *in) + } + if in.CPURequest != nil { + in, out := &in.CPURequest, &out.CPURequest + x := (*in).DeepCopy() + *out = &x + } + if in.CPULimit != nil { + in, out := &in.CPULimit, &out.CPULimit + x := (*in).DeepCopy() + *out = &x + } + if in.MemoryRequest != nil { + in, out := &in.MemoryRequest, &out.MemoryRequest + x := (*in).DeepCopy() + *out = &x + } + if in.MemoryLimit != nil { + in, out := &in.MemoryLimit, &out.MemoryLimit + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VersionProbeSpec. +func (in *VersionProbeSpec) DeepCopy() *VersionProbeSpec { + if in == nil { + return nil + } + out := new(VersionProbeSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/clickhouse.com_clickhouseclusters.yaml b/config/crd/bases/clickhouse.com_clickhouseclusters.yaml index 770801e..d33f63f 100644 --- a/config/crd/bases/clickhouse.com_clickhouseclusters.yaml +++ b/config/crd/bases/clickhouse.com_clickhouseclusters.yaml @@ -1802,6 +1802,14 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic + priorityClassName: + description: PriorityClassName is the name of the PriorityClass + to use for this pod. + type: string + runtimeClassName: + description: RuntimeClassName is the name of the RuntimeClass + to use for this pod. + type: string schedulerName: description: |- If specified, the pod will be dispatched by specified scheduler. @@ -4368,6 +4376,89 @@ spec: When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). pattern: ^(lts|stable|\d+\.\d+)?$ type: string + versionProbe: + description: VersionProbe defines configuration for the version detection + Job. + properties: + annotations: + description: Additional annotations to apply specifically to the + version probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for + labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + cpuLimit: + anyOf: + - type: integer + - type: string + default: 100m + description: CPULimit defines the CPU limit for the version probe + container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cpuRequest: + anyOf: + - type: integer + - type: string + default: 100m + description: CPURequest defines the CPU request for the version + probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + labels: + description: Additional labels to apply specifically to the version + probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for + labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + memoryLimit: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryLimit defines the memory limit for the version + probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryRequest: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryRequest defines the memory request for the + version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object required: - keeperClusterRef type: object diff --git a/config/crd/bases/clickhouse.com_keeperclusters.yaml b/config/crd/bases/clickhouse.com_keeperclusters.yaml index 74eed62..c172fc2 100644 --- a/config/crd/bases/clickhouse.com_keeperclusters.yaml +++ b/config/crd/bases/clickhouse.com_keeperclusters.yaml @@ -1787,6 +1787,14 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic + priorityClassName: + description: PriorityClassName is the name of the PriorityClass + to use for this pod. + type: string + runtimeClassName: + description: RuntimeClassName is the name of the RuntimeClass + to use for this pod. + type: string schedulerName: description: |- If specified, the pod will be dispatched by specified scheduler. @@ -4303,6 +4311,89 @@ spec: When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). pattern: ^(lts|stable|\d+\.\d+)?$ type: string + versionProbe: + description: VersionProbe defines configuration for the version detection + Job. + properties: + annotations: + description: Additional annotations to apply specifically to the + version probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for + labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + cpuLimit: + anyOf: + - type: integer + - type: string + default: 100m + description: CPULimit defines the CPU limit for the version probe + container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cpuRequest: + anyOf: + - type: integer + - type: string + default: 100m + description: CPURequest defines the CPU request for the version + probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + labels: + description: Additional labels to apply specifically to the version + probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for + labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + memoryLimit: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryLimit defines the memory limit for the version + probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryRequest: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryRequest defines the memory request for the + version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object type: object status: description: KeeperClusterStatus defines the observed state of KeeperCluster. diff --git a/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml b/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml index ef6e3db..a8ecc6c 100644 --- a/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/clickhouseclusters.clickhouse.com.yaml @@ -1726,6 +1726,12 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic + priorityClassName: + description: PriorityClassName is the name of the PriorityClass to use for this pod. + type: string + runtimeClassName: + description: RuntimeClassName is the name of the RuntimeClass to use for this pod. + type: string schedulerName: description: |- If specified, the pod will be dispatched by specified scheduler. @@ -4177,6 +4183,80 @@ spec: When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). pattern: ^(lts|stable|\d+\.\d+)?$ type: string + versionProbe: + description: VersionProbe defines configuration for the version detection Job. + properties: + annotations: + description: Additional annotations to apply specifically to the version probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + cpuLimit: + anyOf: + - type: integer + - type: string + default: 100m + description: CPULimit defines the CPU limit for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cpuRequest: + anyOf: + - type: integer + - type: string + default: 100m + description: CPURequest defines the CPU request for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + labels: + description: Additional labels to apply specifically to the version probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + memoryLimit: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryLimit defines the memory limit for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryRequest: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryRequest defines the memory request for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object required: - keeperClusterRef type: object diff --git a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml index 712fb08..ec04096 100644 --- a/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml +++ b/dist/chart/templates/crd/keeperclusters.clickhouse.com.yaml @@ -1712,6 +1712,12 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object x-kubernetes-map-type: atomic + priorityClassName: + description: PriorityClassName is the name of the PriorityClass to use for this pod. + type: string + runtimeClassName: + description: RuntimeClassName is the name of the RuntimeClass to use for this pod. + type: string schedulerName: description: |- If specified, the pod will be dispatched by specified scheduler. @@ -4120,6 +4126,80 @@ spec: When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). pattern: ^(lts|stable|\d+\.\d+)?$ type: string + versionProbe: + description: VersionProbe defines configuration for the version detection Job. + properties: + annotations: + description: Additional annotations to apply specifically to the version probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + cpuLimit: + anyOf: + - type: integer + - type: string + default: 100m + description: CPULimit defines the CPU limit for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + cpuRequest: + anyOf: + - type: integer + - type: string + default: 100m + description: CPURequest defines the CPU request for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + labels: + description: Additional labels to apply specifically to the version probe Job and Pod. + items: + description: MetadataItem defines a single key-value pair for labels or annotations. + properties: + key: + description: Key of the metadata item. + type: string + value: + description: Value of the metadata item. + type: string + required: + - key + - value + type: object + type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map + memoryLimit: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryLimit defines the memory limit for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryRequest: + anyOf: + - type: integer + - type: string + default: 128Mi + description: MemoryRequest defines the memory request for the version probe container. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object type: object status: description: KeeperClusterStatus defines the observed state of KeeperCluster. diff --git a/docs/api_reference.md b/docs/api_reference.md index ee0b653..502a0cf 100644 --- a/docs/api_reference.md +++ b/docs/api_reference.md @@ -57,6 +57,7 @@ ClickHouseClusterSpec defines the desired state of ClickHouseCluster. | `settings` | [ClickHouseSettings](#clickhousesettings) | Configuration parameters for ClickHouse server. | false | | | `clusterDomain` | string | ClusterDomain is the Kubernetes cluster domain suffix used for DNS resolution. | false | cluster.local | | `upgradeChannel` | string | UpgradeChannel specifies the release channel for major version upgrade checks.
When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). | false | | +| `versionProbe` | [VersionProbeSpec](#versionprobespec) | VersionProbe defines configuration for the version detection Job. | false | | Appears in: - [ClickHouseCluster](#clickhousecluster) @@ -228,6 +229,7 @@ KeeperClusterSpec defines the desired state of KeeperCluster. | `settings` | [KeeperSettings](#keepersettings) | Configuration parameters for ClickHouse Keeper server. | false | | | `clusterDomain` | string | ClusterDomain is the Kubernetes cluster domain suffix used for DNS resolution. | false | cluster.local | | `upgradeChannel` | string | UpgradeChannel specifies the release channel for major version upgrade checks.
When empty, only minor updates will be proposed. Allowed values are: stable, lts or specific major.minor version (e.g. 25.8). | false | | +| `versionProbe` | [VersionProbeSpec](#versionprobespec) | VersionProbe defines configuration for the version detection Job. | false | | Appears in: - [KeeperCluster](#keepercluster) @@ -284,6 +286,19 @@ Appears in: - [KeeperSettings](#keepersettings) +## MetadataItem + +MetadataItem defines a single key-value pair for labels or annotations. + +| Field | Type | Description | Required | Default | +|-------|------|-------------|----------|---------| +| `key` | string | Key of the metadata item. | true | | +| `value` | string | Value of the metadata item. | true | | + +Appears in: +- [VersionProbeSpec](#versionprobespec) + + ## PDBPolicy PDBPolicy controls whether PodDisruptionBudgets are created. @@ -330,6 +345,8 @@ PodTemplateSpec describes the pod configuration overrides for the cluster's pods | `tolerations` | [Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#toleration-v1-core) array | If specified, the pod's Tolerations. | false | | | `schedulerName` | string | If specified, the pod will be dispatched by specified scheduler.
If not specified, the pod will be dispatched by default scheduler. | false | | | `serviceAccountName` | string | ServiceAccountName is the name of the ServiceAccount to use to run this pod.
More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | false | | +| `priorityClassName` | string | PriorityClassName is the name of the PriorityClass to use for this pod. | false | | +| `runtimeClassName` | string | RuntimeClassName is the name of the RuntimeClass to use for this pod. | false | | | `volumes` | [Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#volume-v1-core) array | Volumes defines the list of volumes that can be mounted by containers belonging to the pod.
More info: https://kubernetes.io/docs/concepts/storage/volumes | false | | | `securityContext` | [PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podsecuritycontext-v1-core) | SecurityContext holds pod-level security attributes and common container settings. | false | | | `topologyZoneKey` | string | TopologyZoneKey is the key of node labels.
Nodes that have a label with this key and identical values are considered to be in the same topology zone.
Set it to enable default TopologySpreadConstraints and Affinity rules to spread pods across zones.
Recommended to be set to "topology.kubernetes.io/zone" | false | | @@ -353,3 +370,21 @@ Appears in: - [ClusterTLSSpec](#clustertlsspec) - [DefaultPasswordSelector](#defaultpasswordselector) + +## VersionProbeSpec + +VersionProbeSpec describes the configuration for the version probe task. + +| Field | Type | Description | Required | Default | +|-------|------|-------------|----------|---------| +| `labels` | [MetadataItem](#metadataitem) array | Additional labels to apply specifically to the version probe Job and Pod. | false | | +| `annotations` | [MetadataItem](#metadataitem) array | Additional annotations to apply specifically to the version probe Job and Pod. | false | | +| `cpuRequest` | [Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api) | CPURequest defines the CPU request for the version probe container. | false | 100m | +| `cpuLimit` | [Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api) | CPULimit defines the CPU limit for the version probe container. | false | 100m | +| `memoryRequest` | [Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api) | MemoryRequest defines the memory request for the version probe container. | false | 128Mi | +| `memoryLimit` | [Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#quantity-resource-api) | MemoryLimit defines the memory limit for the version probe container. | false | 128Mi | + +Appears in: +- [ClickHouseClusterSpec](#clickhouseclusterspec) +- [KeeperClusterSpec](#keeperclusterspec) + diff --git a/docs/styles/config/vocabularies/ClickHouse/accept.txt b/docs/styles/config/vocabularies/ClickHouse/accept.txt index 5932baf..d403b4a 100644 --- a/docs/styles/config/vocabularies/ClickHouse/accept.txt +++ b/docs/styles/config/vocabularies/ClickHouse/accept.txt @@ -15,3 +15,7 @@ Env ANDed boolean lts +CPURequest +CPULimit +MemoryRequest +MemoryLimit diff --git a/internal/controller/clickhouse/controller_test.go b/internal/controller/clickhouse/controller_test.go index 3da9542..0f8ac50 100644 --- a/internal/controller/clickhouse/controller_test.go +++ b/internal/controller/clickhouse/controller_test.go @@ -60,6 +60,13 @@ var _ = When("reconciling ClickHouseCluster", Ordered, func() { Annotations: map[string]string{ "test-annotation": "test-val", }, + VersionProbe: &v1.VersionProbeSpec{ + Annotations: []v1.MetadataItem{ + {Key: "sidecar.istio.io/inject", Value: "false"}, + }, + CPURequest: ptr.To(resource.MustParse("200m")), + CPULimit: ptr.To(resource.MustParse("200m")), + }, }, } ) @@ -134,9 +141,25 @@ var _ = When("reconciling ClickHouseCluster", Ordered, func() { Expect(suite.Client.List(ctx, &statefulsets, listOpts)).To(Succeed()) Expect(statefulsets.Items).To(HaveLen(4)) + By("verifying version probe job isolation") Expect(suite.Client.List(ctx, &jobs, listOpts)).To(Succeed()) + Expect(jobs.Items).To(BeEmpty()) // Should not be found with cluster app label + + By("verifying version probe job metadata") + Expect(suite.Client.List(ctx, &jobs, client.InNamespace(cr.Namespace), client.MatchingLabels{ + controllerutil.LabelAppKey: cr.SpecificName() + "-version-probe", + controllerutil.LabelRoleKey: controllerutil.LabelVersionProbe, + })).To(Succeed()) Expect(jobs.Items).To(HaveLen(1)) - Expect(jobs.Items[0].Labels[controllerutil.LabelRoleKey]).To(Equal(controllerutil.LabelVersionProbe)) + Expect(jobs.Items[0].Annotations).To(HaveKeyWithValue("sidecar.istio.io/inject", "false")) + Expect(jobs.Items[0].Spec.Template.Annotations).To(HaveKeyWithValue("sidecar.istio.io/inject", "false")) + + By("verifying version probe job resources") + + container := jobs.Items[0].Spec.Template.Spec.Containers[0] + Expect(container.Resources.Requests.Cpu().String()).To(Equal("200m")) // Overridden + Expect(container.Resources.Limits.Cpu().String()).To(Equal("200m")) // Overridden + Expect(container.Resources.Limits.Memory().String()).To(Equal("128Mi")) // Default testutil.AssertEvents(recorder.Events, map[string]int{ "ClusterNotReady": 1, diff --git a/internal/controller/clickhouse/sync.go b/internal/controller/clickhouse/sync.go index ca90aab..eeeb3ec 100644 --- a/internal/controller/clickhouse/sync.go +++ b/internal/controller/clickhouse/sync.go @@ -341,6 +341,7 @@ func (r *clickhouseReconciler) reconcileClusterRevisions(ctx context.Context, lo Annotations: r.Cluster.Spec.Annotations, PodTemplate: r.Cluster.Spec.PodTemplate, ContainerTemplate: r.Cluster.Spec.ContainerTemplate, + VersionProbe: r.Cluster.Spec.VersionProbe, }) if err != nil { return nil, fmt.Errorf("run version probe: %w", err) diff --git a/internal/controller/clickhouse/templates.go b/internal/controller/clickhouse/templates.go index 7654f1e..c52fe74 100644 --- a/internal/controller/clickhouse/templates.go +++ b/internal/controller/clickhouse/templates.go @@ -309,6 +309,8 @@ func templatePodSpec(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (corev1 Tolerations: podTemplate.Tolerations, SchedulerName: podTemplate.SchedulerName, ServiceAccountName: podTemplate.ServiceAccountName, + PriorityClassName: podTemplate.PriorityClassName, + RuntimeClassName: controllerutil.ToPtrOrNil(podTemplate.RuntimeClassName), SecurityContext: podTemplate.SecurityContext, RestartPolicy: corev1.RestartPolicyAlways, DNSPolicy: corev1.DNSClusterFirst, diff --git a/internal/controller/keeper/controller_test.go b/internal/controller/keeper/controller_test.go index 4c5be9c..d5faa75 100644 --- a/internal/controller/keeper/controller_test.go +++ b/internal/controller/keeper/controller_test.go @@ -11,6 +11,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -18,6 +19,7 @@ import ( "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" v1 "github.com/ClickHouse/clickhouse-operator/api/v1alpha1" "github.com/ClickHouse/clickhouse-operator/internal/controller/testutil" @@ -54,6 +56,13 @@ var _ = When("reconciling standalone KeeperCluster resource", Ordered, func() { Annotations: map[string]string{ "test-annotation": "test-val", }, + VersionProbe: &v1.VersionProbeSpec{ + Annotations: []v1.MetadataItem{ + {Key: "sidecar.istio.io/inject", Value: "false"}, + }, + CPURequest: ptr.To(resource.MustParse("200m")), + CPULimit: ptr.To(resource.MustParse("200m")), + }, }, } ) @@ -110,9 +119,25 @@ var _ = When("reconciling standalone KeeperCluster resource", Ordered, func() { Expect(suite.Client.List(ctx, &statefulsets, listOpts)).To(Succeed()) Expect(statefulsets.Items).To(HaveLen(1)) + By("verifying version probe job isolation") Expect(suite.Client.List(ctx, &jobs, listOpts)).To(Succeed()) + Expect(jobs.Items).To(BeEmpty()) // Should not be found with cluster app label + + By("verifying version probe job metadata") + Expect(suite.Client.List(ctx, &jobs, client.InNamespace(cr.Namespace), client.MatchingLabels{ + controllerutil.LabelAppKey: cr.SpecificName() + "-version-probe", + controllerutil.LabelRoleKey: controllerutil.LabelVersionProbe, + })).To(Succeed()) Expect(jobs.Items).To(HaveLen(1)) - Expect(jobs.Items[0].Labels[controllerutil.LabelRoleKey]).To(Equal(controllerutil.LabelVersionProbe)) + Expect(jobs.Items[0].Annotations).To(HaveKeyWithValue("sidecar.istio.io/inject", "false")) + Expect(jobs.Items[0].Spec.Template.Annotations).To(HaveKeyWithValue("sidecar.istio.io/inject", "false")) + + By("verifying version probe job resources") + + container := jobs.Items[0].Spec.Template.Spec.Containers[0] + Expect(container.Resources.Requests.Cpu().String()).To(Equal("200m")) // Overridden + Expect(container.Resources.Limits.Cpu().String()).To(Equal("200m")) // Overridden + Expect(container.Resources.Limits.Memory().String()).To(Equal("128Mi")) // Default testutil.AssertEvents(recorder.Events, map[string]int{ "ReplicaCreated": 1, diff --git a/internal/controller/keeper/sync.go b/internal/controller/keeper/sync.go index 018dde7..3f721e9 100644 --- a/internal/controller/keeper/sync.go +++ b/internal/controller/keeper/sync.go @@ -249,6 +249,7 @@ func (r *keeperReconciler) reconcileClusterRevisions(ctx context.Context, log ct Annotations: r.Cluster.Spec.Annotations, PodTemplate: r.Cluster.Spec.PodTemplate, ContainerTemplate: r.Cluster.Spec.ContainerTemplate, + VersionProbe: r.Cluster.Spec.VersionProbe, }) if err != nil { return nil, fmt.Errorf("run version probe: %w", err) diff --git a/internal/controller/keeper/templates.go b/internal/controller/keeper/templates.go index f6d8576..4159082 100644 --- a/internal/controller/keeper/templates.go +++ b/internal/controller/keeper/templates.go @@ -415,6 +415,8 @@ func templatePodSpec(cr *v1.KeeperCluster, id v1.KeeperReplicaID) (corev1.PodSpe Tolerations: podTemplate.Tolerations, SchedulerName: podTemplate.SchedulerName, ServiceAccountName: podTemplate.ServiceAccountName, + PriorityClassName: podTemplate.PriorityClassName, + RuntimeClassName: controllerutil.ToPtrOrNil(podTemplate.RuntimeClassName), SecurityContext: podTemplate.SecurityContext, RestartPolicy: corev1.RestartPolicyAlways, DNSPolicy: corev1.DNSClusterFirst, diff --git a/internal/controller/versionprobe.go b/internal/controller/versionprobe.go index 3815850..25e2a69 100644 --- a/internal/controller/versionprobe.go +++ b/internal/controller/versionprobe.go @@ -10,6 +10,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -36,6 +37,8 @@ type VersionProbeConfig struct { PodTemplate v1.PodTemplateSpec // ContainerTemplate to apply to the Job, inherited from the cluster spec. ContainerTemplate v1.ContainerTemplateSpec + // VersionProbe defines configuration for the version detection Job. + VersionProbe *v1.VersionProbeSpec } // VersionProbeResult holds the outcome of a version probe reconciliation. @@ -160,15 +163,18 @@ func (r *ResourceReconcilerBase[Status, T, ReplicaID, S]) cleanupVersionProbeJob var jobs batchv1.JobList if err := cli.List(ctx, &jobs, client.InNamespace(r.Cluster.GetNamespace()), client.MatchingLabels(map[string]string{ - controllerutil.LabelAppKey: r.Cluster.SpecificName(), controllerutil.LabelRoleKey: controllerutil.LabelVersionProbe, })); err != nil { - log.Warn("failed to list obsolete version probe jobs", "error", err) + log.Warn("failed to list version probe jobs", "error", err) return } + oldAppLabel := r.Cluster.SpecificName() + newAppLabel := r.Cluster.SpecificName() + "-version-probe" + for _, j := range jobs.Items { - if j.Name != job.Name { + app := j.Labels[controllerutil.LabelAppKey] + if j.Name != job.Name && (app == oldAppLabel || app == newAppLabel) { log.Debug("deleting obsolete version probe job", "job", j.Name) if err := cli.Delete(ctx, &j, client.PropagationPolicy(metav1.DeletePropagationBackground)); err != nil { @@ -179,32 +185,79 @@ func (r *ResourceReconcilerBase[Status, T, ReplicaID, S]) cleanupVersionProbeJob } func (r *ResourceReconcilerBase[Status, T, ReplicaID, S]) buildVersionProbeJob(cfg VersionProbeConfig) (batchv1.Job, error) { + labels := controllerutil.MergeMaps(cfg.Labels, map[string]string{ + controllerutil.LabelAppKey: r.Cluster.SpecificName() + "-version-probe", + controllerutil.LabelRoleKey: controllerutil.LabelVersionProbe, + }) + annotations := cfg.Annotations + + resources := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(v1.DefaultVersionProbeCPU), + corev1.ResourceMemory: resource.MustParse(v1.DefaultVersionProbeMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(v1.DefaultVersionProbeCPU), + corev1.ResourceMemory: resource.MustParse(v1.DefaultVersionProbeMemory), + }, + } + + if cfg.VersionProbe != nil { + labels = controllerutil.MergeMaps(labels, controllerutil.MetadataToMap(cfg.VersionProbe.Labels, + func(m v1.MetadataItem) string { return m.Key }, + func(m v1.MetadataItem) string { return m.Value })) + annotations = controllerutil.MergeMaps(annotations, controllerutil.MetadataToMap(cfg.VersionProbe.Annotations, + func(m v1.MetadataItem) string { return m.Key }, + func(m v1.MetadataItem) string { return m.Value })) + + if cfg.VersionProbe.CPURequest != nil { + resources.Requests[corev1.ResourceCPU] = *cfg.VersionProbe.CPURequest + } + + if cfg.VersionProbe.MemoryRequest != nil { + resources.Requests[corev1.ResourceMemory] = *cfg.VersionProbe.MemoryRequest + } + + if cfg.VersionProbe.CPULimit != nil { + resources.Limits[corev1.ResourceCPU] = *cfg.VersionProbe.CPULimit + } + + if cfg.VersionProbe.MemoryLimit != nil { + resources.Limits[corev1.ResourceMemory] = *cfg.VersionProbe.MemoryLimit + } + } + job := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ - Namespace: r.Cluster.GetNamespace(), - Labels: controllerutil.MergeMaps(cfg.Labels, map[string]string{ - controllerutil.LabelAppKey: r.Cluster.SpecificName(), - controllerutil.LabelRoleKey: controllerutil.LabelVersionProbe, - }), - Annotations: cfg.Annotations, + Namespace: r.Cluster.GetNamespace(), + Labels: labels, + Annotations: annotations, }, Spec: batchv1.JobSpec{ BackoffLimit: ptr.To[int32](0), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: cfg.Labels, - Annotations: cfg.Annotations, + Labels: labels, + Annotations: annotations, }, Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - ImagePullSecrets: cfg.PodTemplate.ImagePullSecrets, - SecurityContext: cfg.PodTemplate.SecurityContext, + RestartPolicy: corev1.RestartPolicyNever, + ImagePullSecrets: cfg.PodTemplate.ImagePullSecrets, + SecurityContext: cfg.PodTemplate.SecurityContext, + NodeSelector: cfg.PodTemplate.NodeSelector, + Tolerations: cfg.PodTemplate.Tolerations, + Affinity: cfg.PodTemplate.Affinity, + ServiceAccountName: cfg.PodTemplate.ServiceAccountName, + PriorityClassName: cfg.PodTemplate.PriorityClassName, + RuntimeClassName: controllerutil.ToPtrOrNil(cfg.PodTemplate.RuntimeClassName), + TopologySpreadConstraints: cfg.PodTemplate.TopologySpreadConstraints, Containers: []corev1.Container{ { Name: versionProbeContainerName, Image: cfg.ContainerTemplate.Image.String(), ImagePullPolicy: cfg.ContainerTemplate.ImagePullPolicy, SecurityContext: cfg.ContainerTemplate.SecurityContext, + Resources: resources, Command: []string{"sh", "-c", cfg.Binary + " --version > /dev/termination-log 2>&1"}, }, }, diff --git a/internal/controllerutil/common.go b/internal/controllerutil/common.go index 792c6a6..2d7f72e 100644 --- a/internal/controllerutil/common.go +++ b/internal/controllerutil/common.go @@ -284,6 +284,29 @@ func ShouldEmitEvent(err error) bool { return false } +// MetadataToMap converts a slice of metadata items (labels or annotations) to a map. +func MetadataToMap[T any](items []T, getKey func(T) string, getValue func(T) string) map[string]string { + if len(items) == 0 { + return nil + } + + res := make(map[string]string, len(items)) + for _, item := range items { + res[getKey(item)] = getValue(item) + } + + return res +} + +// ToPtrOrNil returns a pointer to the given string, or nil if the string is empty. +func ToPtrOrNil(s string) *string { + if s == "" { + return nil + } + + return &s +} + var versionRegex = regexp.MustCompile(`\d+(\.\d+){2,3}`) // ParseVersion extracts a version string from the given raw input.