fix: prevent StatefulSet infinite reconcile restart loop

internal/controller/clickhouse/constants.go

@@ -37,9 +37,10 @@
 	KeeperPathUDF            = "/clickhouse/user_defined"
 	KeeperPathDistributedDDL = "/clickhouse/task_queue/ddl"
 
-	ContainerName          = "clickhouse-server"
-	DefaultRevisionHistory = 10
+	ContainerName                        = "clickhouse-server"
+	DefaultRevisionHistory               = 10
+	DefaultTerminationGracePeriodSeconds = 30
 	MaximalAffinityWeight  = 100
 
 	InterserverUserName        = "interserver"
 	OperatorManagementUsername = "operator"

internal/controller/clickhouse/templates.go

@@ -222,8 +222,15 @@ func templateStatefulSet(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (*a
 		PodManagementPolicy: appsv1.ParallelPodManagement,
 		Replicas:            ptr.To[int32](1),
 		UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
-			Type:          appsv1.RollingUpdateStatefulSetStrategyType,
-			RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{},
+			Type: appsv1.RollingUpdateStatefulSetStrategyType,
+			RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
+				Partition:      ptr.To[int32](0),
+				MaxUnavailable: ptr.To(intstr.FromInt32(1)),
+			},
+		},
+		PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
+			WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
+			WhenScaled:  appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
 		},
 		Template: corev1.PodTemplateSpec{
 			ObjectMeta: metav1.ObjectMeta{
@@ -298,16 +305,34 @@ func templatePodSpec(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (corev1
 	}
 
 	podTemplate := cr.Spec.PodTemplate.DeepCopy()
+
+	// Set K8s-defaulted fields explicitly so the desired spec matches
+	// what the API server returns, preventing spurious reconcile diffs.
+	terminationGracePeriod := podTemplate.TerminationGracePeriodSeconds
+	if terminationGracePeriod == nil {
+		terminationGracePeriod = ptr.To[int64](DefaultTerminationGracePeriodSeconds)
+	}
+
+	schedulerName := podTemplate.SchedulerName
+	if schedulerName == "" {
+		schedulerName = corev1.DefaultSchedulerName
+	}
+
+	securityContext := podTemplate.SecurityContext
+	if securityContext == nil {
+		securityContext = &corev1.PodSecurityContext{}
+	}
+
 	podSpec := corev1.PodSpec{
-		TerminationGracePeriodSeconds: podTemplate.TerminationGracePeriodSeconds,
+		TerminationGracePeriodSeconds: terminationGracePeriod,
 		TopologySpreadConstraints:     podTemplate.TopologySpreadConstraints,
 		ImagePullSecrets:              podTemplate.ImagePullSecrets,
 		NodeSelector:                  podTemplate.NodeSelector,
 		Affinity:                      podTemplate.Affinity,
 		Tolerations:                   podTemplate.Tolerations,
-		SchedulerName:                 podTemplate.SchedulerName,
+		SchedulerName:                 schedulerName,
 		ServiceAccountName:            podTemplate.ServiceAccountName,
-		SecurityContext:               podTemplate.SecurityContext,
+		SecurityContext:               securityContext,
 		RestartPolicy:                 corev1.RestartPolicyAlways,
 		DNSPolicy:                     corev1.DNSClusterFirst,
 		Volumes:                       volumes,

internal/controller/constants.go

@@ -14,15 +14,21 @@ const (
 
 var (
 	// DefaultLivenessProbeSettings defines default settings for Kubernetes liveness probes.
+	// SuccessThreshold must be explicitly set to 1 (the K8s default) to prevent
+	// reconcile drift — omitting it produces a zero value that differs from what
+	// the API server returns, causing an infinite StatefulSet update loop.
 	//nolint: mnd // Magic numbers are used as constants.
 	DefaultLivenessProbeSettings = corev1.Probe{
 		InitialDelaySeconds: 60,
 		TimeoutSeconds:      10,
 		PeriodSeconds:       5,
+		SuccessThreshold:    1,
 		FailureThreshold:    10,
 	}
 
-	// DefaultReadinessProbeSettings defines default settings for Kubernetes liveness probes.
+	// DefaultReadinessProbeSettings defines default settings for Kubernetes readiness probes.
+	// SuccessThreshold is intentionally set to 5 (not the K8s default of 1) to
+	// require multiple consecutive successes before marking the pod ready.
 	//nolint: mnd // Magic numbers are used as constants.
 	DefaultReadinessProbeSettings = corev1.Probe{
 		InitialDelaySeconds: 5,

internal/controller/keeper/constants.go

@@ -29,9 +29,10 @@
 	StorageLogPath      = internal.KeeperDataPath + "/coordination/log/"
 	StorageSnapshotPath = internal.KeeperDataPath + "/coordination/snapshots/"
 
-	ContainerName          = "clickhouse-keeper"
-	DefaultRevisionHistory = 10
+	ContainerName                        = "clickhouse-keeper"
+	DefaultRevisionHistory               = 10
+	DefaultTerminationGracePeriodSeconds = 30
 	MaximalAffinityWeight  = 100
 )
 
 var breakingStatefulSetVersion, _ = semver.Parse("0.0.1")

internal/controller/keeper/templates.go

@@ -277,8 +277,15 @@ func templateStatefulSet(cr *v1.KeeperCluster, id v1.KeeperReplicaID) (*appsv1.S
 		PodManagementPolicy: appsv1.ParallelPodManagement,
 		Replicas:            ptr.To[int32](1),
 		UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
-			Type:          appsv1.RollingUpdateStatefulSetStrategyType,
-			RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{},
+			Type: appsv1.RollingUpdateStatefulSetStrategyType,
+			RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{
+				Partition:      ptr.To[int32](0),
+				MaxUnavailable: ptr.To(intstr.FromInt32(1)),
+			},
+		},
+		PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
+			WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
+			WhenScaled:  appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
 		},
 		Template: corev1.PodTemplateSpec{
 			ObjectMeta: metav1.ObjectMeta{
@@ -404,16 +411,34 @@ func templatePodSpec(cr *v1.KeeperCluster, id v1.KeeperReplicaID) (corev1.PodSpe
 	}
 
 	podTemplate := cr.Spec.PodTemplate.DeepCopy()
+
+	// Set K8s-defaulted fields explicitly so the desired spec matches
+	// what the API server returns, preventing spurious reconcile diffs.
+	terminationGracePeriod := podTemplate.TerminationGracePeriodSeconds
+	if terminationGracePeriod == nil {
+		terminationGracePeriod = ptr.To[int64](DefaultTerminationGracePeriodSeconds)
+	}
+
+	schedulerName := podTemplate.SchedulerName
+	if schedulerName == "" {
+		schedulerName = corev1.DefaultSchedulerName
+	}
+
+	securityContext := podTemplate.SecurityContext
+	if securityContext == nil {
+		securityContext = &corev1.PodSecurityContext{}
+	}
+
 	podSpec := corev1.PodSpec{
-		TerminationGracePeriodSeconds: podTemplate.TerminationGracePeriodSeconds,
+		TerminationGracePeriodSeconds: terminationGracePeriod,
 		TopologySpreadConstraints:     podTemplate.TopologySpreadConstraints,
 		ImagePullSecrets:              podTemplate.ImagePullSecrets,
 		NodeSelector:                  podTemplate.NodeSelector,
 		Affinity:                      podTemplate.Affinity,
 		Tolerations:                   podTemplate.Tolerations,
-		SchedulerName:                 podTemplate.SchedulerName,
+		SchedulerName:                 schedulerName,
 		ServiceAccountName:            podTemplate.ServiceAccountName,
-		SecurityContext:               podTemplate.SecurityContext,
+		SecurityContext:               securityContext,
 		RestartPolicy:                 corev1.RestartPolicyAlways,
 		DNSPolicy:                     corev1.DNSClusterFirst,
 		Volumes:                       volumes,

internal/controller/keeper/templates_test.go

@@ -7,6 +7,7 @@
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"gopkg.in/yaml.v2"
+	appsv1 "k8s.io/api/apps/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -230,6 +231,81 @@
 	})
 })
 
+var _ = Describe("templateStatefulSet K8s default alignment", func() {
+	var cr *v1.KeeperCluster
+
+	BeforeEach(func() {
+		cr = &v1.KeeperCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test",
+				Namespace: "default",
+			},
+			Spec: v1.KeeperClusterSpec{
+				Replicas: ptr.To[int32](1),
+			},
+		}
+	})
+
+	It("should set terminationGracePeriodSeconds to 30 when not specified", func() {
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(sts.Spec.Template.Spec.TerminationGracePeriodSeconds).ToNot(BeNil())
+		Expect(*sts.Spec.Template.Spec.TerminationGracePeriodSeconds).To(Equal(int64(30)))
+	})
+
+	It("should respect user-specified terminationGracePeriodSeconds", func() {
+		cr.Spec.PodTemplate.TerminationGracePeriodSeconds = ptr.To[int64](60)
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(*sts.Spec.Template.Spec.TerminationGracePeriodSeconds).To(Equal(int64(60)))
+	})
+
+	It("should set schedulerName to default-scheduler when not specified", func() {
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(sts.Spec.Template.Spec.SchedulerName).To(Equal("default-scheduler"))
+	})
+
+	It("should respect user-specified schedulerName", func() {
+		cr.Spec.PodTemplate.SchedulerName = "custom-scheduler"
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(sts.Spec.Template.Spec.SchedulerName).To(Equal("custom-scheduler"))
+	})
+
+	It("should set securityContext to empty struct when not specified", func() {
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(sts.Spec.Template.Spec.SecurityContext).ToNot(BeNil())
+	})
+
+	It("should set rollingUpdate partition and maxUnavailable", func() {
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(sts.Spec.UpdateStrategy.RollingUpdate).ToNot(BeNil())
+		Expect(sts.Spec.UpdateStrategy.RollingUpdate.Partition).ToNot(BeNil())
+		Expect(*sts.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal(int32(0)))
+		Expect(sts.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable).ToNot(BeNil())
+		Expect(sts.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable.IntValue()).To(Equal(1))
+	})
+
+	It("should set persistentVolumeClaimRetentionPolicy to Retain", func() {
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy).ToNot(BeNil())
+		Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted).To(Equal(appsv1.RetainPersistentVolumeClaimRetentionPolicyType))
+		Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled).To(Equal(appsv1.RetainPersistentVolumeClaimRetentionPolicyType))
+	})
+
+	It("should set liveness probe successThreshold to 1", func() {
+		sts, err := templateStatefulSet(cr, 0)
+		Expect(err).ToNot(HaveOccurred())
+		container := sts.Spec.Template.Spec.Containers[0]
+		Expect(container.LivenessProbe).ToNot(BeNil())
+		Expect(container.LivenessProbe.SuccessThreshold).To(Equal(int32(1)))
+	})
+})
+
 func FuzzClusterSpec(f *testing.F) {
 	// Manually added cases
 	f.Add([]byte("2"))