Skip to content

Commit b7d4b4d

Browse files
jianzhangbjzjianzhangbjz
committed
sets a sensible default of **2 replicas** and update upgrade e2e test cases
1 parent c85bc55 commit b7d4b4d

13 files changed

+171
-57
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -484,8 +484,8 @@ run-experimental: run-internal #HELP Build the operator-controller then deploy i
484484
CATD_NAMESPACE := olmv1-system
485485
.PHONY: wait
486486
wait:
487-
kubectl wait --for=condition=Available --namespace=$(CATD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s
488-
kubectl wait --for=condition=Ready --namespace=$(CATD_NAMESPACE) certificate/catalogd-service-cert # Avoid upgrade test flakes when reissuing cert
487+
kubectl wait --for=condition=Available --namespace=$(CATD_NAMESPACE) deployment/catalogd-controller-manager --timeout=3m
488+
kubectl wait --for=condition=Ready --namespace=$(CATD_NAMESPACE) certificate/catalogd-service-cert --timeout=3m # Avoid upgrade test flakes when reissuing cert
489489

490490
.PHONY: docker-build
491491
docker-build: build-linux #EXHELP Build docker image for operator-controller and catalog with GOOS=linux and local GOARCH.

‎hack/test/pre-upgrade-setup.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,5 +155,5 @@ spec:
155155
version: 1.0.0
156156
EOF
157157

158-
kubectl wait --for=condition=Serving --timeout=60s ClusterCatalog $TEST_CLUSTER_CATALOG_NAME
159-
kubectl wait --for=condition=Installed --timeout=60s ClusterExtension $TEST_CLUSTER_EXTENSION_NAME
158+
kubectl wait --for=condition=Serving --timeout=5m ClusterCatalog $TEST_CLUSTER_CATALOG_NAME
159+
kubectl wait --for=condition=Installed --timeout=5m ClusterExtension $TEST_CLUSTER_EXTENSION_NAME

‎helm/olmv1/templates/deployment-olmv1-system-catalogd-controller-manager.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@ metadata:
1212
namespace: {{ .Values.namespaces.olmv1.name }}
1313
spec:
1414
minReadySeconds: 5
15-
replicas: 1
15+
replicas: {{ .Values.options.catalogd.deployment.replicas }}
1616
strategy:
1717
type: RollingUpdate
1818
rollingUpdate:
19-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
19+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
2020
maxUnavailable: 0 # Never allow pods to be unavailable during updates
2121
selector:
2222
matchLabels:

‎helm/olmv1/templates/deployment-olmv1-system-operator-controller-controller-manager.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,11 @@ metadata:
1111
name: operator-controller-controller-manager
1212
namespace: {{ .Values.namespaces.olmv1.name }}
1313
spec:
14-
replicas: 1
14+
replicas: {{ .Values.options.operatorController.deployment.replicas }}
1515
strategy:
1616
type: RollingUpdate
1717
rollingUpdate:
18-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
18+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
1919
maxUnavailable: 0 # Never allow pods to be unavailable during updates
2020
selector:
2121
matchLabels:

‎helm/olmv1/values.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ options:
88
enabled: true
99
deployment:
1010
image: quay.io/operator-framework/operator-controller:devel
11+
replicas: 2
1112
extraArguments: []
1213
features:
1314
enabled: []
@@ -19,6 +20,7 @@ options:
1920
enabled: true
2021
deployment:
2122
image: quay.io/operator-framework/catalogd:devel
23+
replicas: 2
2224
extraArguments: []
2325
features:
2426
enabled: []

‎manifests/experimental-e2e.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2107,11 +2107,11 @@ metadata:
21072107
namespace: olmv1-system
21082108
spec:
21092109
minReadySeconds: 5
2110-
replicas: 1
2110+
replicas: 2
21112111
strategy:
21122112
type: RollingUpdate
21132113
rollingUpdate:
2114-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
2114+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
21152115
maxUnavailable: 0 # Never allow pods to be unavailable during updates
21162116
selector:
21172117
matchLabels:
@@ -2258,11 +2258,11 @@ metadata:
22582258
name: operator-controller-controller-manager
22592259
namespace: olmv1-system
22602260
spec:
2261-
replicas: 1
2261+
replicas: 2
22622262
strategy:
22632263
type: RollingUpdate
22642264
rollingUpdate:
2265-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
2265+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
22662266
maxUnavailable: 0 # Never allow pods to be unavailable during updates
22672267
selector:
22682268
matchLabels:

‎manifests/experimental.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2032,11 +2032,11 @@ metadata:
20322032
namespace: olmv1-system
20332033
spec:
20342034
minReadySeconds: 5
2035-
replicas: 1
2035+
replicas: 2
20362036
strategy:
20372037
type: RollingUpdate
20382038
rollingUpdate:
2039-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
2039+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
20402040
maxUnavailable: 0 # Never allow pods to be unavailable during updates
20412041
selector:
20422042
matchLabels:
@@ -2170,11 +2170,11 @@ metadata:
21702170
name: operator-controller-controller-manager
21712171
namespace: olmv1-system
21722172
spec:
2173-
replicas: 1
2173+
replicas: 2
21742174
strategy:
21752175
type: RollingUpdate
21762176
rollingUpdate:
2177-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
2177+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
21782178
maxUnavailable: 0 # Never allow pods to be unavailable during updates
21792179
selector:
21802180
matchLabels:

‎manifests/standard-e2e.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1795,11 +1795,11 @@ metadata:
17951795
namespace: olmv1-system
17961796
spec:
17971797
minReadySeconds: 5
1798-
replicas: 1
1798+
replicas: 2
17991799
strategy:
18001800
type: RollingUpdate
18011801
rollingUpdate:
1802-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
1802+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
18031803
maxUnavailable: 0 # Never allow pods to be unavailable during updates
18041804
selector:
18051805
matchLabels:
@@ -1945,11 +1945,11 @@ metadata:
19451945
name: operator-controller-controller-manager
19461946
namespace: olmv1-system
19471947
spec:
1948-
replicas: 1
1948+
replicas: 2
19491949
strategy:
19501950
type: RollingUpdate
19511951
rollingUpdate:
1952-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
1952+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
19531953
maxUnavailable: 0 # Never allow pods to be unavailable during updates
19541954
selector:
19551955
matchLabels:

‎manifests/standard.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1720,11 +1720,11 @@ metadata:
17201720
namespace: olmv1-system
17211721
spec:
17221722
minReadySeconds: 5
1723-
replicas: 1
1723+
replicas: 2
17241724
strategy:
17251725
type: RollingUpdate
17261726
rollingUpdate:
1727-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
1727+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
17281728
maxUnavailable: 0 # Never allow pods to be unavailable during updates
17291729
selector:
17301730
matchLabels:
@@ -1857,11 +1857,11 @@ metadata:
18571857
name: operator-controller-controller-manager
18581858
namespace: olmv1-system
18591859
spec:
1860-
replicas: 1
1860+
replicas: 2
18611861
strategy:
18621862
type: RollingUpdate
18631863
rollingUpdate:
1864-
maxSurge: 1 # Allow temporary 2 pods (1 + 1) for zero-downtime updates
1864+
maxSurge: 1 # Allow temporary extra pod (e.g., 3 pods with 2 replicas) for zero-downtime updates
18651865
maxUnavailable: 0 # Never allow pods to be unavailable during updates
18661866
selector:
18671867
matchLabels:

‎test/e2e/cluster_extension_install_test.go‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ import (
2727
)
2828

2929
const (
30-
artifactName = "operator-controller-e2e"
31-
pollDuration = time.Minute
30+
artifactName = "operator-controller-e2e"
31+
// pollDuration is set to 3 minutes to account for leader election time in multi-replica deployments.
32+
// In the worst case (previous leader crashed), leader election can take up to 163 seconds
33+
// (LeaseDuration: 137s + RetryPeriod: 26s). Adding buffer for reconciliation time.
34+
pollDuration = 3 * time.Minute
3235
pollInterval = time.Second
3336
testCatalogRefEnvVar = "CATALOG_IMG"
3437
testCatalogName = "test-catalog"
@@ -169,18 +172,19 @@ location = "docker-registry.operator-controller-e2e.svc.cluster.local:5000"`,
169172
require.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
170173
}, 2*time.Minute, pollInterval)
171174

172-
// Give the check 2 minutes instead of the typical 1 for the pod's
173-
// files to update from the configmap change.
175+
// Give the check extra time for the pod's files to update from the configmap change.
174176
// The theoretical max time is the kubelet sync period of 1 minute +
175-
// ConfigMap cache TTL of 1 minute = 2 minutes
177+
// ConfigMap cache TTL of 1 minute = 2 minutes.
178+
// With multi-replica deployments, add leader election time (up to 163s in worst case).
179+
// Total: 2 min (ConfigMap) + 2.7 min (leader election) + buffer = 5 minutes
176180
t.Log("By eventually reporting progressing as True")
177181
require.EventuallyWithT(t, func(ct *assert.CollectT) {
178182
require.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
179183
cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeProgressing)
180184
require.NotNil(ct, cond)
181185
require.Equal(ct, metav1.ConditionTrue, cond.Status)
182186
require.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
183-
}, 2*time.Minute, pollInterval)
187+
}, 5*time.Minute, pollInterval)
184188

185189
t.Log("By eventually installing the package successfully")
186190
require.EventuallyWithT(t, func(ct *assert.CollectT) {
@@ -655,6 +659,8 @@ func TestClusterExtensionRecoversFromNoNamespaceWhenFailureFixed(t *testing.T) {
655659
// backoff of this eventually check we MUST ensure we do not touch the ClusterExtension
656660
// after creating int the Namespace and ServiceAccount.
657661
t.Log("By eventually installing the package successfully")
662+
// Use 5 minutes for recovery tests to account for exponential backoff after repeated failures
663+
// plus leader election time (up to 163s in worst case)
658664
require.EventuallyWithT(t, func(ct *assert.CollectT) {
659665
require.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
660666
cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled)
@@ -663,7 +669,7 @@ func TestClusterExtensionRecoversFromNoNamespaceWhenFailureFixed(t *testing.T) {
663669
require.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
664670
require.Contains(ct, cond.Message, "Installed bundle")
665671
require.NotEmpty(ct, clusterExtension.Status.Install)
666-
}, pollDuration, pollInterval)
672+
}, 5*time.Minute, pollInterval)
667673

668674
t.Log("By eventually reporting Progressing == True with Reason Success")
669675
require.EventuallyWithT(t, func(ct *assert.CollectT) {
@@ -777,6 +783,8 @@ func TestClusterExtensionRecoversFromExistingDeploymentWhenFailureFixed(t *testi
777783
// backoff of this eventually check we MUST ensure we do not touch the ClusterExtension
778784
// after deleting the Deployment.
779785
t.Log("By eventually installing the package successfully")
786+
// Use 5 minutes for recovery tests to account for exponential backoff after repeated failures
787+
// plus leader election time (up to 163s in worst case)
780788
require.EventuallyWithT(t, func(ct *assert.CollectT) {
781789
require.NoError(ct, c.Get(context.Background(), types.NamespacedName{Name: clusterExtension.Name}, clusterExtension))
782790
cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1.TypeInstalled)
@@ -785,7 +793,7 @@ func TestClusterExtensionRecoversFromExistingDeploymentWhenFailureFixed(t *testi
785793
require.Equal(ct, ocv1.ReasonSucceeded, cond.Reason)
786794
require.Contains(ct, cond.Message, "Installed bundle")
787795
require.NotEmpty(ct, clusterExtension.Status.Install)
788-
}, pollDuration, pollInterval)
796+
}, 5*time.Minute, pollInterval)
789797

790798
t.Log("By eventually reporting Progressing == True with Reason Success")
791799
require.EventuallyWithT(t, func(ct *assert.CollectT) {

0 commit comments

Comments
 (0)