Skip to content

Commit dffb2b8

Browse files
ardagucluardaguclu
committed
Enable test-e2e-encryption
1 parent d07df3e commit dffb2b8

2 files changed

Lines changed: 111 additions & 35 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ verify-podnetworkconnectivitychecks:
2424
$(MAKE) -C pkg/operator/connectivitycheckcontroller verify
2525

2626
test-e2e-encryption: GO_TEST_PACKAGES :=./test/e2e-encryption/...
27+
test-e2e-encryption: GO_TEST_FLAGS += -v
28+
test-e2e-encryption: test-unit
2729
.PHONY: test-e2e-encryption
2830

2931
test-e2e-monitoring: GO_TEST_PACKAGES :=./test/e2e-monitoring/...

‎test/e2e-encryption/encryption_test.go‎

Lines changed: 109 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,12 @@ import (
1212

1313
"github.com/stretchr/testify/require"
1414
corev1 "k8s.io/api/core/v1"
15-
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
16-
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
15+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
16+
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
1717
"k8s.io/apimachinery/pkg/api/errors"
1818
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1919
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
20+
"k8s.io/apimachinery/pkg/runtime"
2021
"k8s.io/apimachinery/pkg/runtime/schema"
2122
"k8s.io/apimachinery/pkg/types"
2223
"k8s.io/apimachinery/pkg/util/wait"
@@ -25,7 +26,6 @@ import (
2526
"k8s.io/client-go/kubernetes"
2627
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
2728
"k8s.io/client-go/tools/cache"
28-
"k8s.io/client-go/util/retry"
2929
"k8s.io/utils/clock"
3030
"sigs.k8s.io/yaml"
3131

@@ -67,11 +67,11 @@ func TestEncryptionIntegration(tt *testing.T) {
6767
kubeClient, err := kubernetes.NewForConfig(kubeConfig)
6868
require.NoError(t, err)
6969
kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, "openshift-config-managed")
70-
apiextensionsClient, err := v1beta1.NewForConfig(kubeConfig)
70+
apiextensionsClient, err := v1.NewForConfig(kubeConfig)
7171
require.NoError(t, err)
7272

7373
// create ExtensionTest operator CRD
74-
var operatorCRD apiextensionsv1beta1.CustomResourceDefinition
74+
var operatorCRD apiextensionsv1.CustomResourceDefinition
7575
require.NoError(t, yaml.Unmarshal([]byte(encryptionTestOperatorCRD), &operatorCRD))
7676
crd, err := apiextensionsClient.CustomResourceDefinitions().Create(ctx, &operatorCRD, metav1.CreateOptions{})
7777
if errors.IsAlreadyExists(err) {
@@ -81,6 +81,26 @@ func TestEncryptionIntegration(tt *testing.T) {
8181
}
8282
defer apiextensionsClient.CustomResourceDefinitions().Delete(ctx, crd.Name, metav1.DeleteOptions{})
8383

84+
t.Logf("Waiting for CRD to be ready")
85+
err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
86+
oCRD, crdErr := apiextensionsClient.CustomResourceDefinitions().Get(ctx, operatorCRD.Name, metav1.GetOptions{})
87+
if crdErr != nil {
88+
if errors.IsNotFound(crdErr) {
89+
return false, nil
90+
}
91+
return false, crdErr
92+
}
93+
94+
for _, condition := range oCRD.Status.Conditions {
95+
if condition.Type == apiextensionsv1.Established {
96+
operatorCRD = *oCRD
97+
return condition.Status == apiextensionsv1.ConditionTrue, nil
98+
}
99+
}
100+
return false, nil
101+
})
102+
require.NoError(t, err)
103+
84104
// create operator client and create instance with ManagementState="Managed"
85105
operatorGVR := schema.GroupVersionResource{Group: operatorCRD.Spec.Group, Version: "v1", Resource: operatorCRD.Spec.Names.Plural}
86106
operatorGVK := schema.GroupVersionKind{Group: operatorCRD.Spec.Group, Version: "v1", Kind: operatorCRD.Spec.Names.Kind}
@@ -99,7 +119,7 @@ func TestEncryptionIntegration(tt *testing.T) {
99119
require.NoError(t, err)
100120
dynamicClient, err := dynamic.NewForConfig(kubeConfig)
101121
require.NoError(t, err)
102-
err = wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
122+
err = wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
103123
_, err := dynamicClient.Resource(operatorGVR).Create(ctx, &unstructured.Unstructured{
104124
Object: map[string]interface{}{
105125
"apiVersion": "operator.openshift.io/v1",
@@ -121,7 +141,7 @@ func TestEncryptionIntegration(tt *testing.T) {
121141
require.NoError(t, err)
122142

123143
// create APIServer clients
124-
fakeConfigClient := configv1clientfake.NewSimpleClientset(&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}})
144+
fakeConfigClient := configv1clientfake.NewClientset(&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}})
125145
fakeConfigInformer := configv1informers.NewSharedInformerFactory(fakeConfigClient, 10*time.Minute)
126146
fakeApiServerClient := fakeConfigClient.ConfigV1().APIServers()
127147

@@ -159,6 +179,20 @@ func TestEncryptionIntegration(tt *testing.T) {
159179
operatorInformer.Start(stopCh)
160180
go controllers.Run(ctx, 1)
161181

182+
t.Logf("Waiting for operator informer to sync")
183+
if !cache.WaitForCacheSync(stopCh, operatorClient.Informer().HasSynced) {
184+
t.Fatalf("failed to sync operator informer")
185+
}
186+
kubeInformers.WaitForCacheSync(stopCh)
187+
err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
188+
_, _, _, err := operatorClient.GetOperatorState()
189+
if errors.IsNotFound(err) {
190+
return false, nil
191+
}
192+
return err == nil, err
193+
})
194+
require.NoError(t, err)
195+
162196
waitForConfigEventuallyCond := func(cond func(s string) bool) {
163197
t.Helper()
164198
stopCh := time.After(wait.ForeverTestTimeout)
@@ -208,22 +242,16 @@ func TestEncryptionIntegration(tt *testing.T) {
208242
}
209243
return operatorv1.ConditionUnknown
210244
}
211-
requireConditionStatus := func(condType string, expected operatorv1.ConditionStatus) {
212-
t.Helper()
213-
if status := conditionStatus(condType); status != expected {
214-
t.Errorf("expected condition %s of status %s, found: %q", condType, expected, status)
215-
}
216-
}
217245
waitForConditionStatus := func(condType string, expected operatorv1.ConditionStatus) {
218246
t.Helper()
219-
err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
247+
err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
220248
return conditionStatus(condType) == expected, nil
221249
})
222250
require.NoError(t, err)
223251
}
224252
waitForMigration := func(key string) {
225253
t.Helper()
226-
err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
254+
err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
227255
s, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-%s", component, key), metav1.GetOptions{})
228256
require.NoError(t, err)
229257

@@ -245,7 +273,7 @@ func TestEncryptionIntegration(tt *testing.T) {
245273
keySecretsLabel := fmt.Sprintf("%s=%s", secrets.EncryptionKeySecretsLabel, component)
246274
waitForKeys := func(n int) {
247275
t.Helper()
248-
err := wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
276+
err := wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
249277
l, err := kubeClient.CoreV1().Secrets("openshift-config-managed").List(ctx, metav1.ListOptions{LabelSelector: keySecretsLabel})
250278
if err != nil {
251279
return false, err
@@ -264,7 +292,7 @@ func TestEncryptionIntegration(tt *testing.T) {
264292
"kubeapiservers.operator.openshift.io=aescbc:1,identity;kubeschedulers.operator.openshift.io=aescbc:1,identity",
265293
)
266294
waitForMigration("1")
267-
requireConditionStatus("Encrypted", operatorv1.ConditionTrue)
295+
waitForConditionStatus("Encrypted", operatorv1.ConditionTrue)
268296

269297
t.Logf("Switch to identity")
270298
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"identity"}}}`), metav1.PatchOptions{})
@@ -274,14 +302,14 @@ func TestEncryptionIntegration(tt *testing.T) {
274302
"kubeapiservers.operator.openshift.io=aescbc:1,identity,aesgcm:2;kubeschedulers.operator.openshift.io=aescbc:1,identity,aesgcm:2",
275303
"kubeapiservers.operator.openshift.io=identity,aescbc:1,aesgcm:2;kubeschedulers.operator.openshift.io=identity,aescbc:1,aesgcm:2",
276304
)
277-
requireConditionStatus("Encrypted", operatorv1.ConditionFalse)
305+
waitForConditionStatus("Encrypted", operatorv1.ConditionFalse)
278306

279307
t.Logf("Switch to empty mode")
280308
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":""}}}`), metav1.PatchOptions{})
281309
require.NoError(t, err)
282310
time.Sleep(5 * time.Second) // give controller time to create keys (it shouldn't)
283311
waitForKeys(2)
284-
requireConditionStatus("Encrypted", operatorv1.ConditionFalse)
312+
waitForConditionStatus("Encrypted", operatorv1.ConditionFalse)
285313

286314
t.Logf("Switch to aescbc again")
287315
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"aescbc"}}}`), metav1.PatchOptions{})
@@ -297,15 +325,11 @@ func TestEncryptionIntegration(tt *testing.T) {
297325
t.Logf("Setting external reason")
298326
setExternalReason := func(reason string) {
299327
t.Helper()
300-
err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
301-
spec, _, rv, err := operatorClient.GetOperatorState()
302-
if err != nil {
303-
return err
304-
}
305-
spec.UnsupportedConfigOverrides.Raw = []byte(fmt.Sprintf(`{"encryption":{"reason":%q}}`, reason))
306-
_, _, err = operatorClient.UpdateOperatorSpec(context.TODO(), rv, spec)
307-
return err
308-
})
328+
applyConfig := applyoperatorv1.OperatorSpec().
329+
WithUnsupportedConfigOverrides(runtime.RawExtension{
330+
Raw: []byte(fmt.Sprintf(`{"encryption":{"reason":%q}}`, reason)),
331+
})
332+
err = operatorClient.ApplyOperatorSpec(ctx, "encryption-test", applyConfig)
309333
require.NoError(t, err)
310334
}
311335
setExternalReason("a")
@@ -341,7 +365,7 @@ func TestEncryptionIntegration(tt *testing.T) {
341365
require.NoError(t, err)
342366
err = kubeClient.CoreV1().Secrets("openshift-config-managed").Delete(ctx, fmt.Sprintf("encryption-key-%s-6", component), metav1.DeleteOptions{})
343367
require.NoError(t, err)
344-
err = wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
368+
err = wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
345369
_, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-7", component), metav1.GetOptions{})
346370
if errors.IsNotFound(err) {
347371
return false, nil
@@ -378,16 +402,16 @@ func TestEncryptionIntegration(tt *testing.T) {
378402
t.Logf("Delete the openshift-config-managed config")
379403
deployer.DeleteOperandConfig()
380404
waitForConfigs(
381-
// 7 is migrated and hence only one needed, but we rotate through identity
382-
"kubeapiservers.operator.openshift.io=identity,aescbc:7;kubeschedulers.operator.openshift.io=identity,aescbc:7",
383-
// 7 is migrated, plus one backed key (5). 6 is deleted, and therefore is not preserved (would be if the operand config was not deleted)
405+
// 7 is migrated, backed key (5) is immediately included when rotating through identity. 6 is deleted, not preserved (would be if operand config was not deleted)
406+
"kubeapiservers.operator.openshift.io=identity,aescbc:7,aescbc:5;kubeschedulers.operator.openshift.io=identity,aescbc:7,aescbc:5",
407+
// promote aescbc:7 back to write position
384408
"kubeapiservers.operator.openshift.io=aescbc:7,aescbc:5,identity;kubeschedulers.operator.openshift.io=aescbc:7,aescbc:5,identity",
385409
)
386410
waitForConditionStatus("Encrypted", operatorv1.ConditionTrue)
387411
}
388412

389413
const encryptionTestOperatorCRD = `
390-
apiVersion: apiextensions.k8s.io/v1beta1
414+
apiVersion: apiextensions.k8s.io/v1
391415
kind: CustomResourceDefinition
392416
metadata:
393417
name: encryptiontests.operator.openshift.io
@@ -399,12 +423,62 @@ spec:
399423
plural: encryptiontests
400424
singular: encryptiontest
401425
scope: Cluster
402-
subresources:
403-
status: {}
404426
versions:
405427
- name: v1
406428
served: true
407429
storage: true
430+
subresources:
431+
status: {}
432+
schema:
433+
openAPIV3Schema:
434+
type: object
435+
required:
436+
- spec
437+
properties:
438+
spec:
439+
type: object
440+
required:
441+
- managementState
442+
properties:
443+
managementState:
444+
type: string
445+
enum:
446+
- Managed
447+
- Unmanaged
448+
- Removed
449+
observedConfig:
450+
type: object
451+
nullable: true
452+
x-kubernetes-preserve-unknown-fields: true
453+
unsupportedConfigOverrides:
454+
type: object
455+
nullable: true
456+
x-kubernetes-preserve-unknown-fields: true
457+
status:
458+
type: object
459+
properties:
460+
conditions:
461+
type: array
462+
items:
463+
type: object
464+
required:
465+
- type
466+
- status
467+
properties:
468+
type:
469+
type: string
470+
status:
471+
type: string
472+
lastTransitionTime:
473+
type: string
474+
format: date-time
475+
reason:
476+
type: string
477+
message:
478+
type: string
479+
observedGeneration:
480+
type: integer
481+
format: int64
408482
`
409483

410484
func toString(c *apiserverv1.EncryptionConfiguration) string {

0 commit comments

Comments
 (0)