Skip to content

Commit 8ffbd4c

Browse files
committed
test: Add TLS security profile propagation test with Ginkgo framework
Implement end-to-end test to verify that TLS security profile changes propagate from the APIServer to the OpenShift Controller Manager. Signed-off-by: Kaleemullah Siddiqui <ksiddiqu@redhat.com>
1 parent fa996b0 commit 8ffbd4c

5 files changed

Lines changed: 347 additions & 13 deletions

File tree

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
// This file imports test packages to ensure they are included in the build.
2+
// These imports are necessary to register Ginkgo tests with the OpenShift Tests Extension framework.
3+
package main
4+
5+
import (
6+
// Import test packages to register Ginkgo tests
7+
_ "github.com/openshift/cluster-openshift-controller-manager-operator/test/e2e"
8+
)
Lines changed: 42 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,45 @@
1+
/*
2+
This command is used to run the Cluster Controller Manager Operator tests extension for OpenShift.
3+
It registers the Cluster Controller Manager Operator tests with the OpenShift Tests Extension framework
4+
and provides a command-line interface to execute them.
5+
For further information, please refer to the documentation at:
6+
https://github.com/openshift-eng/openshift-tests-extension/blob/main/cmd/example-tests/main.go
7+
*/
8+
19
package main
210

311
import (
4-
"context"
12+
"fmt"
513
"os"
614

715
"github.com/spf13/cobra"
816
"k8s.io/component-base/cli"
17+
"k8s.io/klog/v2"
918

1019
otecmd "github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
1120
oteextension "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
21+
oteginkgo "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
1222
"github.com/openshift/cluster-openshift-controller-manager-operator/pkg/version"
13-
14-
"k8s.io/klog/v2"
1523
)
1624

1725
func main() {
18-
command := newOperatorTestCommand(context.Background())
19-
code := cli.Run(command)
26+
cmd, err := newOperatorTestCommand()
27+
if err != nil {
28+
klog.Fatal(err)
29+
}
30+
31+
code := cli.Run(cmd)
2032
os.Exit(code)
2133
}
2234

23-
func newOperatorTestCommand(ctx context.Context) *cobra.Command {
24-
registry := prepareOperatorTestsRegistry()
35+
func newOperatorTestCommand() (*cobra.Command, error) {
36+
registry, err := prepareOperatorTestsRegistry()
37+
if err != nil {
38+
return nil, fmt.Errorf("failed to prepare test registry: %w", err)
39+
}
2540

2641
cmd := &cobra.Command{
27-
Use: "cluster-openshift-controller-manager-operator-tests-ext",
42+
Use: "cluster-openshift-controller-manager-operator-tests",
2843
Short: "A binary used to run cluster-openshift-controller-manager-operator tests as part of OTE.",
2944
Run: func(cmd *cobra.Command, args []string) {
3045
if err := cmd.Help(); err != nil {
@@ -41,13 +56,29 @@ func newOperatorTestCommand(ctx context.Context) *cobra.Command {
4156

4257
cmd.AddCommand(otecmd.DefaultExtensionCommands(registry)...)
4358

44-
return cmd
59+
return cmd, nil
4560
}
4661

47-
func prepareOperatorTestsRegistry() *oteextension.Registry {
62+
// prepareOperatorTestsRegistry creates the OTE registry for this operator.
63+
// This method must be called before adding the registry to the OTE framework.
64+
func prepareOperatorTestsRegistry() (*oteextension.Registry, error) {
4865
registry := oteextension.NewRegistry()
4966
extension := oteextension.NewExtension("openshift", "payload", "cluster-openshift-controller-manager-operator")
5067

68+
extension.AddSuite(oteextension.Suite{
69+
Name: "openshift/cluster-openshift-controller-manager-operator/operator/serial",
70+
Parallelism: 1,
71+
Qualifiers: []string{
72+
`name.contains("[Operator]") && name.contains("[Serial]")`,
73+
},
74+
})
75+
76+
specs, err := oteginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
77+
if err != nil {
78+
return nil, fmt.Errorf("couldn't build extension test specs from ginkgo: %w", err)
79+
}
80+
81+
extension.AddSpecs(specs)
5182
registry.Register(extension)
52-
return registry
83+
return registry, nil
5384
}

test/e2e/tls_security_profile.go

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,293 @@
1+
package e2e
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"strings"
7+
"testing"
8+
"time"
9+
10+
g "github.com/onsi/ginkgo/v2"
11+
"github.com/stretchr/testify/require"
12+
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
15+
"k8s.io/apimachinery/pkg/util/wait"
16+
17+
configv1 "github.com/openshift/api/config/v1"
18+
"github.com/openshift/cluster-openshift-controller-manager-operator/test/framework"
19+
)
20+
21+
var _ = g.Describe("[sig-openshift-controller-manager] TLS Security Profile", func() {
22+
g.It("[Operator][TLS][Serial] should propagate Modern TLS profile from APIServer to OpenShift Controller Manager", func() {
23+
testTLSSecurityProfilePropagation(g.GinkgoTB())
24+
})
25+
})
26+
27+
func testTLSSecurityProfilePropagation(t testing.TB) {
28+
ctx := context.Background()
29+
client := framework.MustNewClientset(t, nil)
30+
31+
// Make sure the operator is fully up
32+
framework.MustEnsureClusterOperatorStatusIsSet(t, client)
33+
34+
// Get the current APIServer config
35+
apiServer, err := client.APIServers().Get(ctx, "cluster", metav1.GetOptions{})
36+
require.NoError(t, err, "failed to get APIServer config")
37+
38+
// Save the original TLS profile for cleanup
39+
originalTLSProfile := apiServer.Spec.TLSSecurityProfile
40+
41+
// Modify the TLS security profile to use Modern profile
42+
// Modern profile uses TLS 1.3 with modern cipher suites
43+
apiServer.Spec.TLSSecurityProfile = &configv1.TLSSecurityProfile{
44+
Type: configv1.TLSProfileModernType,
45+
Modern: &configv1.ModernTLSProfile{},
46+
}
47+
48+
_, err = client.APIServers().Update(ctx, apiServer, metav1.UpdateOptions{})
49+
require.NoError(t, err, "failed to update APIServer TLS profile to Modern")
50+
51+
// Cleanup: restore original TLS profile and verify restoration
52+
t.Cleanup(func() {
53+
t.Log("Restoring original TLS profile")
54+
apiServer, err := client.APIServers().Get(ctx, "cluster", metav1.GetOptions{})
55+
if err != nil {
56+
t.Logf("failed to get APIServer for cleanup: %v", err)
57+
return
58+
}
59+
apiServer.Spec.TLSSecurityProfile = originalTLSProfile
60+
if _, err := client.APIServers().Update(ctx, apiServer, metav1.UpdateOptions{}); err != nil {
61+
t.Logf("failed to restore original TLS profile: %v", err)
62+
return
63+
}
64+
65+
// Wait for operator to reconcile the restoration
66+
t.Log("Waiting for operator to reconcile TLS profile restoration")
67+
err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) {
68+
co, err := client.ClusterOperators().Get(ctx, "openshift-controller-manager", metav1.GetOptions{})
69+
if err != nil {
70+
t.Logf("error getting clusteroperator during cleanup: %v", err)
71+
return false, nil
72+
}
73+
74+
isAvailable := false
75+
isProgressing := true
76+
77+
for _, c := range co.Status.Conditions {
78+
if c.Type == configv1.OperatorAvailable && c.Status == configv1.ConditionTrue {
79+
isAvailable = true
80+
}
81+
if c.Type == configv1.OperatorProgressing && c.Status == configv1.ConditionFalse {
82+
isProgressing = false
83+
}
84+
}
85+
86+
if isAvailable && !isProgressing {
87+
t.Log("Operator reconciliation after restoration complete")
88+
return true, nil
89+
}
90+
91+
return false, nil
92+
})
93+
if err != nil {
94+
t.Logf("operator did not complete reconciliation after restoration: %v", err)
95+
return
96+
}
97+
98+
// Verify TLS profile was restored (should be back to default TLS 1.2 or original setting)
99+
t.Log("Verifying TLS profile was restored correctly")
100+
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
101+
cfg, err := client.OpenShiftControllerManagers().Get(ctx, "cluster", metav1.GetOptions{})
102+
if err != nil {
103+
t.Logf("error getting openshift controller manager config during cleanup verification: %v", err)
104+
return false, nil
105+
}
106+
107+
observedConfig := map[string]interface{}{}
108+
if err := json.Unmarshal(cfg.Spec.ObservedConfig.Raw, &observedConfig); err != nil {
109+
t.Logf("failed to unmarshal observed config during cleanup: %v", err)
110+
return false, nil
111+
}
112+
113+
// Check the restored TLS version
114+
minTLSVersion, found, err := unstructured.NestedString(observedConfig, "servingInfo", "minTLSVersion")
115+
if err != nil {
116+
t.Logf("error reading minTLSVersion during cleanup: %v", err)
117+
return false, nil
118+
}
119+
120+
// If original profile was nil, expect default (typically VersionTLS12)
121+
// If original profile was set, it should match
122+
if originalTLSProfile == nil {
123+
// Default OpenShift TLS profile is typically TLS 1.2
124+
if found && minTLSVersion == "VersionTLS12" {
125+
t.Logf("TLS profile restored to default: %s", minTLSVersion)
126+
return true, nil
127+
}
128+
// Also accept if TLS config is removed entirely (using cluster defaults)
129+
if !found || minTLSVersion == "" {
130+
t.Log("TLS profile restored to cluster defaults (no explicit TLS version)")
131+
return true, nil
132+
}
133+
} else {
134+
// If there was an original profile, verify it's not TLS 1.3 anymore
135+
if found && minTLSVersion != "VersionTLS13" {
136+
t.Logf("TLS profile restored from Modern: %s", minTLSVersion)
137+
return true, nil
138+
}
139+
}
140+
141+
t.Logf("Waiting for TLS profile restoration to propagate, current: %s", minTLSVersion)
142+
return false, nil
143+
})
144+
if err != nil {
145+
t.Logf("TLS profile was not properly restored in observed config: %v", err)
146+
}
147+
})
148+
149+
// Wait for the operator to start progressing (detecting the change)
150+
t.Log("Waiting for operator to detect TLS profile change and start progressing")
151+
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
152+
co, err := client.ClusterOperators().Get(ctx, "openshift-controller-manager", metav1.GetOptions{})
153+
if err != nil {
154+
t.Logf("error getting clusteroperator: %v", err)
155+
return false, nil
156+
}
157+
for _, c := range co.Status.Conditions {
158+
if c.Type == configv1.OperatorProgressing && c.Status == configv1.ConditionTrue {
159+
t.Logf("Operator is now progressing, reason: %s", c.Reason)
160+
return true, nil
161+
}
162+
}
163+
return false, nil
164+
})
165+
if err != nil {
166+
t.Logf("Warning: operator did not start progressing within 5 minutes, continuing anyway: %v", err)
167+
}
168+
169+
// Wait for the operator to finish progressing (reconciliation complete)
170+
// This typically takes 12-15 minutes for TLS changes to propagate
171+
t.Log("Waiting for operator to complete reconciliation (may take up to 15 minutes)")
172+
err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) {
173+
co, err := client.ClusterOperators().Get(ctx, "openshift-controller-manager", metav1.GetOptions{})
174+
if err != nil {
175+
t.Logf("error getting clusteroperator: %v", err)
176+
return false, nil
177+
}
178+
179+
isAvailable := false
180+
isProgressing := true
181+
isDegraded := false
182+
183+
for _, c := range co.Status.Conditions {
184+
if c.Type == configv1.OperatorAvailable && c.Status == configv1.ConditionTrue {
185+
isAvailable = true
186+
}
187+
if c.Type == configv1.OperatorProgressing && c.Status == configv1.ConditionFalse {
188+
isProgressing = false
189+
}
190+
if c.Type == configv1.OperatorDegraded && c.Status == configv1.ConditionTrue {
191+
isDegraded = true
192+
}
193+
}
194+
195+
if isDegraded {
196+
t.Log("Warning: operator is degraded")
197+
return false, nil
198+
}
199+
200+
if isAvailable && !isProgressing {
201+
t.Log("Operator reconciliation complete")
202+
return true, nil
203+
}
204+
205+
t.Logf("Operator still reconciling, available=%v, progressing=%v", isAvailable, isProgressing)
206+
return false, nil
207+
})
208+
require.NoError(t, err, "operator did not complete reconciliation")
209+
210+
// Now verify the TLS config was propagated to the observed config
211+
t.Log("Verifying TLS config in observed config")
212+
err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
213+
cfg, err := client.OpenShiftControllerManagers().Get(ctx, "cluster", metav1.GetOptions{})
214+
if err != nil {
215+
t.Logf("error getting openshift controller manager config: %v", err)
216+
return false, nil
217+
}
218+
219+
observed := string(cfg.Spec.ObservedConfig.Raw)
220+
221+
// The Modern TLS profile should set minTLSVersion to TLS 1.3
222+
// We're looking for the propagated TLS settings
223+
hasTLSVersion := strings.Contains(observed, "\"minTLSVersion\"")
224+
hasCipherSuites := strings.Contains(observed, "\"cipherSuites\"")
225+
226+
if !hasTLSVersion || !hasCipherSuites {
227+
t.Logf("TLS config not yet observed in config: %s", observed)
228+
return false, nil
229+
}
230+
231+
t.Logf("TLS config successfully observed: %s", observed)
232+
233+
// Additional validation: parse the observed config
234+
observedConfig := map[string]interface{}{}
235+
if err := json.Unmarshal(cfg.Spec.ObservedConfig.Raw, &observedConfig); err != nil {
236+
t.Logf("failed to unmarshal observed config: %v", err)
237+
return false, nil
238+
}
239+
240+
// Verify servingInfo exists
241+
_, found, err := unstructured.NestedMap(observedConfig, "servingInfo")
242+
if err != nil || !found {
243+
t.Log("servingInfo not found in observed config")
244+
return false, nil
245+
}
246+
247+
// Verify minTLSVersion is set to TLS 1.3 (Modern profile)
248+
minTLSVersion, found, err := unstructured.NestedString(observedConfig, "servingInfo", "minTLSVersion")
249+
if err != nil || !found || minTLSVersion == "" {
250+
t.Logf("minTLSVersion not properly set, found=%v, value=%s", found, minTLSVersion)
251+
return false, nil
252+
}
253+
254+
// Modern profile should use VersionTLS13 (exact string match)
255+
if minTLSVersion != "VersionTLS13" {
256+
t.Logf("minTLSVersion not VersionTLS13 yet, got=%s, expected=VersionTLS13", minTLSVersion)
257+
return false, nil
258+
}
259+
260+
// Verify cipherSuites is set and contains the expected Modern profile ciphers
261+
cipherSuites, found, err := unstructured.NestedStringSlice(observedConfig, "servingInfo", "cipherSuites")
262+
if err != nil || !found || len(cipherSuites) == 0 {
263+
t.Logf("cipherSuites not properly set, found=%v, count=%d", found, len(cipherSuites))
264+
return false, nil
265+
}
266+
267+
// Modern profile should have exactly these TLS 1.3 cipher suites
268+
expectedCiphers := []string{
269+
"TLS_AES_128_GCM_SHA256",
270+
"TLS_AES_256_GCM_SHA384",
271+
"TLS_CHACHA20_POLY1305_SHA256",
272+
}
273+
274+
// Verify all expected ciphers are present
275+
cipherSet := make(map[string]bool)
276+
for _, cipher := range cipherSuites {
277+
cipherSet[cipher] = true
278+
}
279+
280+
for _, expected := range expectedCiphers {
281+
if !cipherSet[expected] {
282+
// Don't fail immediately, keep polling
283+
t.Logf("expected cipher suite not found yet: %s, got: %v", expected, cipherSuites)
284+
return false, nil
285+
}
286+
}
287+
288+
t.Logf("Validated Modern TLS config: minTLSVersion=%s, cipherSuites=%v", minTLSVersion, cipherSuites)
289+
return true, nil
290+
})
291+
292+
require.NoError(t, err, "Modern TLS security profile from APIServer was not propagated to OpenShift Controller Manager observed config")
293+
}

test/framework/clientset.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ func NewClientset(kubeconfig *restclient.Config) (clientset *Clientset, err erro
5656

5757
// MustNewClientset is like NewClienset but aborts the test if clienset cannot
5858
// be constructed.
59-
func MustNewClientset(t *testing.T, kubeconfig *restclient.Config) *Clientset {
59+
func MustNewClientset(t testing.TB, kubeconfig *restclient.Config) *Clientset {
60+
t.Helper()
6061
clientset, err := NewClientset(kubeconfig)
6162
if err != nil {
6263
t.Fatal(err)

0 commit comments

Comments
 (0)