diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index d6744dc288..cdbcd38237 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -40,6 +40,8 @@ import ( envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/go-logr/logr" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" @@ -274,12 +276,28 @@ func (r *ReconcileGatewayAPI) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } + // Build a trust bundle containing public CA roots (extracted from the operator's + // UBI base image) plus the Calico operator CA. Envoy-gateway pulls wasm OCI + // images and envoy-proxy may originate TLS to public upstreams, JWT/OIDC + // providers, and tracing exporters -- none of which work without public CAs. + certificateManager, err := certificatemanager.Create(r.client, installationSpec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) + return reconcile.Result{}, err + } + trustedBundle, err := certificateManager.CreateTrustedBundleWithSystemRootCertificates() + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create gateway trust bundle", err, reqLogger) + return reconcile.Result{}, err + } + gatewayConfig := &gatewayapi.GatewayAPIImplementationConfig{ Installation: installationSpec, PullSecrets: pullSecrets, GatewayAPI: gatewayAPI, CustomEnvoyProxies: make(map[string]*envoyapi.EnvoyProxy), CurrentGatewayClasses: set.New[string](), + TrustedBundle: trustedBundle, } if gatewayAPI.Spec.EnvoyGatewayConfigRef != nil { diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index 90437ac610..cf918699a0 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -39,9 +39,12 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/gatewayapi" ) @@ -67,6 +70,14 @@ var _ = Describe("Gateway API controller tests", func() { // Create a client that will have a CRUD interface of k8s objects. c = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() ctx = context.Background() + + // Seed the operator CA secret in the fake client so that certificatemanager.Create + // (called by ReconcileGatewayAPI.Reconcile to build the trusted bundle) does not + // fail with "CA secret does not exist". + certificateManager, err := certificatemanager.Create(c, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Create(ctx, certificateManager.KeyPair().Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + installation = &operatorv1.Installation{ ObjectMeta: metav1.ObjectMeta{Name: "default"}, Spec: operatorv1.InstallationSpec{ diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index 24d9bb6ed4..a9d51c8faa 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -30,6 +30,7 @@ import ( rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/tls/certificatemanagement" admissionregv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -407,6 +408,11 @@ type GatewayAPIImplementationConfig struct { CustomEnvoyGateway *envoyapi.EnvoyGateway CustomEnvoyProxies map[string]*envoyapi.EnvoyProxy CurrentGatewayClasses set.Set[string] + // TrustedBundle carries the public CA bundle (extracted from the operator's UBI + // base image) plus Calico's internal CA. Mounted on the envoy-gateway controller + // and on every provisioned envoy-proxy pod so outbound TLS (OCI wasm fetch, + // JWT/OIDC providers, public upstreams, tracing exporters) can validate peers. + TrustedBundle certificatemanagement.TrustedBundle } type gatewayAPIImplementationComponent struct { @@ -600,6 +606,13 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien objs = append(objs, envoyGatewayConfigMap) + // Ship the operator's trust bundle (public CAs + Calico CA) into the gateway + // namespace as a ConfigMap so it can be mounted on both the envoy-gateway + // controller and every provisioned envoy-proxy pod. + if pr.cfg.TrustedBundle != nil { + objs = append(objs, pr.cfg.TrustedBundle.ConfigMap(resources.namespace.Name)) + } + // Deep copy the controller deployment, controllerDeployment := resources.controllerDeployment.DeepCopyObject().(*appsv1.Deployment) @@ -614,6 +627,29 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien // Add a k8s-app label that we can use to provide API access for the controller. controllerDeployment.Spec.Template.Labels["k8s-app"] = GatewayControllerLabel + // Mount the trust bundle on the envoy-gateway controller. The controller pulls + // wasm OCI images and may call out to JWT/OIDC providers, both of which need + // public CA roots to validate TLS. + if pr.cfg.TrustedBundle != nil { + controllerDeployment.Spec.Template.Spec.Volumes = append( + controllerDeployment.Spec.Template.Spec.Volumes, + pr.cfg.TrustedBundle.Volume(), + ) + bundleMounts := pr.cfg.TrustedBundle.VolumeMounts(pr.SupportedOSType()) + for i := range controllerDeployment.Spec.Template.Spec.Containers { + controllerDeployment.Spec.Template.Spec.Containers[i].VolumeMounts = append( + controllerDeployment.Spec.Template.Spec.Containers[i].VolumeMounts, + bundleMounts..., + ) + } + if controllerDeployment.Spec.Template.Annotations == nil { + controllerDeployment.Spec.Template.Annotations = map[string]string{} + } + for k, v := range pr.cfg.TrustedBundle.HashAnnotations() { + controllerDeployment.Spec.Template.Annotations[k] = v + } + } + // Apply customizations from the GatewayControllerDeployment field of the GatewayAPI CR. rcomp.ApplyDeploymentOverrides(controllerDeployment, pr.cfg.GatewayAPI.Spec.GatewayControllerDeployment) @@ -752,6 +788,24 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className string, envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container.Image = &pr.envoyProxyImage } + // Mount the operator's trust bundle on the data-plane envoy-proxy pod so that + // outbound TLS (wasm OCI fetch, JWT/OIDC, HTTPS upstreams, tracing exporters) + // can validate public CAs. The bundle is the same ConfigMap mounted on the + // envoy-gateway controller, in the same namespace as the proxy. + if pr.cfg.TrustedBundle != nil { + bundleVolume := pr.cfg.TrustedBundle.Volume() + bundleMounts := pr.cfg.TrustedBundle.VolumeMounts(pr.SupportedOSType()) + if envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet != nil { + ds := envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet + ds.Pod.Volumes = append(ds.Pod.Volumes, bundleVolume) + ds.Container.VolumeMounts = append(ds.Container.VolumeMounts, bundleMounts...) + } else { + dep := envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment + dep.Pod.Volumes = append(dep.Pod.Volumes, bundleVolume) + dep.Container.VolumeMounts = append(dep.Container.VolumeMounts, bundleMounts...) + } + } + // Apply overrides. if envoyProxy.Spec.Provider.Kubernetes.EnvoyDaemonSet != nil { rcomp.ApplyEnvoyProxyOverrides(envoyProxy, classSpec.GatewayDaemonSet) diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index 1152d48149..e4c0acea21 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -24,6 +24,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/tls/certificatemanagement" admissionregv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -926,6 +927,61 @@ value: Expect(ep.Spec.Provider.Kubernetes.EnvoyService.Patch).To(Equal(patch)) }) + It("mounts the trust bundle on envoy-gateway and envoy-proxy when provided", func() { + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + } + gatewayAPI := &operatorv1.GatewayAPI{ + Spec: operatorv1.GatewayAPISpec{ + GatewayClasses: []operatorv1.GatewayClassSpec{{Name: "tigera-gateway-class"}}, + }, + } + bundle, err := certificatemanagement.CreateTrustedBundleWithSystemRootCertificates(nil) + Expect(err).NotTo(HaveOccurred()) + gatewayComp := GatewayAPIImplementationComponent(&GatewayAPIImplementationConfig{ + Installation: installation, + GatewayAPI: gatewayAPI, + TrustedBundle: bundle, + }) + + objsToCreate, _ := gatewayComp.Objects() + + // The bundle ConfigMap is materialised in the gateway namespace. + bundleCM, err := rtest.GetResourceOfType[*corev1.ConfigMap](objsToCreate, certificatemanagement.TrustedCertConfigMapName, "tigera-gateway") + Expect(err).NotTo(HaveOccurred()) + Expect(bundleCM.Data).To(HaveKey(certificatemanagement.TrustedCertConfigMapKeyName)) + + // The envoy-gateway controller mounts the bundle. + controller, err := rtest.GetResourceOfType[*appsv1.Deployment](objsToCreate, "envoy-gateway", "tigera-gateway") + Expect(err).NotTo(HaveOccurred()) + volNames := []string{} + for _, v := range controller.Spec.Template.Spec.Volumes { + volNames = append(volNames, v.Name) + } + Expect(volNames).To(ContainElement(certificatemanagement.TrustedCertConfigMapName)) + mountPaths := []string{} + for _, m := range controller.Spec.Template.Spec.Containers[0].VolumeMounts { + mountPaths = append(mountPaths, m.MountPath) + } + Expect(mountPaths).To(ContainElement("/etc/pki/tls/certs")) + + // The envoy-proxy data plane (patched via EnvoyProxy) mounts the bundle. + proxy, err := rtest.GetResourceOfType[*envoyapi.EnvoyProxy](objsToCreate, "tigera-gateway-class", "tigera-gateway") + Expect(err).NotTo(HaveOccurred()) + dep := proxy.Spec.Provider.Kubernetes.EnvoyDeployment + Expect(dep).NotTo(BeNil()) + proxyVolNames := []string{} + for _, v := range dep.Pod.Volumes { + proxyVolNames = append(proxyVolNames, v.Name) + } + Expect(proxyVolNames).To(ContainElement(certificatemanagement.TrustedCertConfigMapName)) + proxyMountPaths := []string{} + for _, m := range dep.Container.VolumeMounts { + proxyMountPaths = append(proxyMountPaths, m.MountPath) + } + Expect(proxyMountPaths).To(ContainElement("/etc/pki/tls/certs")) + }) + It("should not deploy waf-http-filter or l7-log-collector for open-source", func() { installation := &operatorv1.InstallationSpec{ Variant: operatorv1.Calico, diff --git a/test/gatewayapi_test.go b/test/gatewayapi_test.go index 7ac007e315..01f455ea4d 100644 --- a/test/gatewayapi_test.go +++ b/test/gatewayapi_test.go @@ -36,8 +36,11 @@ import ( envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" operator "github.com/tigera/operator/api/v1" "github.com/tigera/operator/internal/controller" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/dns" logf "sigs.k8s.io/controller-runtime/pkg/log" gapi "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/yaml" // gopkg.in/yaml.v2 didn't parse all the fields but this package did @@ -88,6 +91,14 @@ var _ = Describe("GatewayAPI tests", func() { Expect(err).NotTo(HaveOccurred()) } + By("Seeding the operator CA secret (normally created by the core controller, which is not running in this FV suite)") + certificateManager, err := certificatemanager.Create(c, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(context.Background(), certificateManager.KeyPair().Secret(common.OperatorNamespace())) + if err != nil && !kerror.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + By("Checking no Installation is left over from previous tests") instance := &operator.Installation{ TypeMeta: metav1.TypeMeta{Kind: "Installation", APIVersion: "operator.tigera.io/v1"},