From dc891997fb25df4d8d11d9644a91a1be81a875cf Mon Sep 17 00:00:00 2001 From: Alex Harford Date: Thu, 19 Mar 2026 16:18:55 -0700 Subject: [PATCH 1/4] Add calico-system tier network policy for gateway certgen job The certgen job in the tigera-gateway namespace needs DNS and kube-apiserver egress to create TLS secrets. Under a global default-deny policy, this traffic is blocked causing the job to fail with i/o timeout reaching the API server. Add a NetworkPolicy in the calico-system tier that allows the certgen pods (app == 'certgen') DNS and kube-apiserver egress, plus a default-deny policy for the tigera-gateway namespace, matching the pattern used by other components. Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/common/common.go | 3 ++ pkg/render/gatewayapi/gateway_api.go | 46 ++++++++++++++++++++--- pkg/render/gatewayapi/gateway_api_test.go | 9 ++++- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/pkg/common/common.go b/pkg/common/common.go index efdf171732..731511771f 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -21,6 +21,9 @@ const ( KubeControllersDeploymentName = "calico-kube-controllers" WindowsDaemonSetName = "calico-node-windows" + // Gateway API related const + TigeraGatewayNamespace = "tigera-gateway" + // Monitor + Prometheus related const TigeraPrometheusNamespace = "tigera-prometheus" diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index e64cbe6f95..0429278300 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -22,12 +22,14 @@ import ( "sync" envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/render" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" + networkpolicy "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" admissionregv1 "k8s.io/api/admissionregistration/v1" @@ -90,6 +92,7 @@ const ( EnvoyGatewayConfigKey = "envoy-gateway.yaml" EnvoyGatewayDeploymentContainerName = "envoy-gateway" EnvoyGatewayJobContainerName = "envoy-gateway-certgen" + GatewayCertgenPolicyName = networkpolicy.TigeraComponentPolicyPrefix + "gateway-api-certgen-access" wafFilterName = "waf-http-filter" ) @@ -635,6 +638,13 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien objs = append(objs, certgenJob) + // Add network policies to allow gateway components to function under a + // default-deny policy. + objs = append(objs, + pr.gatewayCertgenAllowTigeraPolicy(), + networkpolicy.CalicoSystemDefaultDeny(common.TigeraGatewayNamespace), + ) + // Provision GatewayClasses. for i := range pr.cfg.GatewayAPI.Spec.GatewayClasses { className := pr.cfg.GatewayAPI.Spec.GatewayClasses[i].Name @@ -671,7 +681,7 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien }, ObjectMeta: metav1.ObjectMeta{ Name: gcName, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, }, }, ) @@ -706,7 +716,7 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className string, envoyProxy.APIVersion = "gateway.envoyproxy.io/v1alpha1" } envoyProxy.Name = className - envoyProxy.Namespace = "tigera-gateway" + envoyProxy.Namespace = common.TigeraGatewayNamespace if envoyProxy.Spec.Provider == nil { envoyProxy.Spec.Provider = &envoyapi.EnvoyProxyProvider{} } @@ -1052,7 +1062,7 @@ func (pr *gatewayAPIImplementationComponent) gatewayClass(className, controllerN TypeMeta: metav1.TypeMeta{Kind: "GatewayClass", APIVersion: "gateway.networking.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ Name: className, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, }, Spec: gapi.GatewayClassSpec{ ControllerName: gapi.GatewayController(controllerName), @@ -1114,7 +1124,7 @@ func (pr *gatewayAPIImplementationComponent) wafHttpFilterServiceAccount() *core TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: wafFilterName, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, }, } } @@ -1165,8 +1175,34 @@ func (pr *gatewayAPIImplementationComponent) wafHttpFilterClusterRoleBinding() * { Kind: "ServiceAccount", Name: wafFilterName, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, }, }, } } + +// gatewayCertgenAllowTigeraPolicy creates a NetworkPolicy that allows the certgen job +// to access DNS and the Kubernetes API server, which it needs to create TLS secrets. +func (pr *gatewayAPIImplementationComponent) gatewayCertgenAllowTigeraPolicy() *v3.NetworkPolicy { + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, pr.cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerServiceSelectorEntityRule, + }) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: GatewayCertgenPolicyName, + Namespace: common.TigeraGatewayNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Tier: networkpolicy.TigeraComponentTierName, + Selector: "app == 'certgen'", + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index 1ace54dcd1..d5049abaf6 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -235,7 +236,7 @@ var _ = Describe("Gateway API rendering tests", func() { objsToCreate, objsToDelete := gatewayComp.Objects() Expect(objsToDelete).To(HaveLen(0)) Expect(objsToCreate).NotTo(BeEmpty()) - Expect(objsToCreate).To(HaveLen(20 + len(gatewayAPI.Spec.GatewayClasses))) // 20 core objects plus one per GatewayClass + Expect(objsToCreate).To(HaveLen(22 + len(gatewayAPI.Spec.GatewayClasses))) // 22 core objects plus one per GatewayClass rtest.ExpectResources(objsToCreate, []client.Object{ &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway"}}, @@ -257,6 +258,8 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &envoyapi.EnvoyProxy{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, }) @@ -358,6 +361,8 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &envoyapi.EnvoyProxy{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret1", Namespace: "tigera-gateway"}}, &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, @@ -451,6 +456,8 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter", Namespace: "tigera-gateway"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter"}}, From 35ffb73b03542809db26cf7a80ae6c1ef6c2dd01 Mon Sep 17 00:00:00 2001 From: Alex Harford Date: Thu, 19 Mar 2026 16:19:24 -0700 Subject: [PATCH 2/4] Add calico-system tier network policy for gateway controller The envoy-gateway controller needs DNS and kube-apiserver egress to function. Under a global default-deny policy, the controller fails with i/o timeout reaching the API server. Add a NetworkPolicy in the calico-system tier that allows the controller pods (k8s-app == 'gateway-api-controller') DNS egress, kube-apiserver egress, and ingress on its serving ports (xDS gRPC, ratelimit, wasm, metrics, webhook). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../gatewayapi/gatewayapi_controller.go | 3 +- pkg/render/gatewayapi/gateway_api.go | 47 +++++++++++++++++++ pkg/render/gatewayapi/gateway_api_test.go | 5 +- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index d6744dc288..cf06316985 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -40,6 +40,7 @@ import ( envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/go-logr/logr" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" @@ -496,6 +497,6 @@ func (r *ReconcileGatewayAPI) getPolicySyncPathPrefix(fcSpec *v3.FelixConfigurat // The bool return value indicates if the finalizer is Set func (r *ReconcileGatewayAPI) maintainFinalizer(ctx context.Context, gatewayAPI client.Object) (bool, error) { // These objects require graceful termination before the CNI plugin is torn down. - gatewayAPIDeployment := v1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway", Namespace: "tigera-gateway"}} + gatewayAPIDeployment := v1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway", Namespace: common.TigeraGatewayNamespace}} return utils.MaintainInstallationFinalizer(ctx, r.client, gatewayAPI, render.GatewayAPIFinalizer, &gatewayAPIDeployment) } diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index 0429278300..19b4de0899 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -92,8 +92,16 @@ const ( EnvoyGatewayConfigKey = "envoy-gateway.yaml" EnvoyGatewayDeploymentContainerName = "envoy-gateway" EnvoyGatewayJobContainerName = "envoy-gateway-certgen" + GatewayControllerPolicyName = networkpolicy.TigeraComponentPolicyPrefix + "gateway-api-controller-access" GatewayCertgenPolicyName = networkpolicy.TigeraComponentPolicyPrefix + "gateway-api-certgen-access" wafFilterName = "waf-http-filter" + + // Envoy gateway controller serving ports. + EnvoyGatewayPortGRPC = 18000 + EnvoyGatewayPortRateLimit = 18001 + EnvoyGatewayPortWasm = 18002 + EnvoyGatewayPortMetrics = 19001 + EnvoyGatewayPortWebhook = 9443 ) var ( @@ -642,6 +650,7 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien // default-deny policy. objs = append(objs, pr.gatewayCertgenAllowTigeraPolicy(), + pr.gatewayControllerAllowTigeraPolicy(), networkpolicy.CalicoSystemDefaultDeny(common.TigeraGatewayNamespace), ) @@ -1206,3 +1215,41 @@ func (pr *gatewayAPIImplementationComponent) gatewayCertgenAllowTigeraPolicy() * }, } } + +// gatewayControllerAllowTigeraPolicy creates a NetworkPolicy that allows the envoy-gateway +// controller to access DNS and the Kubernetes API server, and to receive ingress on its +// serving ports (xDS gRPC, ratelimit, wasm, metrics, webhook). +func (pr *gatewayAPIImplementationComponent) gatewayControllerAllowTigeraPolicy() *v3.NetworkPolicy { + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, pr.cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerServiceSelectorEntityRule, + }) + + ingressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(EnvoyGatewayPortGRPC, EnvoyGatewayPortRateLimit, EnvoyGatewayPortWasm, EnvoyGatewayPortMetrics, EnvoyGatewayPortWebhook), + }, + }, + } + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: GatewayControllerPolicyName, + Namespace: common.TigeraGatewayNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Tier: networkpolicy.TigeraComponentTierName, + Selector: "k8s-app == '" + GatewayControllerLabel + "'", + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index d5049abaf6..270b142646 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -236,7 +236,7 @@ var _ = Describe("Gateway API rendering tests", func() { objsToCreate, objsToDelete := gatewayComp.Objects() Expect(objsToDelete).To(HaveLen(0)) Expect(objsToCreate).NotTo(BeEmpty()) - Expect(objsToCreate).To(HaveLen(22 + len(gatewayAPI.Spec.GatewayClasses))) // 22 core objects plus one per GatewayClass + Expect(objsToCreate).To(HaveLen(23 + len(gatewayAPI.Spec.GatewayClasses))) // 23 core objects plus one per GatewayClass rtest.ExpectResources(objsToCreate, []client.Object{ &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway"}}, @@ -259,6 +259,7 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-controller-access", Namespace: "tigera-gateway"}}, &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &envoyapi.EnvoyProxy{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, @@ -362,6 +363,7 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-controller-access", Namespace: "tigera-gateway"}}, &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &envoyapi.EnvoyProxy{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret1", Namespace: "tigera-gateway"}}, @@ -457,6 +459,7 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-controller-access", Namespace: "tigera-gateway"}}, &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter", Namespace: "tigera-gateway"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter"}}, From f06c2184b390137349a3dda20a336c3772483a52 Mon Sep 17 00:00:00 2001 From: Alex Harford Date: Tue, 14 Apr 2026 13:03:22 -0700 Subject: [PATCH 3/4] Add tier and network policy watches to gateway controller Add WaitToAddTierWatch and WaitToAddNetworkPolicyWatches for the gateway controller's allow policies, matching the pattern used by other controllers. Add tier existence check during reconciliation. Remove the namespace-scoped default deny from tigera-gateway because Gateway resources dynamically create Envoy proxy pods whose traffic patterns are determined by the customer's Gateway and HTTPRoute configuration. Keep the controller-access and certgen-access allow policies so control plane pods still function under a customer's global default deny. Add FV test verifying the allow policies are created. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../gatewayapi/gatewayapi_controller.go | 34 ++++++++++- pkg/render/gatewayapi/gateway_api.go | 49 +++++++++------ pkg/render/gatewayapi/gateway_api_test.go | 61 +++++++++++++++---- 3 files changed, 112 insertions(+), 32 deletions(-) diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index cf06316985..1041224891 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -47,6 +48,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/gatewayapi" ) @@ -62,6 +64,8 @@ var log = logf.Log.WithName("controller_gatewayapi") // Start Watches within the Add function for any resources that this controller creates or monitors. This will trigger // calls to Reconcile() when an instance of one of the watched resources is modified. func Add(mgr manager.Manager, opts options.ControllerOptions) error { + tierWatchReady := &utils.ReadyFlag{} + r := &ReconcileGatewayAPI{ client: mgr.GetClient(), scheme: mgr.GetScheme(), @@ -90,6 +94,12 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("gatewayapi-controller failed to watch Installation resource: %w", err) } + go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, tierWatchReady) + go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ + {Name: gatewayapi.GatewayControllerPolicyName, Namespace: common.TigeraGatewayNamespace}, + {Name: gatewayapi.GatewayCertgenPolicyName, Namespace: common.TigeraGatewayNamespace}, + }) + // Perform periodic reconciliation. This acts as a backstop to catch reconcile issues, // and also makes sure we spot when things change that might not trigger a reconciliation. if err = utils.AddPeriodicReconcile(c, utils.PeriodicReconcileTime, &handler.EnqueueRequestForObject{}); err != nil { @@ -417,12 +427,34 @@ func (r *ReconcileGatewayAPI) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - err = r.newComponentHandler(log, r.client, r.scheme, gatewayAPI).CreateOrUpdateOrDelete(ctx, nonCRDComponent, r.status) + hdler := r.newComponentHandler(log, r.client, r.scheme, gatewayAPI) + err = hdler.CreateOrUpdateOrDelete(ctx, nonCRDComponent, r.status) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error rendering GatewayAPI resources", err, log) return reconcile.Result{}, err } + // v3 NetworkPolicy will fail to reconcile if the calico-system Tier does not exist or if + // the v3 API is not yet available. Render policies separately so this does not block the + // rest of the gateway resources. + includeV3NetworkPolicy := false + if err := r.client.Get(ctx, client.ObjectKey{Name: networkpolicy.CalicoTierName}, &v3.Tier{}); err != nil { + if !errors.IsNotFound(err) && !meta.IsNoMatchError(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying calico-system tier", err, reqLogger) + return reconcile.Result{}, err + } + } else { + includeV3NetworkPolicy = true + } + + if includeV3NetworkPolicy { + policyComponent := gatewayapi.GatewayPolicy(gatewayConfig) + if err = hdler.CreateOrUpdateOrDelete(ctx, policyComponent, r.status); err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error rendering GatewayAPI network policies", err, log) + return reconcile.Result{}, err + } + } + // Clear the degraded bit if we've reached this far. r.status.ClearDegraded() diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index 19b4de0899..8ae083ee0f 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -92,8 +92,8 @@ const ( EnvoyGatewayConfigKey = "envoy-gateway.yaml" EnvoyGatewayDeploymentContainerName = "envoy-gateway" EnvoyGatewayJobContainerName = "envoy-gateway-certgen" - GatewayControllerPolicyName = networkpolicy.TigeraComponentPolicyPrefix + "gateway-api-controller-access" - GatewayCertgenPolicyName = networkpolicy.TigeraComponentPolicyPrefix + "gateway-api-certgen-access" + GatewayControllerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "gateway-api-controller-access" + GatewayCertgenPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "gateway-api-certgen-access" wafFilterName = "waf-http-filter" // Envoy gateway controller serving ports. @@ -435,6 +435,23 @@ func GatewayAPIImplementationComponent(cfg *GatewayAPIImplementationConfig) rend } } +// GatewayPolicy returns a Component that renders the calico-system tier network policies +// for the gateway control plane. This is rendered separately from the main gateway component +// so that a failure to create v3 NetworkPolicy resources does not block the rest of the +// gateway resources. We intentionally do not include a namespace-scoped default deny here +// because Gateway resources dynamically create Envoy proxy pods whose traffic patterns +// (listener ports, backend destinations) are determined by the customer's Gateway and +// HTTPRoute configuration. +func GatewayPolicy(cfg *GatewayAPIImplementationConfig) render.Component { + return render.NewPassthrough( + []client.Object{ + gatewayCertgenAllowCalicoSystemPolicy(cfg.Installation.KubernetesProvider.IsOpenShift()), + gatewayControllerAllowCalicoSystemPolicy(cfg.Installation.KubernetesProvider.IsOpenShift()), + }, + nil, + ) +} + func (pr *gatewayAPIImplementationComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := pr.cfg.Installation.Registry path := pr.cfg.Installation.ImagePath @@ -646,14 +663,6 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien objs = append(objs, certgenJob) - // Add network policies to allow gateway components to function under a - // default-deny policy. - objs = append(objs, - pr.gatewayCertgenAllowTigeraPolicy(), - pr.gatewayControllerAllowTigeraPolicy(), - networkpolicy.CalicoSystemDefaultDeny(common.TigeraGatewayNamespace), - ) - // Provision GatewayClasses. for i := range pr.cfg.GatewayAPI.Spec.GatewayClasses { className := pr.cfg.GatewayAPI.Spec.GatewayClasses[i].Name @@ -1190,15 +1199,15 @@ func (pr *gatewayAPIImplementationComponent) wafHttpFilterClusterRoleBinding() * } } -// gatewayCertgenAllowTigeraPolicy creates a NetworkPolicy that allows the certgen job +// gatewayCertgenAllowCalicoSystemPolicy creates a NetworkPolicy that allows the certgen job // to access DNS and the Kubernetes API server, which it needs to create TLS secrets. -func (pr *gatewayAPIImplementationComponent) gatewayCertgenAllowTigeraPolicy() *v3.NetworkPolicy { +func gatewayCertgenAllowCalicoSystemPolicy(openShift bool) *v3.NetworkPolicy { egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, pr.cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, openShift) egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.KubeAPIServerServiceSelectorEntityRule, + Destination: networkpolicy.KubeAPIServerEntityRule, }) return &v3.NetworkPolicy{ @@ -1208,7 +1217,7 @@ func (pr *gatewayAPIImplementationComponent) gatewayCertgenAllowTigeraPolicy() * Namespace: common.TigeraGatewayNamespace, }, Spec: v3.NetworkPolicySpec{ - Tier: networkpolicy.TigeraComponentTierName, + Tier: networkpolicy.CalicoTierName, Selector: "app == 'certgen'", Types: []v3.PolicyType{v3.PolicyTypeEgress}, Egress: egressRules, @@ -1216,16 +1225,16 @@ func (pr *gatewayAPIImplementationComponent) gatewayCertgenAllowTigeraPolicy() * } } -// gatewayControllerAllowTigeraPolicy creates a NetworkPolicy that allows the envoy-gateway +// gatewayControllerAllowCalicoSystemPolicy creates a NetworkPolicy that allows the envoy-gateway // controller to access DNS and the Kubernetes API server, and to receive ingress on its // serving ports (xDS gRPC, ratelimit, wasm, metrics, webhook). -func (pr *gatewayAPIImplementationComponent) gatewayControllerAllowTigeraPolicy() *v3.NetworkPolicy { +func gatewayControllerAllowCalicoSystemPolicy(openShift bool) *v3.NetworkPolicy { egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, pr.cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, openShift) egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.KubeAPIServerServiceSelectorEntityRule, + Destination: networkpolicy.KubeAPIServerEntityRule, }) ingressRules := []v3.Rule{ @@ -1245,7 +1254,7 @@ func (pr *gatewayAPIImplementationComponent) gatewayControllerAllowTigeraPolicy( Namespace: common.TigeraGatewayNamespace, }, Spec: v3.NetworkPolicySpec{ - Tier: networkpolicy.TigeraComponentTierName, + Tier: networkpolicy.CalicoTierName, Selector: "k8s-app == '" + GatewayControllerLabel + "'", Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, Ingress: ingressRules, diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index 270b142646..5e40b9b7df 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/gomega" envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -236,7 +235,7 @@ var _ = Describe("Gateway API rendering tests", func() { objsToCreate, objsToDelete := gatewayComp.Objects() Expect(objsToDelete).To(HaveLen(0)) Expect(objsToCreate).NotTo(BeEmpty()) - Expect(objsToCreate).To(HaveLen(23 + len(gatewayAPI.Spec.GatewayClasses))) // 23 core objects plus one per GatewayClass + Expect(objsToCreate).To(HaveLen(20 + len(gatewayAPI.Spec.GatewayClasses))) // 20 core objects plus one per GatewayClass rtest.ExpectResources(objsToCreate, []client.Object{ &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway"}}, @@ -258,9 +257,6 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-controller-access", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &envoyapi.EnvoyProxy{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, }) @@ -362,9 +358,6 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-controller-access", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &envoyapi.EnvoyProxy{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret1", Namespace: "tigera-gateway"}}, &gapi.GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-class", Namespace: "tigera-gateway"}}, @@ -458,9 +451,6 @@ var _ = Describe("Gateway API rendering tests", func() { &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-certgen-access", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.gateway-api-controller-access", Namespace: "tigera-gateway"}}, - &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "calico-system.default-deny", Namespace: "tigera-gateway"}}, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter", Namespace: "tigera-gateway"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "waf-http-filter"}}, @@ -1305,4 +1295,53 @@ var _ = Describe("Gateway API rendering tests", func() { Expect(serviceAccount.Name).To(Equal("waf-http-filter")) Expect(serviceAccount.Namespace).To(Equal("tigera-gateway")) }) + + Context("GatewayPolicy component", func() { + It("should render the calico-system tier allow policies", func() { + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + } + gatewayAPI := &operatorv1.GatewayAPI{} + policyComp := GatewayPolicy(&GatewayAPIImplementationConfig{ + Installation: installation, + GatewayAPI: gatewayAPI, + }) + + objsToCreate, objsToDelete := policyComp.Objects() + Expect(objsToDelete).To(BeEmpty()) + Expect(objsToCreate).To(HaveLen(2)) + + expectedPolicies := []struct { + name string + namespace string + }{ + {name: "calico-system.gateway-api-certgen-access", namespace: "tigera-gateway"}, + {name: "calico-system.gateway-api-controller-access", namespace: "tigera-gateway"}, + } + for _, expected := range expectedPolicies { + Expect(objsToCreate).To(ContainElement(&matchObject{name: expected.name}), + fmt.Sprintf("expected NetworkPolicy %q in namespace %q", expected.name, expected.namespace)) + } + + // Spot-check that the rendered objects are Calico v3 NetworkPolicies in the calico-system tier. + for _, obj := range objsToCreate { + Expect(obj.GetObjectKind().GroupVersionKind().Kind).To(Equal("NetworkPolicy")) + Expect(obj.GetObjectKind().GroupVersionKind().GroupVersion().String()).To(Equal("projectcalico.org/v3")) + Expect(obj.GetNamespace()).To(Equal("tigera-gateway")) + } + }) + + It("should render OpenShift-appropriate DNS egress rules when on OpenShift", func() { + installation := &operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderOpenShift, + } + policyComp := GatewayPolicy(&GatewayAPIImplementationConfig{ + Installation: installation, + GatewayAPI: &operatorv1.GatewayAPI{}, + }) + + objsToCreate, _ := policyComp.Objects() + Expect(objsToCreate).To(HaveLen(2)) + }) + }) }) From cba40167299f7061e6bd1c58fef854e0347f9c4f Mon Sep 17 00:00:00 2001 From: Alex Harford Date: Wed, 15 Apr 2026 09:28:25 -0700 Subject: [PATCH 4/4] Add FV test for gateway controller coming up under default-deny Covers the scenario the calico-system tier allow policies were added to support: a default-tier v3 NetworkPolicy with selector all() is applied to tigera-gateway, then a GatewayAPI CR is created. The test asserts the envoy-gateway-certgen Job reaches Succeeded and the envoy-gateway Deployment becomes Ready despite the default-deny, which is only possible if gateway-api-certgen-access and gateway-api-controller-access in the higher-precedence calico-system tier correctly permit the traffic those workloads need. Also exercises the v3.NetworkPolicy watch by deleting each allow policy and asserting it's re-created, guarding against regressions in WaitToAddNetworkPolicyWatches. Adds calico/envoy-gateway to the FV kind cluster image load list so the gateway Deployment's image is available. Co-Authored-By: Claude Opus 4.6 (1M context) --- Makefile | 9 +- test/gatewayapi_test.go | 218 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 372418f02a..5e9dab49e2 100644 --- a/Makefile +++ b/Makefile @@ -388,6 +388,7 @@ NODE_DRIVER_REGISTRAR_IMAGE := calico/node-driver-registrar GOLDMANE_IMAGE := calico/goldmane WHISKER_IMAGE := calico/whisker WHISKER_BACKEND_IMAGE := calico/whisker-backend +ENVOY_GATEWAY_IMAGE := calico/envoy-gateway .PHONY: calico-node.tar calico-node.tar: @@ -444,6 +445,11 @@ calico-whisker-backend.tar: docker pull $(FV_IMAGE_REGISTRY)/$(WHISKER_BACKEND_IMAGE):$(VERSION_TAG) docker save --output $@ $(WHISKER_BACKEND_IMAGE):$(VERSION_TAG) +.PHONY: calico-envoy-gateway.tar +calico-envoy-gateway.tar: + docker pull $(FV_IMAGE_REGISTRY)/$(ENVOY_GATEWAY_IMAGE):$(VERSION_TAG) + docker save --output $@ $(ENVOY_GATEWAY_IMAGE):$(VERSION_TAG) + IMAGE_TARS := calico-node.tar \ calico-apiserver.tar \ calico-cni.tar \ @@ -454,7 +460,8 @@ IMAGE_TARS := calico-node.tar \ calico-node-driver-registrar.tar \ calico-goldmane.tar \ calico-whisker.tar \ - calico-whisker-backend.tar + calico-whisker-backend.tar \ + calico-envoy-gateway.tar load-container-images: ./test/load_images_on_kind_cluster.sh $(IMAGE_TARS) # Load the latest tar files onto the currently running kind cluster. diff --git a/test/gatewayapi_test.go b/test/gatewayapi_test.go index 7ac007e315..bc1304b48a 100644 --- a/test/gatewayapi_test.go +++ b/test/gatewayapi_test.go @@ -25,6 +25,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" kerror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,11 +36,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operator "github.com/tigera/operator/api/v1" "github.com/tigera/operator/internal/controller" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" logf "sigs.k8s.io/controller-runtime/pkg/log" + gapi "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/yaml" // gopkg.in/yaml.v2 didn't parse all the fields but this package did ) @@ -515,6 +519,220 @@ var _ = Describe("GatewayAPI tests", func() { }) }) +// This Describe exercises the gateway controller's v3 NetworkPolicy rendering path. Unlike the +// suite above (which runs only the GatewayAPI controller against a cluster without the v3 API +// available), this one stands up the full Calico + APIServer stack so that the projectcalico.org/v3 +// aggregated API becomes reachable. With v3 present, the controller should render the two +// calico-system tier allow policies (GatewayPolicy component) and keep them in place via the +// NetworkPolicy watch. +var _ = Describe("GatewayAPI NetworkPolicy tests", func() { + var c client.Client + var clientv3 client.Client + var mgr manager.Manager + var shutdownContext context.Context + var cancel context.CancelFunc + var operatorDone chan struct{} + + BeforeEach(func() { + c, shutdownContext, cancel, mgr = setupManager(ManageCRDsDisable, SingleTenant, EnterpriseCRDsExist) + + // Separate client for projectcalico.org/v3 resources. + var err error + clientv3, err = utils.V3Client(mgr.GetConfig()) + Expect(err).NotTo(HaveOccurred()) + + By("Cleaning up resources before the test") + cleanupGatewayResources(c) + cleanupResources(c) + + By("Verifying CRDs are installed") + verifyCRDsExist(c, operator.CalicoEnterprise) + + By("Creating the tigera-operator namespace, if it doesn't exist") + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-operator"}, + Spec: corev1.NamespaceSpec{}, + } + err = c.Create(context.Background(), ns) + if err != nil && !kerror.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + By("Checking no Installation is left over from previous tests") + instance := &operator.Installation{ + TypeMeta: metav1.TypeMeta{Kind: "Installation", APIVersion: "operator.tigera.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + } + err = c.Get(context.Background(), types.NamespacedName{Name: "default"}, instance) + Expect(kerror.IsNotFound(err)).To(BeTrue(), fmt.Sprintf("Expected Installation not to exist, but got: %s", err)) + }) + + AfterEach(func() { + defer func() { + cancel() + if operatorDone != nil { + Eventually(func() error { + select { + case <-operatorDone: + return nil + default: + return fmt.Errorf("operator did not shutdown") + } + }, 60*time.Second).ShouldNot(HaveOccurred()) + } + }() + + By("Cleaning up resources after the test") + cleanupGatewayResources(c) + cleanupResources(c) + + // Clean up Calico data that might be left behind. + Eventually(func() error { + cs := kubernetes.NewForConfigOrDie(mgr.GetConfig()) + nodes, err := cs.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + if len(nodes.Items) == 0 { + return fmt.Errorf("No nodes found") + } + for _, n := range nodes.Items { + for k := range n.ObjectMeta.Annotations { + if strings.Contains(k, "projectcalico") { + delete(n.ObjectMeta.Annotations, k) + } + } + _, err = cs.CoreV1().Nodes().Update(context.Background(), &n, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + return nil + }, 30*time.Second).Should(BeNil()) + + mgr = nil + }) + + It("comes up under default-deny and re-creates allow policies on deletion", func() { + By("Installing Calico (OSS variant — the FV kind cluster only has OSS images loaded)") + // We intentionally leave Variant unset so the installation controller uses the + // Calico variant images that are pre-loaded into the kind cluster. Tiers and v3 + // NetworkPolicies are part of OSS, so the gateway controller's v3 NetworkPolicy + // rendering path is fully exercised regardless of variant. + operatorDone = createInstallation(c, mgr, shutdownContext, nil) + verifyCalicoHasDeployed(c) + + By("Installing the APIServer so the projectcalico.org/v3 aggregated API becomes available") + createAPIServer(c, mgr, shutdownContext, nil) + verifyAPIServerHasDeployed(c) + + By("Pre-creating the tigera-gateway namespace so we can apply default-deny before the Gateway controller starts rendering into it") + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway"}, + } + if err := c.Create(shutdownContext, ns); err != nil && !kerror.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + By("Applying the recommended default-tier default-deny (v3 NetworkPolicy, selector all()) in tigera-gateway") + // This mirrors the docs.tigera.io/.../default-deny guidance: a default-tier NetworkPolicy + // with selector all() and no rules, denying all ingress/egress to pods in the namespace. + // Our two allow policies in the higher-precedence calico-system tier must let the + // envoy-gateway certgen Job and controller Deployment complete despite this deny. + defaultDeny := &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default.default-deny", + Namespace: "tigera-gateway", + }, + Spec: v3.NetworkPolicySpec{ + Tier: "default", + Selector: "all()", + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + }, + } + Expect(clientv3.Create(shutdownContext, defaultDeny)).To(Succeed()) + + By("Creating the default GatewayAPI") + gatewayAPI := &operator.GatewayAPI{ + TypeMeta: metav1.TypeMeta{Kind: "GatewayAPI", APIVersion: "operator.tigera.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + } + err := c.Create(shutdownContext, gatewayAPI) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the envoy-gateway-certgen Job to complete under default-deny") + // If gateway-api-certgen-access isn't letting certgen reach DNS + the kube API server, + // the Job will never succeed and this Eventually will time out. + Eventually(func() error { + job := &batchv1.Job{} + if err := c.Get(shutdownContext, types.NamespacedName{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}, job); err != nil { + return err + } + if job.Status.Succeeded < 1 { + return fmt.Errorf("certgen Job not yet succeeded (succeeded=%d, failed=%d)", job.Status.Succeeded, job.Status.Failed) + } + return nil + }, 180*time.Second, 2*time.Second).ShouldNot(HaveOccurred()) + + By("Waiting for the envoy-gateway Deployment to become Ready under default-deny") + // If gateway-api-controller-access isn't letting the controller reach DNS/API or accept + // its listener traffic, the Deployment will never report AvailableReplicas >= 1. + Eventually(func() error { + deploy := &appsv1.Deployment{} + if err := c.Get(shutdownContext, types.NamespacedName{Name: "envoy-gateway", Namespace: "tigera-gateway"}, deploy); err != nil { + return err + } + if deploy.Status.AvailableReplicas < 1 { + return fmt.Errorf("envoy-gateway Deployment not ready (available=%d)", deploy.Status.AvailableReplicas) + } + return nil + }, 180*time.Second, 2*time.Second).ShouldNot(HaveOccurred()) + + certgenPolicyKey := types.NamespacedName{ + Name: "calico-system.gateway-api-certgen-access", + Namespace: "tigera-gateway", + } + controllerPolicyKey := types.NamespacedName{ + Name: "calico-system.gateway-api-controller-access", + Namespace: "tigera-gateway", + } + + By("Verifying both calico-system tier allow NetworkPolicies are created in tigera-gateway") + // Generous timeout: the gateway controller has to observe the tier becoming available, + // establish its Tier and NetworkPolicy watches, and render the policy component. + Eventually(func() error { + for _, key := range []types.NamespacedName{certgenPolicyKey, controllerPolicyKey} { + np := &v3.NetworkPolicy{} + if err := clientv3.Get(shutdownContext, key, np); err != nil { + return fmt.Errorf("NetworkPolicy %s/%s not found: %w", key.Namespace, key.Name, err) + } + if np.Spec.Tier != "calico-system" { + return fmt.Errorf("NetworkPolicy %s/%s has unexpected tier %q", key.Namespace, key.Name, np.Spec.Tier) + } + } + return nil + }, 120*time.Second, 2*time.Second).ShouldNot(HaveOccurred()) + + By("Deleting each NetworkPolicy and verifying the watch triggers re-creation") + // Both policies are registered in the same WaitToAddNetworkPolicyWatches call; exercising + // each one guards against a future refactor accidentally dropping an entry from that list. + for _, key := range []types.NamespacedName{certgenPolicyKey, controllerPolicyKey} { + Expect(clientv3.Delete(shutdownContext, &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + })).To(Succeed()) + + Eventually(func() error { + return clientv3.Get(shutdownContext, key, &v3.NetworkPolicy{}) + }, 60*time.Second, 2*time.Second).ShouldNot(HaveOccurred(), + fmt.Sprintf("expected gateway controller's NetworkPolicy watch to re-create deleted policy %s", key.Name)) + } + }) +}) + func cleanupGatewayResources(c client.Client) { By("Cleaning up custom EnvoyGateway") Eventually(func() error {