diff --git a/Makefile b/Makefile index 372418f02a..5e9dab49e2 100644 --- a/Makefile +++ b/Makefile @@ -388,6 +388,7 @@ NODE_DRIVER_REGISTRAR_IMAGE := calico/node-driver-registrar GOLDMANE_IMAGE := calico/goldmane WHISKER_IMAGE := calico/whisker WHISKER_BACKEND_IMAGE := calico/whisker-backend +ENVOY_GATEWAY_IMAGE := calico/envoy-gateway .PHONY: calico-node.tar calico-node.tar: @@ -444,6 +445,11 @@ calico-whisker-backend.tar: docker pull $(FV_IMAGE_REGISTRY)/$(WHISKER_BACKEND_IMAGE):$(VERSION_TAG) docker save --output $@ $(WHISKER_BACKEND_IMAGE):$(VERSION_TAG) +.PHONY: calico-envoy-gateway.tar +calico-envoy-gateway.tar: + docker pull $(FV_IMAGE_REGISTRY)/$(ENVOY_GATEWAY_IMAGE):$(VERSION_TAG) + docker save --output $@ $(ENVOY_GATEWAY_IMAGE):$(VERSION_TAG) + IMAGE_TARS := calico-node.tar \ calico-apiserver.tar \ calico-cni.tar \ @@ -454,7 +460,8 @@ IMAGE_TARS := calico-node.tar \ calico-node-driver-registrar.tar \ calico-goldmane.tar \ calico-whisker.tar \ - calico-whisker-backend.tar + calico-whisker-backend.tar \ + calico-envoy-gateway.tar load-container-images: ./test/load_images_on_kind_cluster.sh $(IMAGE_TARS) # Load the latest tar files onto the currently running kind cluster. diff --git a/pkg/common/common.go b/pkg/common/common.go index efdf171732..731511771f 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -21,6 +21,9 @@ const ( KubeControllersDeploymentName = "calico-kube-controllers" WindowsDaemonSetName = "calico-node-windows" + // Gateway API related const + TigeraGatewayNamespace = "tigera-gateway" + // Monitor + Prometheus related const TigeraPrometheusNamespace = "tigera-prometheus" diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index d6744dc288..1041224891 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -40,12 +41,14 @@ import ( envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/go-logr/logr" operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/gatewayapi" ) @@ -61,6 +64,8 @@ var log = logf.Log.WithName("controller_gatewayapi") // Start Watches within the Add function for any resources that this controller creates or monitors. This will trigger // calls to Reconcile() when an instance of one of the watched resources is modified. func Add(mgr manager.Manager, opts options.ControllerOptions) error { + tierWatchReady := &utils.ReadyFlag{} + r := &ReconcileGatewayAPI{ client: mgr.GetClient(), scheme: mgr.GetScheme(), @@ -89,6 +94,12 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("gatewayapi-controller failed to watch Installation resource: %w", err) } + go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, tierWatchReady) + go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ + {Name: gatewayapi.GatewayControllerPolicyName, Namespace: common.TigeraGatewayNamespace}, + {Name: gatewayapi.GatewayCertgenPolicyName, Namespace: common.TigeraGatewayNamespace}, + }) + // Perform periodic reconciliation. This acts as a backstop to catch reconcile issues, // and also makes sure we spot when things change that might not trigger a reconciliation. if err = utils.AddPeriodicReconcile(c, utils.PeriodicReconcileTime, &handler.EnqueueRequestForObject{}); err != nil { @@ -416,12 +427,34 @@ func (r *ReconcileGatewayAPI) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - err = r.newComponentHandler(log, r.client, r.scheme, gatewayAPI).CreateOrUpdateOrDelete(ctx, nonCRDComponent, r.status) + hdler := r.newComponentHandler(log, r.client, r.scheme, gatewayAPI) + err = hdler.CreateOrUpdateOrDelete(ctx, nonCRDComponent, r.status) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error rendering GatewayAPI resources", err, log) return reconcile.Result{}, err } + // v3 NetworkPolicy will fail to reconcile if the calico-system Tier does not exist or if + // the v3 API is not yet available. Render policies separately so this does not block the + // rest of the gateway resources. + includeV3NetworkPolicy := false + if err := r.client.Get(ctx, client.ObjectKey{Name: networkpolicy.CalicoTierName}, &v3.Tier{}); err != nil { + if !errors.IsNotFound(err) && !meta.IsNoMatchError(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying calico-system tier", err, reqLogger) + return reconcile.Result{}, err + } + } else { + includeV3NetworkPolicy = true + } + + if includeV3NetworkPolicy { + policyComponent := gatewayapi.GatewayPolicy(gatewayConfig) + if err = hdler.CreateOrUpdateOrDelete(ctx, policyComponent, r.status); err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error rendering GatewayAPI network policies", err, log) + return reconcile.Result{}, err + } + } + // Clear the degraded bit if we've reached this far. r.status.ClearDegraded() @@ -496,6 +529,6 @@ func (r *ReconcileGatewayAPI) getPolicySyncPathPrefix(fcSpec *v3.FelixConfigurat // The bool return value indicates if the finalizer is Set func (r *ReconcileGatewayAPI) maintainFinalizer(ctx context.Context, gatewayAPI client.Object) (bool, error) { // These objects require graceful termination before the CNI plugin is torn down. - gatewayAPIDeployment := v1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway", Namespace: "tigera-gateway"}} + gatewayAPIDeployment := v1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "envoy-gateway", Namespace: common.TigeraGatewayNamespace}} return utils.MaintainInstallationFinalizer(ctx, r.client, gatewayAPI, render.GatewayAPIFinalizer, &gatewayAPIDeployment) } diff --git a/pkg/render/gatewayapi/gateway_api.go b/pkg/render/gatewayapi/gateway_api.go index e64cbe6f95..8ae083ee0f 100644 --- a/pkg/render/gatewayapi/gateway_api.go +++ b/pkg/render/gatewayapi/gateway_api.go @@ -22,12 +22,14 @@ import ( "sync" envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/render" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" + networkpolicy "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" admissionregv1 "k8s.io/api/admissionregistration/v1" @@ -90,7 +92,16 @@ const ( EnvoyGatewayConfigKey = "envoy-gateway.yaml" EnvoyGatewayDeploymentContainerName = "envoy-gateway" EnvoyGatewayJobContainerName = "envoy-gateway-certgen" + GatewayControllerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "gateway-api-controller-access" + GatewayCertgenPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "gateway-api-certgen-access" wafFilterName = "waf-http-filter" + + // Envoy gateway controller serving ports. + EnvoyGatewayPortGRPC = 18000 + EnvoyGatewayPortRateLimit = 18001 + EnvoyGatewayPortWasm = 18002 + EnvoyGatewayPortMetrics = 19001 + EnvoyGatewayPortWebhook = 9443 ) var ( @@ -424,6 +435,23 @@ func GatewayAPIImplementationComponent(cfg *GatewayAPIImplementationConfig) rend } } +// GatewayPolicy returns a Component that renders the calico-system tier network policies +// for the gateway control plane. This is rendered separately from the main gateway component +// so that a failure to create v3 NetworkPolicy resources does not block the rest of the +// gateway resources. We intentionally do not include a namespace-scoped default deny here +// because Gateway resources dynamically create Envoy proxy pods whose traffic patterns +// (listener ports, backend destinations) are determined by the customer's Gateway and +// HTTPRoute configuration. +func GatewayPolicy(cfg *GatewayAPIImplementationConfig) render.Component { + return render.NewPassthrough( + []client.Object{ + gatewayCertgenAllowCalicoSystemPolicy(cfg.Installation.KubernetesProvider.IsOpenShift()), + gatewayControllerAllowCalicoSystemPolicy(cfg.Installation.KubernetesProvider.IsOpenShift()), + }, + nil, + ) +} + func (pr *gatewayAPIImplementationComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := pr.cfg.Installation.Registry path := pr.cfg.Installation.ImagePath @@ -671,7 +699,7 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien }, ObjectMeta: metav1.ObjectMeta{ Name: gcName, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, }, }, ) @@ -706,7 +734,7 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className string, envoyProxy.APIVersion = "gateway.envoyproxy.io/v1alpha1" } envoyProxy.Name = className - envoyProxy.Namespace = "tigera-gateway" + envoyProxy.Namespace = common.TigeraGatewayNamespace if envoyProxy.Spec.Provider == nil { envoyProxy.Spec.Provider = &envoyapi.EnvoyProxyProvider{} } @@ -1052,7 +1080,7 @@ func (pr *gatewayAPIImplementationComponent) gatewayClass(className, controllerN TypeMeta: metav1.TypeMeta{Kind: "GatewayClass", APIVersion: "gateway.networking.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ Name: className, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, }, Spec: gapi.GatewayClassSpec{ ControllerName: gapi.GatewayController(controllerName), @@ -1114,7 +1142,7 @@ func (pr *gatewayAPIImplementationComponent) wafHttpFilterServiceAccount() *core TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: wafFilterName, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, }, } } @@ -1165,8 +1193,72 @@ func (pr *gatewayAPIImplementationComponent) wafHttpFilterClusterRoleBinding() * { Kind: "ServiceAccount", Name: wafFilterName, - Namespace: "tigera-gateway", + Namespace: common.TigeraGatewayNamespace, + }, + }, + } +} + +// gatewayCertgenAllowCalicoSystemPolicy creates a NetworkPolicy that allows the certgen job +// to access DNS and the Kubernetes API server, which it needs to create TLS secrets. +func gatewayCertgenAllowCalicoSystemPolicy(openShift bool) *v3.NetworkPolicy { + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, openShift) + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerEntityRule, + }) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: GatewayCertgenPolicyName, + Namespace: common.TigeraGatewayNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Tier: networkpolicy.CalicoTierName, + Selector: "app == 'certgen'", + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} + +// gatewayControllerAllowCalicoSystemPolicy creates a NetworkPolicy that allows the envoy-gateway +// controller to access DNS and the Kubernetes API server, and to receive ingress on its +// serving ports (xDS gRPC, ratelimit, wasm, metrics, webhook). +func gatewayControllerAllowCalicoSystemPolicy(openShift bool) *v3.NetworkPolicy { + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, openShift) + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerEntityRule, + }) + + ingressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(EnvoyGatewayPortGRPC, EnvoyGatewayPortRateLimit, EnvoyGatewayPortWasm, EnvoyGatewayPortMetrics, EnvoyGatewayPortWebhook), }, }, } + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: GatewayControllerPolicyName, + Namespace: common.TigeraGatewayNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Tier: networkpolicy.CalicoTierName, + Selector: "k8s-app == '" + GatewayControllerLabel + "'", + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, + } } diff --git a/pkg/render/gatewayapi/gateway_api_test.go b/pkg/render/gatewayapi/gateway_api_test.go index 1ace54dcd1..5e40b9b7df 100644 --- a/pkg/render/gatewayapi/gateway_api_test.go +++ b/pkg/render/gatewayapi/gateway_api_test.go @@ -1295,4 +1295,53 @@ var _ = Describe("Gateway API rendering tests", func() { Expect(serviceAccount.Name).To(Equal("waf-http-filter")) Expect(serviceAccount.Namespace).To(Equal("tigera-gateway")) }) + + Context("GatewayPolicy component", func() { + It("should render the calico-system tier allow policies", func() { + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + } + gatewayAPI := &operatorv1.GatewayAPI{} + policyComp := GatewayPolicy(&GatewayAPIImplementationConfig{ + Installation: installation, + GatewayAPI: gatewayAPI, + }) + + objsToCreate, objsToDelete := policyComp.Objects() + Expect(objsToDelete).To(BeEmpty()) + Expect(objsToCreate).To(HaveLen(2)) + + expectedPolicies := []struct { + name string + namespace string + }{ + {name: "calico-system.gateway-api-certgen-access", namespace: "tigera-gateway"}, + {name: "calico-system.gateway-api-controller-access", namespace: "tigera-gateway"}, + } + for _, expected := range expectedPolicies { + Expect(objsToCreate).To(ContainElement(&matchObject{name: expected.name}), + fmt.Sprintf("expected NetworkPolicy %q in namespace %q", expected.name, expected.namespace)) + } + + // Spot-check that the rendered objects are Calico v3 NetworkPolicies in the calico-system tier. + for _, obj := range objsToCreate { + Expect(obj.GetObjectKind().GroupVersionKind().Kind).To(Equal("NetworkPolicy")) + Expect(obj.GetObjectKind().GroupVersionKind().GroupVersion().String()).To(Equal("projectcalico.org/v3")) + Expect(obj.GetNamespace()).To(Equal("tigera-gateway")) + } + }) + + It("should render OpenShift-appropriate DNS egress rules when on OpenShift", func() { + installation := &operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderOpenShift, + } + policyComp := GatewayPolicy(&GatewayAPIImplementationConfig{ + Installation: installation, + GatewayAPI: &operatorv1.GatewayAPI{}, + }) + + objsToCreate, _ := policyComp.Objects() + Expect(objsToCreate).To(HaveLen(2)) + }) + }) }) diff --git a/test/gatewayapi_test.go b/test/gatewayapi_test.go index 7ac007e315..bc1304b48a 100644 --- a/test/gatewayapi_test.go +++ b/test/gatewayapi_test.go @@ -25,6 +25,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" kerror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,11 +36,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" envoyapi "github.com/envoyproxy/gateway/api/v1alpha1" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" operator "github.com/tigera/operator/api/v1" "github.com/tigera/operator/internal/controller" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" logf "sigs.k8s.io/controller-runtime/pkg/log" + gapi "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/yaml" // gopkg.in/yaml.v2 didn't parse all the fields but this package did ) @@ -515,6 +519,220 @@ var _ = Describe("GatewayAPI tests", func() { }) }) +// This Describe exercises the gateway controller's v3 NetworkPolicy rendering path. Unlike the +// suite above (which runs only the GatewayAPI controller against a cluster without the v3 API +// available), this one stands up the full Calico + APIServer stack so that the projectcalico.org/v3 +// aggregated API becomes reachable. With v3 present, the controller should render the two +// calico-system tier allow policies (GatewayPolicy component) and keep them in place via the +// NetworkPolicy watch. +var _ = Describe("GatewayAPI NetworkPolicy tests", func() { + var c client.Client + var clientv3 client.Client + var mgr manager.Manager + var shutdownContext context.Context + var cancel context.CancelFunc + var operatorDone chan struct{} + + BeforeEach(func() { + c, shutdownContext, cancel, mgr = setupManager(ManageCRDsDisable, SingleTenant, EnterpriseCRDsExist) + + // Separate client for projectcalico.org/v3 resources. + var err error + clientv3, err = utils.V3Client(mgr.GetConfig()) + Expect(err).NotTo(HaveOccurred()) + + By("Cleaning up resources before the test") + cleanupGatewayResources(c) + cleanupResources(c) + + By("Verifying CRDs are installed") + verifyCRDsExist(c, operator.CalicoEnterprise) + + By("Creating the tigera-operator namespace, if it doesn't exist") + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-operator"}, + Spec: corev1.NamespaceSpec{}, + } + err = c.Create(context.Background(), ns) + if err != nil && !kerror.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + By("Checking no Installation is left over from previous tests") + instance := &operator.Installation{ + TypeMeta: metav1.TypeMeta{Kind: "Installation", APIVersion: "operator.tigera.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + } + err = c.Get(context.Background(), types.NamespacedName{Name: "default"}, instance) + Expect(kerror.IsNotFound(err)).To(BeTrue(), fmt.Sprintf("Expected Installation not to exist, but got: %s", err)) + }) + + AfterEach(func() { + defer func() { + cancel() + if operatorDone != nil { + Eventually(func() error { + select { + case <-operatorDone: + return nil + default: + return fmt.Errorf("operator did not shutdown") + } + }, 60*time.Second).ShouldNot(HaveOccurred()) + } + }() + + By("Cleaning up resources after the test") + cleanupGatewayResources(c) + cleanupResources(c) + + // Clean up Calico data that might be left behind. + Eventually(func() error { + cs := kubernetes.NewForConfigOrDie(mgr.GetConfig()) + nodes, err := cs.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + if len(nodes.Items) == 0 { + return fmt.Errorf("No nodes found") + } + for _, n := range nodes.Items { + for k := range n.ObjectMeta.Annotations { + if strings.Contains(k, "projectcalico") { + delete(n.ObjectMeta.Annotations, k) + } + } + _, err = cs.CoreV1().Nodes().Update(context.Background(), &n, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + return nil + }, 30*time.Second).Should(BeNil()) + + mgr = nil + }) + + It("comes up under default-deny and re-creates allow policies on deletion", func() { + By("Installing Calico (OSS variant — the FV kind cluster only has OSS images loaded)") + // We intentionally leave Variant unset so the installation controller uses the + // Calico variant images that are pre-loaded into the kind cluster. Tiers and v3 + // NetworkPolicies are part of OSS, so the gateway controller's v3 NetworkPolicy + // rendering path is fully exercised regardless of variant. + operatorDone = createInstallation(c, mgr, shutdownContext, nil) + verifyCalicoHasDeployed(c) + + By("Installing the APIServer so the projectcalico.org/v3 aggregated API becomes available") + createAPIServer(c, mgr, shutdownContext, nil) + verifyAPIServerHasDeployed(c) + + By("Pre-creating the tigera-gateway namespace so we can apply default-deny before the Gateway controller starts rendering into it") + ns := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-gateway"}, + } + if err := c.Create(shutdownContext, ns); err != nil && !kerror.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + By("Applying the recommended default-tier default-deny (v3 NetworkPolicy, selector all()) in tigera-gateway") + // This mirrors the docs.tigera.io/.../default-deny guidance: a default-tier NetworkPolicy + // with selector all() and no rules, denying all ingress/egress to pods in the namespace. + // Our two allow policies in the higher-precedence calico-system tier must let the + // envoy-gateway certgen Job and controller Deployment complete despite this deny. + defaultDeny := &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "default.default-deny", + Namespace: "tigera-gateway", + }, + Spec: v3.NetworkPolicySpec{ + Tier: "default", + Selector: "all()", + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + }, + } + Expect(clientv3.Create(shutdownContext, defaultDeny)).To(Succeed()) + + By("Creating the default GatewayAPI") + gatewayAPI := &operator.GatewayAPI{ + TypeMeta: metav1.TypeMeta{Kind: "GatewayAPI", APIVersion: "operator.tigera.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + } + err := c.Create(shutdownContext, gatewayAPI) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the envoy-gateway-certgen Job to complete under default-deny") + // If gateway-api-certgen-access isn't letting certgen reach DNS + the kube API server, + // the Job will never succeed and this Eventually will time out. + Eventually(func() error { + job := &batchv1.Job{} + if err := c.Get(shutdownContext, types.NamespacedName{Name: "tigera-gateway-api-gateway-helm-certgen", Namespace: "tigera-gateway"}, job); err != nil { + return err + } + if job.Status.Succeeded < 1 { + return fmt.Errorf("certgen Job not yet succeeded (succeeded=%d, failed=%d)", job.Status.Succeeded, job.Status.Failed) + } + return nil + }, 180*time.Second, 2*time.Second).ShouldNot(HaveOccurred()) + + By("Waiting for the envoy-gateway Deployment to become Ready under default-deny") + // If gateway-api-controller-access isn't letting the controller reach DNS/API or accept + // its listener traffic, the Deployment will never report AvailableReplicas >= 1. + Eventually(func() error { + deploy := &appsv1.Deployment{} + if err := c.Get(shutdownContext, types.NamespacedName{Name: "envoy-gateway", Namespace: "tigera-gateway"}, deploy); err != nil { + return err + } + if deploy.Status.AvailableReplicas < 1 { + return fmt.Errorf("envoy-gateway Deployment not ready (available=%d)", deploy.Status.AvailableReplicas) + } + return nil + }, 180*time.Second, 2*time.Second).ShouldNot(HaveOccurred()) + + certgenPolicyKey := types.NamespacedName{ + Name: "calico-system.gateway-api-certgen-access", + Namespace: "tigera-gateway", + } + controllerPolicyKey := types.NamespacedName{ + Name: "calico-system.gateway-api-controller-access", + Namespace: "tigera-gateway", + } + + By("Verifying both calico-system tier allow NetworkPolicies are created in tigera-gateway") + // Generous timeout: the gateway controller has to observe the tier becoming available, + // establish its Tier and NetworkPolicy watches, and render the policy component. + Eventually(func() error { + for _, key := range []types.NamespacedName{certgenPolicyKey, controllerPolicyKey} { + np := &v3.NetworkPolicy{} + if err := clientv3.Get(shutdownContext, key, np); err != nil { + return fmt.Errorf("NetworkPolicy %s/%s not found: %w", key.Namespace, key.Name, err) + } + if np.Spec.Tier != "calico-system" { + return fmt.Errorf("NetworkPolicy %s/%s has unexpected tier %q", key.Namespace, key.Name, np.Spec.Tier) + } + } + return nil + }, 120*time.Second, 2*time.Second).ShouldNot(HaveOccurred()) + + By("Deleting each NetworkPolicy and verifying the watch triggers re-creation") + // Both policies are registered in the same WaitToAddNetworkPolicyWatches call; exercising + // each one guards against a future refactor accidentally dropping an entry from that list. + for _, key := range []types.NamespacedName{certgenPolicyKey, controllerPolicyKey} { + Expect(clientv3.Delete(shutdownContext, &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + })).To(Succeed()) + + Eventually(func() error { + return clientv3.Get(shutdownContext, key, &v3.NetworkPolicy{}) + }, 60*time.Second, 2*time.Second).ShouldNot(HaveOccurred(), + fmt.Sprintf("expected gateway controller's NetworkPolicy watch to re-create deleted policy %s", key.Name)) + } + }) +}) + func cleanupGatewayResources(c client.Client) { By("Cleaning up custom EnvoyGateway") Eventually(func() error {