From acb65a7bc9adfd9500f7811288158d75b907216b Mon Sep 17 00:00:00 2001 From: pythoner6 Date: Tue, 20 Aug 2024 01:14:14 -0400 Subject: [PATCH] Add talos provider --- api/v1/installation_types.go | 9 +++-- pkg/controller/k8sapi/k8s-endpoint.go | 6 ++++ pkg/controller/utils/discovery.go | 25 ++++++++++++++ pkg/controller/utils/discovery_test.go | 17 ++++++++++ .../operator.tigera.io_installations.yaml | 2 ++ pkg/render/apiserver_test.go | 34 +++++++++++++++++++ .../kubecontrollers/kube-controllers_test.go | 15 ++++++++ 7 files changed, 106 insertions(+), 2 deletions(-) diff --git a/api/v1/installation_types.go b/api/v1/installation_types.go index be4991d92c..545f0f1ba0 100644 --- a/api/v1/installation_types.go +++ b/api/v1/installation_types.go @@ -82,7 +82,7 @@ type InstallationSpec struct { // If the specified value is not empty, the Operator will still attempt auto-detection, but // will additionally compare the auto-detected value to the specified value to confirm they match. // +optional - // +kubebuilder:validation:Enum="";EKS;GKE;AKS;OpenShift;DockerEnterprise;RKE2;TKG; + // +kubebuilder:validation:Enum="";EKS;GKE;AKS;OpenShift;DockerEnterprise;RKE2;TKG;Talos; KubernetesProvider Provider `json:"kubernetesProvider,omitempty"` // CNI specifies the CNI that will be used by this installation. @@ -366,7 +366,7 @@ type ComponentResource struct { } // Provider represents a particular provider or flavor of Kubernetes. Valid options -// are: EKS, GKE, AKS, RKE2, OpenShift, DockerEnterprise, TKG. +// are: EKS, GKE, AKS, RKE2, OpenShift, DockerEnterprise, TKG, Talos. type Provider string var ( @@ -378,6 +378,7 @@ var ( ProviderOpenShift Provider = "OpenShift" ProviderDockerEE Provider = "DockerEnterprise" ProviderTKG Provider = "TKG" + ProviderTalos Provider = "Talos" ) func (p Provider) IsNone() bool { @@ -412,6 +413,10 @@ func (p Provider) IsTKG() bool { return p == ProviderTKG } +func (p Provider) IsTalos() bool { + return p == ProviderTalos +} + // ProductVariant represents the variant of the product. // // One of: Calico, TigeraSecureEnterprise diff --git a/pkg/controller/k8sapi/k8s-endpoint.go b/pkg/controller/k8sapi/k8s-endpoint.go index 5a9ddcac2c..c4a37abe40 100644 --- a/pkg/controller/k8sapi/k8s-endpoint.go +++ b/pkg/controller/k8sapi/k8s-endpoint.go @@ -27,6 +27,8 @@ import ( ) const dockerEEProxyLocal = "proxy.local" +const talosKubePrismHost = "localhost" +const talosKubePrismPort = "7445" // Endpoint is the default ServiceEndpoint learned from environment variables. var Endpoint ServiceEndpoint @@ -62,6 +64,10 @@ func (k8s ServiceEndpoint) EnvVars(hostNetworked bool, provider operator.Provide return nil } + if provider == operator.ProviderTalos && !hostNetworked && k8s.Host == talosKubePrismHost && k8s.Port == talosKubePrismPort { + return nil + } + return []v1.EnvVar{ {Name: "KUBERNETES_SERVICE_HOST", Value: k8s.Host}, {Name: "KUBERNETES_SERVICE_PORT", Value: k8s.Port}, diff --git a/pkg/controller/utils/discovery.go b/pkg/controller/utils/discovery.go index 4d022c48d4..97ef2de11e 100644 --- a/pkg/controller/utils/discovery.go +++ b/pkg/controller/utils/discovery.go @@ -114,6 +114,13 @@ func AutoDiscoverProvider(ctx context.Context, clientset kubernetes.Interface) ( return operatorv1.ProviderRKE2, nil } + // We failed to determine the platform based on API groups. Some platforms can be detected in other ways, though. + if talos, err := isTalos(ctx, clientset); err != nil { + return operatorv1.ProviderNone, fmt.Errorf("failed to check if Talos is the provider: %s", err) + } else if talos { + return operatorv1.ProviderTalos, nil + } + // Couldn't detect any specific platform. return operatorv1.ProviderNone, nil } @@ -239,6 +246,24 @@ func isRKE2(ctx context.Context, c kubernetes.Interface) (bool, error) { return foundRKE2Resource, nil } +// isTalos returns true if running on a Talos cluster, and false otherwise. +// Talos doesn't have any provider-specific API groups, so we need to use a different approach than +// we use for other platforms in autodetectFromGroup. +func isTalos(ctx context.Context, c kubernetes.Interface) (bool, error) { + masterNodes, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/control-plane"}) + if err != nil { + return false, err + } + for _, n := range masterNodes.Items { + for l := range n.Annotations { + if strings.HasPrefix(l, "talos.dev") { + return true, nil + } + } + } + return false, nil +} + // UseExternalElastic returns true if this cluster is configured to use an external elasticsearch cluster, // and false otherwise. func UseExternalElastic(config *corev1.ConfigMap) bool { diff --git a/pkg/controller/utils/discovery_test.go b/pkg/controller/utils/discovery_test.go index 384abe62e5..8220a125c8 100644 --- a/pkg/controller/utils/discovery_test.go +++ b/pkg/controller/utils/discovery_test.go @@ -95,6 +95,23 @@ var _ = Describe("provider discovery", func() { Expect(p).To(Equal(operatorv1.ProviderTKG)) }) + It("should detect Talos if a control-plane node has annotations prefixed with talos.dev", func() { + c := fake.NewSimpleClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "control-plane-1", + Labels: map[string]string{ + "node-role.kubernetes.io/control-plane": "", + }, + Annotations: map[string]string{ + "talos.dev/owned-labels": "[\"node-role.kubernetes.io/control-plane\"]", + }, + }, + }) + p, e := AutoDiscoverProvider(context.Background(), c) + Expect(e).To(BeNil()) + Expect(p).To(Equal(operatorv1.ProviderTalos)) + }) + It("should detect EKS based on eks-certificates-controller ConfigMap", func() { c := fake.NewSimpleClientset(&corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/crds/operator/operator.tigera.io_installations.yaml b/pkg/crds/operator/operator.tigera.io_installations.yaml index 68fdc8e51f..0bbb895666 100644 --- a/pkg/crds/operator/operator.tigera.io_installations.yaml +++ b/pkg/crds/operator/operator.tigera.io_installations.yaml @@ -6816,6 +6816,7 @@ spec: - DockerEnterprise - RKE2 - TKG + - Talos type: string logging: description: Logging Configuration for Components @@ -15689,6 +15690,7 @@ spec: - DockerEnterprise - RKE2 - TKG + - Talos type: string logging: description: Logging Configuration for Components diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index 3a49e2da56..5fbf62b73f 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -824,6 +824,40 @@ var _ = Describe("API server rendering tests (Calico Enterprise)", func() { rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") }) + It("should not set KUBERENETES_SERVICE_... variables if not host networked on Talos with KubePrism", func() { + cfg.K8SServiceEndpoint.Host = "localhost" + cfg.K8SServiceEndpoint.Port = "7445" + cfg.Installation.KubernetesProvider = operatorv1.ProviderTalos + + component, err := render.APIServer(cfg) + Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + deploymentResource := rtest.GetResource(resources, "tigera-apiserver", "tigera-system", "apps", "v1", "Deployment") + Expect(deploymentResource).ToNot(BeNil()) + + deployment := deploymentResource.(*appsv1.Deployment) + rtest.ExpectNoK8sServiceEpEnvVars(deployment.Spec.Template.Spec) + }) + + It("should set KUBERENETES_SERVICE_... variables if not host networked on Talos with non-KubePrism address", func() { + cfg.K8SServiceEndpoint.Host = "k8shost" + cfg.K8SServiceEndpoint.Port = "1234" + cfg.Installation.KubernetesProvider = operatorv1.ProviderTalos + + component, err := render.APIServer(cfg) + Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + deploymentResource := rtest.GetResource(resources, "tigera-apiserver", "tigera-system", "apps", "v1", "Deployment") + Expect(deploymentResource).ToNot(BeNil()) + + deployment := deploymentResource.(*appsv1.Deployment) + rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") + }) + It("should render an API server with custom configuration with MCM enabled at startup", func() { cfg.ManagementCluster = managementCluster component, err := render.APIServer(cfg) diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index bfe6d7d4f1..fddebfd8c7 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -996,6 +996,21 @@ var _ = Describe("kube-controllers rendering tests", func() { rtest.ExpectNoK8sServiceEpEnvVars(deployment.Spec.Template.Spec) }) + It("should not add the KUBERNETES_SERVICE_... variables on Talos using KubePrism", func() { + k8sServiceEp.Host = "localhost" + k8sServiceEp.Port = "7445" + instance.KubernetesProvider = operatorv1.ProviderTalos + + component := kubecontrollers.NewCalicoKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + depResource := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + Expect(depResource).ToNot(BeNil()) + deployment := depResource.(*appsv1.Deployment) + rtest.ExpectNoK8sServiceEpEnvVars(deployment.Spec.Template.Spec) + }) + It("should add prometheus annotations to metrics service", func() { for _, variant := range []operatorv1.ProductVariant{operatorv1.Calico, operatorv1.TigeraSecureEnterprise} { cfg.Installation.Variant = variant