Elemento dns

cmd/kops-controller/controllers/awsipam.go

@@ -160,8 +160,9 @@ func (r *AWSIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 type nodePatchSpec struct {
-	PodCIDR  string   `json:"podCIDR,omitempty"`
-	PodCIDRs []string `json:"podCIDRs,omitempty"`
+	PodCIDR    string   `json:"podCIDR,omitempty"`
+	PodCIDRs   []string `json:"podCIDRs,omitempty"`
+	ProviderID *string  `json:"providerID,omitempty"`
 }
 
 // patchNodePodCIDRs patches the node podCIDRs to the specified value(s).

cmd/kops-controller/controllers/node_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"reflect"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
@@ -68,7 +69,10 @@ type NodeReconciler struct {
 	identifier nodeidentity.Identifier
 }
 
+const externalCloudProviderTaint = "node.cloudprovider.kubernetes.io/uninitialized"
+
 // +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
+// +kubebuilder:rbac:groups=,resources=nodes/status,verbs=get;patch;update
 // Reconcile is the main reconciler function that observes node changes.
 func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	_ = r.log.WithValues("nodecontroller", req.NamespacedName)
@@ -111,16 +115,32 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 		}
 	}
 
-	if len(updateLabels) == 0 && len(deleteLabels) == 0 {
-		klog.V(4).Infof("no label changes needed for %s", node.Name)
-		return ctrl.Result{}, nil
+	providerID := ""
+	if info.ProviderID != "" && node.Spec.ProviderID != info.ProviderID {
+		providerID = info.ProviderID
 	}
 
-	if err := patchNodeLabels(r.coreV1Client, ctx, node, updateLabels, deleteLabels); err != nil {
-		klog.Warningf("failed to patch node labels on %s: %v", node.Name, err)
+	var taints *[]corev1.Taint
+	if info.Initialized {
+		if updatedTaints, changed := removeTaint(node.Spec.Taints, externalCloudProviderTaint); changed {
+			taints = &updatedTaints
+		}
+	}
+
+	if len(updateLabels) == 0 && len(deleteLabels) == 0 && providerID == "" && taints == nil {
+		klog.V(4).Infof("no spec or label changes needed for %s", node.Name)
+	} else if err := patchNode(r.coreV1Client, ctx, node, updateLabels, deleteLabels, providerID, taints); err != nil {
+		klog.Warningf("failed to patch node on %s: %v", node.Name, err)
 		return ctrl.Result{}, err
 	}
 
+	if len(info.Addresses) != 0 && !reflect.DeepEqual(node.Status.Addresses, info.Addresses) {
+		if err := patchNodeStatusAddresses(r.coreV1Client, ctx, node, info.Addresses); err != nil {
+			klog.Warningf("failed to patch node status addresses on %s: %v", node.Name, err)
+			return ctrl.Result{}, err
+		}
+	}
+
 	return ctrl.Result{}, nil
 }
 
@@ -142,6 +162,11 @@ type nodePatchMetadata struct {
 
 // patchNodeLabels patches the node labels to set the specified labels
 func patchNodeLabels(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, setLabels map[string]string, deleteLabels map[string]struct{}) error {
+	return patchNode(client, ctx, node, setLabels, deleteLabels, "", nil)
+}
+
+// patchNode patches node metadata and spec fields managed by the node controller.
+func patchNode(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, setLabels map[string]string, deleteLabels map[string]struct{}, providerID string, taints *[]corev1.Taint) error {
 	nodePatchMetadata := &nodePatchMetadata{
 		Labels: make(map[string]*string),
 	}
@@ -153,20 +178,90 @@ func patchNodeLabels(client *corev1client.CoreV1Client, ctx context.Context, nod
 		nodePatchMetadata.Labels[k] = nil
 	}
 
-	nodePatch := &nodePatch{
-		Metadata: nodePatchMetadata,
+	nodePatch := &nodePatch{}
+	if len(nodePatchMetadata.Labels) != 0 {
+		nodePatch.Metadata = nodePatchMetadata
+	}
+	if providerID != "" {
+		nodePatch.Spec = &nodePatchSpec{ProviderID: &providerID}
+	}
+
+	if nodePatch.Metadata != nil || nodePatch.Spec != nil {
+		nodePatchJson, err := json.Marshal(nodePatch)
+		if err != nil {
+			return fmt.Errorf("error building node patch: %v", err)
+		}
+
+		klog.V(2).Infof("sending patch for node %q: %q", node.Name, string(nodePatchJson))
+
+		_, err = client.Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, nodePatchJson, metav1.PatchOptions{})
+		if err != nil {
+			return fmt.Errorf("error applying patch to node: %v", err)
+		}
+	}
+
+	if taints != nil {
+		if err := patchNodeTaints(client, ctx, node, *taints); err != nil {
+			return err
+		}
 	}
-	nodePatchJson, err := json.Marshal(nodePatch)
+
+	return nil
+}
+
+func patchNodeTaints(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, taints []corev1.Taint) error {
+	nodePatchJson, err := json.Marshal(struct {
+		Spec struct {
+			Taints []corev1.Taint `json:"taints"`
+		} `json:"spec"`
+	}{Spec: struct {
+		Taints []corev1.Taint `json:"taints"`
+	}{Taints: taints}})
+	if err != nil {
+		return fmt.Errorf("error building node taints patch: %v", err)
+	}
+
+	klog.V(2).Infof("sending taints patch for node %q: %q", node.Name, string(nodePatchJson))
+
+	_, err = client.Nodes().Patch(ctx, node.Name, types.MergePatchType, nodePatchJson, metav1.PatchOptions{})
+	if err != nil {
+		return fmt.Errorf("error applying taints patch to node: %v", err)
+	}
+
+	return nil
+}
+
+func patchNodeStatusAddresses(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, addresses []corev1.NodeAddress) error {
+	nodePatchJson, err := json.Marshal(struct {
+		Status struct {
+			Addresses []corev1.NodeAddress `json:"addresses"`
+		} `json:"status"`
+	}{Status: struct {
+		Addresses []corev1.NodeAddress `json:"addresses"`
+	}{Addresses: addresses}})
 	if err != nil {
-		return fmt.Errorf("error building node patch: %v", err)
+		return fmt.Errorf("error building node status patch: %v", err)
 	}
 
-	klog.V(2).Infof("sending patch for node %q: %q", node.Name, string(nodePatchJson))
+	klog.V(2).Infof("sending status patch for node %q: %q", node.Name, string(nodePatchJson))
 
-	_, err = client.Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, nodePatchJson, metav1.PatchOptions{})
+	_, err = client.Nodes().Patch(ctx, node.Name, types.MergePatchType, nodePatchJson, metav1.PatchOptions{}, "status")
 	if err != nil {
-		return fmt.Errorf("error applying patch to node: %v", err)
+		return fmt.Errorf("error applying status patch to node: %v", err)
 	}
 
 	return nil
 }
+
+func removeTaint(taints []corev1.Taint, key string) ([]corev1.Taint, bool) {
+	updated := make([]corev1.Taint, 0, len(taints))
+	changed := false
+	for _, taint := range taints {
+		if taint.Key == key {
+			changed = true
+			continue
+		}
+		updated = append(updated, taint)
+	}
+	return updated, changed
+}

cmd/kops-controller/main.go

@@ -39,20 +39,20 @@ import (
 	nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
 	nodeidentityazure "k8s.io/kops/pkg/nodeidentity/azure"
 	nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do"
+	nodeidentityelemento "k8s.io/kops/pkg/nodeidentity/elemento"
 	nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce"
 	nodeidentityhetzner "k8s.io/kops/pkg/nodeidentity/hetzner"
 	nodeidentitymetal "k8s.io/kops/pkg/nodeidentity/metal"
 	nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
 	nodeidentityscw "k8s.io/kops/pkg/nodeidentity/scaleway"
-	nodeidentityelemento "k8s.io/kops/pkg/nodeidentity/elemento"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/azure"
 	"k8s.io/kops/upup/pkg/fi/cloudup/do"
+	"k8s.io/kops/upup/pkg/fi/cloudup/elemento"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
-	"k8s.io/kops/upup/pkg/fi/cloudup/elemento"
 	"k8s.io/kops/util/pkg/vfs"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -189,6 +189,7 @@ func main() {
 			verifiers = append(verifiers, verifier)
 		}
 		if opt.Server.Provider.Elemento != nil {
+
 			verifier, err := elemento.NewElementoVerifier(opt.Server.Provider.Elemento)
 			if err != nil {
 				setupLog.Error(err, "unable to create verifier")
@@ -317,9 +318,9 @@ func addNodeController(ctx context.Context, mgr manager.Manager, vfsContext *vfs
 		if err != nil {
 			return fmt.Errorf("error building identifier: %w", err)
 		}
-	
+
 	case "elemento":
-		identifier, err = nodeidentityelemento.New(opt.CacheNodeidentityInfo)
+		identifier, err = nodeidentityelemento.New(opt.CacheNodeidentityInfo, opt.ClusterName)
 		if err != nil {
 			return fmt.Errorf("error building identifier: %w", err)
 		}

go.mod

@@ -102,7 +102,7 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
-// replace github.com/Elemento-Modular-Cloud/ecloud-go v1.0.1 => ../ecloud-go
+replace github.com/Elemento-Modular-Cloud/ecloud-go => ../ecloud-go
 
 require (
 	cloud.google.com/go/auth v0.9.5 // indirect

nodeup/pkg/model/bootstrap_client.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/azure"
 	"k8s.io/kops/upup/pkg/fi/cloudup/do"
+	"k8s.io/kops/upup/pkg/fi/cloudup/elemento"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
@@ -94,8 +95,7 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		}
 		authenticator = a
 	case kops.CloudProviderElemento:
-		// Use PKI file-based authentication like Metal provider to avoid kops-controller dependency
-		a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem")
+		a, err := elemento.NewElementoAuthenticator()
 		if err != nil {
 			return err
 		}

pkg/apis/kops/model/features.go

@@ -31,8 +31,6 @@ func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool {
 		return true
 	case kops.CloudProviderAzure:
 		return true
-	case kops.CloudProviderElemento:
-		return true
 	default:
 		return false
 	}

pkg/bootstrap/pkibootstrap/pkiverifier.go

@@ -26,6 +26,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"net"
 	"net/http"
 	"strings"
 	"time"
@@ -128,11 +129,25 @@ func (v *verifier) VerifyToken(ctx context.Context, rawRequest *http.Request, au
 	*/
 
 	// DISABLED: Return a dummy successful verification result
+	nodeName := "nodes-europe-1"
+	certificateNames := []string{nodeName}
+	if host, _, err := net.SplitHostPort(rawRequest.RemoteAddr); err == nil {
+		certificateNames = append(certificateNames, host)
+		switch host {
+		case "192.168.100.10":
+			nodeName = "control-plane-europe-1"
+		case "192.168.100.11":
+			nodeName = "nodes-europe-1"
+		case "192.168.100.12":
+			nodeName = "nodes-europe-2"
+		}
+		certificateNames[0] = nodeName
+	}
 	result := &bootstrap.VerifyResult{
-		NodeName:          "test-node",
-		CertificateNames:  []string{"127.0.0.1"},
-		ChallengeEndpoint: "127.0.0.1:10000",
-		InstanceGroupName: "nodes",
+		NodeName:          nodeName,
+		CertificateNames:  certificateNames,
+		ChallengeEndpoint: "",
+		InstanceGroupName: "nodes-europe",
 	}
 
 	return result, nil

pkg/commands/toolbox_enroll.go

@@ -687,7 +687,22 @@ func (b *ConfigBuilder) GetWellKnownAddresses(ctx context.Context) (model.WellKn
 		}
 	}
 	if len(wellKnownAddresses[wellknownservices.KubeAPIServer]) == 0 {
-		// TODO: Should we support DNS?
+		names := []string{fullCluster.APIInternalName()}
+		if fullCluster.Spec.API.PublicName != "" {
+			names = append(names, fullCluster.Spec.API.PublicName)
+		}
+		for _, name := range names {
+			ips, err := net.LookupIP(name)
+			if err != nil {
+				klog.Warningf("unable to resolve kube-apiserver DNS name %q: %v", name, err)
+				continue
+			}
+			for _, ip := range ips {
+				wellKnownAddresses[wellknownservices.KubeAPIServer] = append(wellKnownAddresses[wellknownservices.KubeAPIServer], ip.String())
+			}
+		}
+	}
+	if len(wellKnownAddresses[wellknownservices.KubeAPIServer]) == 0 {
 		return nil, fmt.Errorf("unable to determine IP address for kube-apiserver")
 	}
 	for k := range wellKnownAddresses {