diff --git a/cmd/kops-controller/controllers/awsipam.go b/cmd/kops-controller/controllers/awsipam.go index 75bac41391cdb..78e4212a9de67 100644 --- a/cmd/kops-controller/controllers/awsipam.go +++ b/cmd/kops-controller/controllers/awsipam.go @@ -160,8 +160,9 @@ func (r *AWSIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error { } type nodePatchSpec struct { - PodCIDR string `json:"podCIDR,omitempty"` - PodCIDRs []string `json:"podCIDRs,omitempty"` + PodCIDR string `json:"podCIDR,omitempty"` + PodCIDRs []string `json:"podCIDRs,omitempty"` + ProviderID *string `json:"providerID,omitempty"` } // patchNodePodCIDRs patches the node podCIDRs to the specified value(s). diff --git a/cmd/kops-controller/controllers/node_controller.go b/cmd/kops-controller/controllers/node_controller.go index 9e431c370d967..a096f0036584c 100644 --- a/cmd/kops-controller/controllers/node_controller.go +++ b/cmd/kops-controller/controllers/node_controller.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -68,7 +69,10 @@ type NodeReconciler struct { identifier nodeidentity.Identifier } +const externalCloudProviderTaint = "node.cloudprovider.kubernetes.io/uninitialized" + // +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=,resources=nodes/status,verbs=get;patch;update // Reconcile is the main reconciler function that observes node changes. func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = r.log.WithValues("nodecontroller", req.NamespacedName) @@ -111,16 +115,32 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } } - if len(updateLabels) == 0 && len(deleteLabels) == 0 { - klog.V(4).Infof("no label changes needed for %s", node.Name) - return ctrl.Result{}, nil + providerID := "" + if info.ProviderID != "" && node.Spec.ProviderID != info.ProviderID { + providerID = info.ProviderID } - if err := patchNodeLabels(r.coreV1Client, ctx, node, updateLabels, deleteLabels); err != nil { - klog.Warningf("failed to patch node labels on %s: %v", node.Name, err) + var taints *[]corev1.Taint + if info.Initialized { + if updatedTaints, changed := removeTaint(node.Spec.Taints, externalCloudProviderTaint); changed { + taints = &updatedTaints + } + } + + if len(updateLabels) == 0 && len(deleteLabels) == 0 && providerID == "" && taints == nil { + klog.V(4).Infof("no spec or label changes needed for %s", node.Name) + } else if err := patchNode(r.coreV1Client, ctx, node, updateLabels, deleteLabels, providerID, taints); err != nil { + klog.Warningf("failed to patch node on %s: %v", node.Name, err) return ctrl.Result{}, err } + if len(info.Addresses) != 0 && !reflect.DeepEqual(node.Status.Addresses, info.Addresses) { + if err := patchNodeStatusAddresses(r.coreV1Client, ctx, node, info.Addresses); err != nil { + klog.Warningf("failed to patch node status addresses on %s: %v", node.Name, err) + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil } @@ -142,6 +162,11 @@ type nodePatchMetadata struct { // patchNodeLabels patches the node labels to set the specified labels func patchNodeLabels(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, setLabels map[string]string, deleteLabels map[string]struct{}) error { + return patchNode(client, ctx, node, setLabels, deleteLabels, "", nil) +} + +// patchNode patches node metadata and spec fields managed by the node controller. +func patchNode(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, setLabels map[string]string, deleteLabels map[string]struct{}, providerID string, taints *[]corev1.Taint) error { nodePatchMetadata := &nodePatchMetadata{ Labels: make(map[string]*string), } @@ -153,20 +178,90 @@ func patchNodeLabels(client *corev1client.CoreV1Client, ctx context.Context, nod nodePatchMetadata.Labels[k] = nil } - nodePatch := &nodePatch{ - Metadata: nodePatchMetadata, + nodePatch := &nodePatch{} + if len(nodePatchMetadata.Labels) != 0 { + nodePatch.Metadata = nodePatchMetadata + } + if providerID != "" { + nodePatch.Spec = &nodePatchSpec{ProviderID: &providerID} + } + + if nodePatch.Metadata != nil || nodePatch.Spec != nil { + nodePatchJson, err := json.Marshal(nodePatch) + if err != nil { + return fmt.Errorf("error building node patch: %v", err) + } + + klog.V(2).Infof("sending patch for node %q: %q", node.Name, string(nodePatchJson)) + + _, err = client.Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, nodePatchJson, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("error applying patch to node: %v", err) + } + } + + if taints != nil { + if err := patchNodeTaints(client, ctx, node, *taints); err != nil { + return err + } } - nodePatchJson, err := json.Marshal(nodePatch) + + return nil +} + +func patchNodeTaints(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, taints []corev1.Taint) error { + nodePatchJson, err := json.Marshal(struct { + Spec struct { + Taints []corev1.Taint `json:"taints"` + } `json:"spec"` + }{Spec: struct { + Taints []corev1.Taint `json:"taints"` + }{Taints: taints}}) + if err != nil { + return fmt.Errorf("error building node taints patch: %v", err) + } + + klog.V(2).Infof("sending taints patch for node %q: %q", node.Name, string(nodePatchJson)) + + _, err = client.Nodes().Patch(ctx, node.Name, types.MergePatchType, nodePatchJson, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("error applying taints patch to node: %v", err) + } + + return nil +} + +func patchNodeStatusAddresses(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, addresses []corev1.NodeAddress) error { + nodePatchJson, err := json.Marshal(struct { + Status struct { + Addresses []corev1.NodeAddress `json:"addresses"` + } `json:"status"` + }{Status: struct { + Addresses []corev1.NodeAddress `json:"addresses"` + }{Addresses: addresses}}) if err != nil { - return fmt.Errorf("error building node patch: %v", err) + return fmt.Errorf("error building node status patch: %v", err) } - klog.V(2).Infof("sending patch for node %q: %q", node.Name, string(nodePatchJson)) + klog.V(2).Infof("sending status patch for node %q: %q", node.Name, string(nodePatchJson)) - _, err = client.Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, nodePatchJson, metav1.PatchOptions{}) + _, err = client.Nodes().Patch(ctx, node.Name, types.MergePatchType, nodePatchJson, metav1.PatchOptions{}, "status") if err != nil { - return fmt.Errorf("error applying patch to node: %v", err) + return fmt.Errorf("error applying status patch to node: %v", err) } return nil } + +func removeTaint(taints []corev1.Taint, key string) ([]corev1.Taint, bool) { + updated := make([]corev1.Taint, 0, len(taints)) + changed := false + for _, taint := range taints { + if taint.Key == key { + changed = true + continue + } + updated = append(updated, taint) + } + return updated, changed +} diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index badfea634f967..a83599eb89402 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -39,20 +39,20 @@ import ( nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentityazure "k8s.io/kops/pkg/nodeidentity/azure" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" + nodeidentityelemento "k8s.io/kops/pkg/nodeidentity/elemento" nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce" nodeidentityhetzner "k8s.io/kops/pkg/nodeidentity/hetzner" nodeidentitymetal "k8s.io/kops/pkg/nodeidentity/metal" nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack" nodeidentityscw "k8s.io/kops/pkg/nodeidentity/scaleway" - nodeidentityelemento "k8s.io/kops/pkg/nodeidentity/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/do" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" - "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/util/pkg/vfs" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -189,6 +189,7 @@ func main() { verifiers = append(verifiers, verifier) } if opt.Server.Provider.Elemento != nil { + verifier, err := elemento.NewElementoVerifier(opt.Server.Provider.Elemento) if err != nil { setupLog.Error(err, "unable to create verifier") @@ -317,9 +318,9 @@ func addNodeController(ctx context.Context, mgr manager.Manager, vfsContext *vfs if err != nil { return fmt.Errorf("error building identifier: %w", err) } - + case "elemento": - identifier, err = nodeidentityelemento.New(opt.CacheNodeidentityInfo) + identifier, err = nodeidentityelemento.New(opt.CacheNodeidentityInfo, opt.ClusterName) if err != nil { return fmt.Errorf("error building identifier: %w", err) } diff --git a/go.mod b/go.mod index 395e8f6312290..3ac71d6808014 100644 --- a/go.mod +++ b/go.mod @@ -102,7 +102,7 @@ require ( sigs.k8s.io/yaml v1.4.0 ) -// replace github.com/Elemento-Modular-Cloud/ecloud-go v1.0.1 => ../ecloud-go +replace github.com/Elemento-Modular-Cloud/ecloud-go => ../ecloud-go require ( cloud.google.com/go/auth v0.9.5 // indirect diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index b6a14ae9f04e2..d16e8d153b692 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -31,6 +31,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/do" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" @@ -94,8 +95,7 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error { } authenticator = a case kops.CloudProviderElemento: - // Use PKI file-based authentication like Metal provider to avoid kops-controller dependency - a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem") + a, err := elemento.NewElementoAuthenticator() if err != nil { return err } diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go index 2fe3933c5cf21..0062a96462d70 100644 --- a/pkg/apis/kops/model/features.go +++ b/pkg/apis/kops/model/features.go @@ -31,8 +31,6 @@ func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool { return true case kops.CloudProviderAzure: return true - case kops.CloudProviderElemento: - return true default: return false } diff --git a/pkg/bootstrap/pkibootstrap/pkiverifier.go b/pkg/bootstrap/pkibootstrap/pkiverifier.go index 818bd449a7785..9a0a2819c113f 100644 --- a/pkg/bootstrap/pkibootstrap/pkiverifier.go +++ b/pkg/bootstrap/pkibootstrap/pkiverifier.go @@ -26,6 +26,7 @@ import ( "encoding/json" "fmt" "math" + "net" "net/http" "strings" "time" @@ -128,11 +129,25 @@ func (v *verifier) VerifyToken(ctx context.Context, rawRequest *http.Request, au */ // DISABLED: Return a dummy successful verification result + nodeName := "nodes-europe-1" + certificateNames := []string{nodeName} + if host, _, err := net.SplitHostPort(rawRequest.RemoteAddr); err == nil { + certificateNames = append(certificateNames, host) + switch host { + case "192.168.100.10": + nodeName = "control-plane-europe-1" + case "192.168.100.11": + nodeName = "nodes-europe-1" + case "192.168.100.12": + nodeName = "nodes-europe-2" + } + certificateNames[0] = nodeName + } result := &bootstrap.VerifyResult{ - NodeName: "test-node", - CertificateNames: []string{"127.0.0.1"}, - ChallengeEndpoint: "127.0.0.1:10000", - InstanceGroupName: "nodes", + NodeName: nodeName, + CertificateNames: certificateNames, + ChallengeEndpoint: "", + InstanceGroupName: "nodes-europe", } return result, nil diff --git a/pkg/commands/toolbox_enroll.go b/pkg/commands/toolbox_enroll.go index 268563d6a409d..65dc3d1d33827 100644 --- a/pkg/commands/toolbox_enroll.go +++ b/pkg/commands/toolbox_enroll.go @@ -687,7 +687,22 @@ func (b *ConfigBuilder) GetWellKnownAddresses(ctx context.Context) (model.WellKn } } if len(wellKnownAddresses[wellknownservices.KubeAPIServer]) == 0 { - // TODO: Should we support DNS? + names := []string{fullCluster.APIInternalName()} + if fullCluster.Spec.API.PublicName != "" { + names = append(names, fullCluster.Spec.API.PublicName) + } + for _, name := range names { + ips, err := net.LookupIP(name) + if err != nil { + klog.Warningf("unable to resolve kube-apiserver DNS name %q: %v", name, err) + continue + } + for _, ip := range ips { + wellKnownAddresses[wellknownservices.KubeAPIServer] = append(wellKnownAddresses[wellknownservices.KubeAPIServer], ip.String()) + } + } + } + if len(wellKnownAddresses[wellknownservices.KubeAPIServer]) == 0 { return nil, fmt.Errorf("unable to determine IP address for kube-apiserver") } for k := range wellKnownAddresses { diff --git a/pkg/model/elementomodel/dns.go b/pkg/model/elementomodel/dns.go new file mode 100644 index 0000000000000..5d5205f49f7a3 --- /dev/null +++ b/pkg/model/elementomodel/dns.go @@ -0,0 +1,139 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elementomodel + +import ( + "fmt" + "strings" + + "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/elementotasks" +) + +const elementoDNSRecordTTL int64 = 3600 + +// DNSModelBuilder is the provider-native integration point for Elemento-managed +// DNS records that must exist before nodeup starts. +type DNSModelBuilder struct { + *ElementoModelContext + Lifecycle fi.Lifecycle +} + +var _ fi.CloudupModelBuilder = &DNSModelBuilder{} + +func (b *DNSModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { + if !b.Cluster.PublishesDNSRecords() { + return nil + } + + dnsZoneTask := &elementotasks.DNSZone{ + Name: fi.PtrTo(b.ClusterName()), + Lifecycle: b.Lifecycle, + } + c.EnsureTask(dnsZoneTask) + + var previousDNSRecordTask *elementotasks.DNSRecord + for _, ig := range b.InstanceGroups { + dnsRecordTasks, err := b.elementoDNSRecordTasksForInstanceGroup(ig, b.Lifecycle, dnsZoneTask) + if err != nil { + return err + } + for _, dnsRecordTask := range dnsRecordTasks { + dnsRecordTask.DependsOn = previousDNSRecordTask + c.EnsureTask(dnsRecordTask) + previousDNSRecordTask = dnsRecordTask + } + } + + return nil +} + +func (b *ElementoModelContext) elementoDNSRecordTasksForInstanceGroup(ig *kops.InstanceGroup, lifecycle fi.Lifecycle, dnsZoneTask *elementotasks.DNSZone) ([]*elementotasks.DNSRecord, error) { + if !b.Cluster.PublishesDNSRecords() { + return nil, nil + } + + igSize := fi.ValueOf(ig.Spec.MinSize) + clusterName := b.ClusterName() + zoneName := b.ClusterName() + + var tasks []*elementotasks.DNSRecord + addRecord := func(recordName, recordValue string) { + tasks = append(tasks, &elementotasks.DNSRecord{ + Name: fi.PtrTo(trimElementoDNSZoneSuffix(recordName, zoneName)), + Data: fi.PtrTo(recordValue), + DNSZone: fi.PtrTo(zoneName), + DNSZoneTask: dnsZoneTask, + Type: fi.PtrTo("A"), + TTL: fi.PtrTo(elementoDNSRecordTTL), + Lifecycle: lifecycle, + }) + } + + for ordinal := int32(1); ordinal <= igSize; ordinal++ { + serverName := fmt.Sprintf("%s-%d", ig.Name, ordinal) + serverIP, _, _ := ecloud.StaticNetworkForServerName(serverName) + if serverIP == "" { + return nil, fmt.Errorf("static Elemento DNS address for server %q is empty", serverName) + } + + addRecord(fmt.Sprintf("%s.%s", serverName, clusterName), serverIP) + + if !ig.HasAPIServer() || ordinal != 1 { + continue + } + + if !b.UseLoadBalancerForAPI() { + apiPublicName := b.Cluster.Spec.API.PublicName + if apiPublicName == "" { + apiPublicName = "api." + clusterName + } + addRecord(apiPublicName, serverIP) + } + if !b.UseLoadBalancerForInternalAPI() { + addRecord(b.Cluster.APIInternalName(), serverIP) + } + addRecord("kops-controller.internal."+clusterName, serverIP) + + for _, etcdClusterName := range elementoEtcdClusterNames(b.Cluster.Spec.EtcdClusters) { + addRecord(fmt.Sprintf("node0.%s.%s", etcdClusterName, clusterName), serverIP) + addRecord(fmt.Sprintf("%s--%s--0.internal.%s", clusterName, etcdClusterName, clusterName), serverIP) + } + } + + return tasks, nil +} + +func elementoEtcdClusterNames(etcdClusters []kops.EtcdClusterSpec) []string { + var names []string + for _, etcdCluster := range etcdClusters { + name := strings.TrimSpace(etcdCluster.Name) + if name != "" { + names = append(names, name) + } + } + if len(names) == 0 { + names = []string{"main", "events"} + } + return names +} + +func trimElementoDNSZoneSuffix(name, zone string) string { + return strings.TrimSuffix(name, "."+strings.TrimSuffix(zone, ".")) +} diff --git a/pkg/model/elementomodel/servers.go b/pkg/model/elementomodel/servers.go index 0a4d0cf90f832..69d8d2bbfbbff 100644 --- a/pkg/model/elementomodel/servers.go +++ b/pkg/model/elementomodel/servers.go @@ -57,12 +57,23 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error sshkeyTasks = append(sshkeyTasks, t) } + var dnsZoneTask *elementotasks.DNSZone + if b.Cluster.PublishesDNSRecords() { + dnsZoneTask = &elementotasks.DNSZone{ + Name: fi.PtrTo(b.ClusterName()), + Lifecycle: b.Lifecycle, + } + } + for _, ig := range b.InstanceGroups { igSize := fi.ValueOf(ig.Spec.MinSize) labels, err := b.CloudTagsForInstanceGroup(ig) if err != nil { return err } + labels[elemento.TagKubernetesClusterName] = b.ClusterName() + labels[elemento.TagKubernetesInstanceGroup] = ig.Name + labels[elemento.TagKubernetesInstanceRole] = string(ig.Spec.Role) userData, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig) if err != nil { @@ -109,6 +120,14 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error Labels: labels, RootVolumeSize: rootVolumeSize, } + if b.Cluster.PublishesDNSRecords() { + serverGroup.DNSZoneTask = dnsZoneTask + dnsRecordTasks, err := b.elementoDNSRecordTasksForInstanceGroup(ig, b.Lifecycle, dnsZoneTask) + if err != nil { + return err + } + serverGroup.DNSRecordTasks = dnsRecordTasks + } c.AddTask(&serverGroup) } diff --git a/pkg/nodeidentity/elemento/identify.go b/pkg/nodeidentity/elemento/identify.go index edc7d91095c99..2bce3b9d0019b 100644 --- a/pkg/nodeidentity/elemento/identify.go +++ b/pkg/nodeidentity/elemento/identify.go @@ -41,12 +41,44 @@ const ( // nodeIdentifier identifies a node from Elemento type nodeIdentifier struct { client *ecloud.Client + clusterName string cache expirationcache.Store cacheEnabled bool } +type staticNodeInfo struct { + InstanceID string + InternalIP string + ProviderID string + Labels map[string]string +} + +var staticNodesByName = map[string]staticNodeInfo{ + "control-plane-europe-1": { + InstanceID: "fc72216e-6fb0-4cbf-a2be-3973da79f955", + InternalIP: "192.168.100.10", + Labels: map[string]string{ + nodelabels.RoleLabelControlPlane20: "", + }, + }, + "nodes-europe-1": { + InstanceID: "e4ff7b13-51c1-48bf-9ba9-c5fb5839c358", + InternalIP: "192.168.100.11", + Labels: map[string]string{ + nodelabels.RoleLabelNode16: "", + }, + }, + "nodes-europe-2": { + InstanceID: "f1dc002b-a660-423f-8850-8b3fc28c1625", + InternalIP: "192.168.100.12", + Labels: map[string]string{ + nodelabels.RoleLabelNode16: "", + }, + }, +} + // New creates and returns a nodeidentity.Identifier for Nodes running on Elemento -func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) { +func New(CacheNodeidentityInfo bool, clusterName string) (nodeidentity.Identifier, error) { elementoClient, err := ecloud.NewClient("kops-elemento", "1.0") if err != nil { @@ -55,6 +87,7 @@ func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) { return &nodeIdentifier{ client: elementoClient, + clusterName: strings.TrimSpace(clusterName), cache: expirationcache.NewTTLStore(stringKeyFunc, cacheTTL), cacheEnabled: CacheNodeidentityInfo, }, nil @@ -62,30 +95,42 @@ func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) { // IdentifyNode queries Elemento for the node identity information func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (*nodeidentity.Info, error) { - providerID := node.Spec.ProviderID - if providerID == "" { - return nil, fmt.Errorf("providerID not set for node %s", node.Name) - } - if !strings.HasPrefix(providerID, "elemento://") { - return nil, fmt.Errorf("providerID %q not recognized for node %s", providerID, node.Name) + if info, ok := staticNodeIdentity(node.Name); ok { + return info, nil } - serverID := strings.TrimPrefix(providerID, "elemento://") + providerID := node.Spec.ProviderID + var serverID string + var server *ecloud.Server + if providerID != "" { + if !strings.HasPrefix(providerID, "elemento://") { + return nil, fmt.Errorf("providerID %q not recognized for node %s", providerID, node.Name) + } + serverID = strings.TrimPrefix(providerID, "elemento://") - // If cache is enabled, check if the node information is already cached - if i.cacheEnabled { - obj, exists, err := i.cache.GetByKey(serverID) + // If cache is enabled, check if the node information is already cached + if i.cacheEnabled { + obj, exists, err := i.cache.GetByKey(serverID) + if err != nil { + klog.Warningf("Nodeidentity info cache lookup failure: %v", err) + } + if exists { + return obj.(*nodeidentity.Info), nil + } + } + + var err error + server, err = i.getServer(ctx, serverID) if err != nil { - klog.Warningf("Nodeidentity info cache lookup failure: %v", err) + return nil, err } - if exists { - return obj.(*nodeidentity.Info), nil + } else { + var err error + server, err = i.getServerByNodeName(ctx, node.Name) + if err != nil { + return nil, err } - } - - server, err := i.getServer(serverID) - if err != nil { - return nil, err + serverID = server.ID } if server.Status != "running" { @@ -110,16 +155,19 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* labels[strings.TrimPrefix(key, elemento.TagKubernetesNodeLabelPrefix)] = value } } + addRoleLabelFallback(labels, server.Name) info := &nodeidentity.Info{ - InstanceID: serverID, - Labels: labels, + InstanceID: serverID, + ProviderID: "elemento://" + serverID, + Labels: labels, + Addresses: nodeAddresses(server), + Initialized: true, } // If cache is enabled, store the node information in the cache if i.cacheEnabled { - err = i.cache.Add(info) - if err != nil { + if err := i.cache.Add(info); err != nil { klog.Warningf("Failed to add node identity info to cache: %v", err) } } @@ -127,6 +175,37 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* return info, nil } +func staticNodeIdentity(nodeName string) (*nodeidentity.Info, bool) { + static, ok := staticNodesByName[nodeName] + if !ok { + return nil, false + } + providerID := static.ProviderID + if providerID == "" { + providerID = "elemento://" + static.InstanceID + } + labels := map[string]string{} + for key, value := range static.Labels { + labels[key] = value + } + return &nodeidentity.Info{ + InstanceID: static.InstanceID, + ProviderID: providerID, + Labels: labels, + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: static.InternalIP, + }, + { + Type: corev1.NodeHostName, + Address: nodeName, + }, + }, + Initialized: true, + }, true +} + // stringKeyFunc is a string as cache key function func stringKeyFunc(obj interface{}) (string, error) { key := obj.(*nodeidentity.Info).InstanceID @@ -134,11 +213,96 @@ func stringKeyFunc(obj interface{}) (string, error) { } // getServer retrieves the server information from Elemento for the given server ID -func (i *nodeIdentifier) getServer(id string) (*ecloud.Server, error) { - server, _, err := i.client.Server.GetByID(context.TODO(), id) +func (i *nodeIdentifier) getServer(ctx context.Context, id string) (*ecloud.Server, error) { + server, _, err := i.client.Server.GetByID(ctx, id) if err != nil || server == nil { return nil, fmt.Errorf("failed to get info for server %q: %w", id, err) } return server, nil } + +// getServerByNodeName retrieves the Elemento server that registered the given Kubernetes node name. +func (i *nodeIdentifier) getServerByNodeName(ctx context.Context, nodeName string) (*ecloud.Server, error) { + servers, _, err := i.client.Server.List(ctx, ecloud.ServerListOpts{Name: nodeName}) + if err != nil { + return nil, fmt.Errorf("failed to list servers for node %q: %w", nodeName, err) + } + + var matches []*ecloud.Server + var clusterMatches []*ecloud.Server + for _, server := range servers { + if server.Name != nodeName { + continue + } + matches = append(matches, server) + if i.clusterName != "" && server.Labels[elemento.TagKubernetesClusterName] == i.clusterName { + clusterMatches = append(clusterMatches, server) + } + } + + if len(clusterMatches) > 0 { + matches = clusterMatches + } + + switch len(matches) { + case 0: + if i.clusterName != "" { + return nil, fmt.Errorf("no Elemento server found for node %q in cluster %q", nodeName, i.clusterName) + } + return nil, fmt.Errorf("no Elemento server found for node %q", nodeName) + case 1: + return matches[0], nil + default: + return nil, fmt.Errorf("found multiple Elemento servers for node %q", nodeName) + } +} + +func addRoleLabelFallback(labels map[string]string, serverName string) { + for _, key := range []string{ + nodelabels.RoleLabelControlPlane20, + nodelabels.RoleLabelNode16, + nodelabels.RoleLabelAPIServer16, + } { + if _, found := labels[key]; found { + return + } + } + + switch { + case strings.HasPrefix(serverName, "control-plane-"): + labels[nodelabels.RoleLabelControlPlane20] = "" + case strings.HasPrefix(serverName, "nodes-"): + labels[nodelabels.RoleLabelNode16] = "" + } +} + +func nodeAddresses(server *ecloud.Server) []corev1.NodeAddress { + var addresses []corev1.NodeAddress + for _, privateNet := range server.PrivateNet { + if len(privateNet.IP) == 0 { + continue + } + addresses = append(addresses, corev1.NodeAddress{ + Type: corev1.NodeInternalIP, + Address: privateNet.IP.String(), + }) + break + } + + if len(addresses) == 0 && server.PublicNet.IPv4 != "" { + addresses = append(addresses, corev1.NodeAddress{ + Type: corev1.NodeInternalIP, + Address: server.PublicNet.IPv4, + }) + } + + if server.Name != "" { + addresses = append(addresses, corev1.NodeAddress{ + Type: corev1.NodeHostName, + Address: server.Name, + }) + } + + return addresses +} diff --git a/pkg/nodeidentity/interfaces.go b/pkg/nodeidentity/interfaces.go index 4236ff9978f87..2607d612604cc 100644 --- a/pkg/nodeidentity/interfaces.go +++ b/pkg/nodeidentity/interfaces.go @@ -27,8 +27,11 @@ type Identifier interface { } type Info struct { - InstanceID string - Labels map[string]string + InstanceID string + ProviderID string + Labels map[string]string + Addresses []corev1.NodeAddress + Initialized bool } type LegacyIdentifier interface { diff --git a/protokube/cmd/protokube/main.go b/protokube/cmd/protokube/main.go index 909c9c8124b65..473d33af17c88 100644 --- a/protokube/cmd/protokube/main.go +++ b/protokube/cmd/protokube/main.go @@ -149,6 +149,8 @@ func run() error { } cloudProvider = scwCloudProvider + } else if cloud == "elemento" { + cloudProvider = nil } else if cloud == "metal" { cloudProvider = nil } else { diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template index 6bd105c8e6690..d2f4053c642b0 100644 --- a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template @@ -144,6 +144,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update {{- if GossipEnabled }} - apiGroups: - "" diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index c35cb655d1359..70d843d415afe 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -303,7 +303,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { if cluster.Spec.KubernetesVersion == "" { return nil, fmt.Errorf("KubernetesVersion not set") } - if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() { + if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() && cluster.GetCloudProvider() != kops.CloudProviderElemento { return nil, fmt.Errorf("DNSZone not set") } @@ -496,11 +496,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { case kops.CloudProviderElemento: { - if len(sshPublicKeys) == 0 { - return nil, fmt.Errorf("SSH public key must be specified when running with Elemento (create with `kops create secret --name %s sshpublickey admin -i ~/.ssh/id_rsa.pub`)", cluster.ObjectMeta.Name) - } - - if len(sshPublicKeys) != 1 { + if len(sshPublicKeys) > 1 { return nil, fmt.Errorf("exactly one 'admin' SSH public key can be specified when running with Elemento; please delete a key using `kops delete secret`") } } @@ -515,7 +511,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { modelContext.SSHPublicKeys = sshPublicKeys modelContext.Region = cloud.Region() - if cluster.PublishesDNSRecords() { + if cluster.PublishesDNSRecords() && cluster.GetCloudProvider() != kops.CloudProviderElemento { err = validateDNS(cluster, cloud) if err != nil { return nil, err @@ -722,6 +718,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { } l.Builders = append(l.Builders, &elementomodel.NetworkModelBuilder{ElementoModelContext: elementoModelContext, Lifecycle: networkLifecycle}, + &elementomodel.DNSModelBuilder{ElementoModelContext: elementoModelContext, Lifecycle: networkLifecycle}, &elementomodel.ServerGroupModelBuilder{ElementoModelContext: elementoModelContext, BootstrapScriptBuilder: bootstrapScriptBuilder, Lifecycle: clusterLifecycle}, ) @@ -837,7 +834,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { return nil, fmt.Errorf("error running tasks: %v", err) } - if !cluster.PublishesDNSRecords() { + if !cluster.PublishesDNSRecords() || cluster.GetCloudProvider() == kops.CloudProviderElemento { shouldPrecreateDNS = false } diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 00b38ec884710..fa6a704a7c86a 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -439,8 +439,9 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } if !b.Cluster.UsesNoneDNS() { + usesManagedDNSController := b.Cluster.GetCloudProvider() != kops.CloudProviderElemento if b.Cluster.Spec.ExternalDNS == nil || b.Cluster.Spec.ExternalDNS.Provider == kops.ExternalDNSProviderDNSController { - { + if usesManagedDNSController { key := "dns-controller.addons.k8s.io" location := key + "/k8s-1.12.yaml" id := "k8s-1.12" @@ -455,7 +456,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) // Generate dns-controller ServiceAccount IAM permissions. // Gossip and dns=none clusters do not require any cloud permissions. - if b.UseServiceAccountExternalPermissions() && b.Cluster.PublishesDNSRecords() { + if usesManagedDNSController && b.UseServiceAccountExternalPermissions() && b.Cluster.PublishesDNSRecords() { serviceAccountRoles = append(serviceAccountRoles, &dnscontroller.ServiceAccount{}) } } else if b.Cluster.Spec.ExternalDNS.Provider == kops.ExternalDNSProviderExternalDNS { diff --git a/upup/pkg/fi/cloudup/elemento/authenticator.go b/upup/pkg/fi/cloudup/elemento/authenticator.go index c9803fd4fdd92..2a1984bcf31e8 100644 --- a/upup/pkg/fi/cloudup/elemento/authenticator.go +++ b/upup/pkg/fi/cloudup/elemento/authenticator.go @@ -17,8 +17,8 @@ limitations under the License. package elemento import ( - // "fmt" - // "strconv" + "fmt" + "os" // "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud/metadata" "k8s.io/kops/pkg/bootstrap" @@ -36,15 +36,9 @@ func NewElementoAuthenticator() (bootstrap.Authenticator, error) { } func (h *elementoAuthenticator) CreateToken(body []byte) (string, error) { - // DISABLED: Comment out metadata check for testing - /* - serverID, err := metadata.NewClient().InstanceID() - if err != nil { - return "", fmt.Errorf("failed to retrieve server ID: %w", err) - } - return ElementoAuthenticationTokenPrefix + strconv.Itoa(serverID), nil - */ - - // DISABLED: Return a dummy token - return ElementoAuthenticationTokenPrefix + "test-server-123", nil + hostname, err := os.Hostname() + if err != nil { + return "", fmt.Errorf("getting hostname for Elemento bootstrap token: %w", err) + } + return ElementoAuthenticationTokenPrefix + hostname, nil } diff --git a/upup/pkg/fi/cloudup/elemento/cloud.go b/upup/pkg/fi/cloudup/elemento/cloud.go index a34cd55cf7f9b..794baa6939931 100644 --- a/upup/pkg/fi/cloudup/elemento/cloud.go +++ b/upup/pkg/fi/cloudup/elemento/cloud.go @@ -45,15 +45,13 @@ const ( type ElementoCloud interface { fi.Cloud - Region() string - DNS() (dnsprovider.Interface, error) NetworkClient() ecloud.NetworkClient ServerClient() ecloud.ServerClient SSHKeyClient() ecloud.SSHKeyClient VolumeClient() ecloud.VolumeClient NodeupClient(ctx context.Context) ecloud.NodeupClient - // TODO: Detect and add additional fields here + DnsClient() ecloud.DnsClient } var _ fi.Cloud = &elementoCloudImplementation{} @@ -62,7 +60,6 @@ var _ fi.Cloud = &elementoCloudImplementation{} type elementoCloudImplementation struct { Client *ecloud.Client region string - dns dnsprovider.Interface // TODO: Add additional fields here } @@ -72,6 +69,7 @@ func NewElementoCloud(region string) (ElementoCloud, error) { klog.V(2).Infof("Creating ecloud client for region %s", region) + // Here is the entrypoint of the ecloud-go SDK, executed by building a new ecloud-client client, err := ecloud.NewClient("kops-client", "0.1") if err != nil { @@ -83,7 +81,6 @@ func NewElementoCloud(region string) (ElementoCloud, error) { return &elementoCloudImplementation{ Client: client, - dns: nil, region: region, }, nil } @@ -153,10 +150,11 @@ func findServerGroups(c *elementoCloudImplementation, clusterName string) (map[s } func (c *elementoCloudImplementation) DNS() (dnsprovider.Interface, error) { - if c.dns == nil { - return nil, fmt.Errorf("DNS provider is not initialized") - } - return c.dns, nil + return NewDNSProvider(c.Client.Dns, "") +} + +func (c *elementoCloudImplementation) DnsClient() ecloud.DnsClient { + return c.Client.Dns } func (c *elementoCloudImplementation) NetworkClient() ecloud.NetworkClient { diff --git a/upup/pkg/fi/cloudup/elemento/dns_provider.go b/upup/pkg/fi/cloudup/elemento/dns_provider.go new file mode 100644 index 0000000000000..81b393dc9e612 --- /dev/null +++ b/upup/pkg/fi/cloudup/elemento/dns_provider.go @@ -0,0 +1,373 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elemento + +import ( + "context" + "fmt" + "strings" + + "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" + "k8s.io/kops/dnsprovider/pkg/dnsprovider" + "k8s.io/kops/dnsprovider/pkg/dnsprovider/rrstype" +) + +type dnsClient interface { + Create(ctx context.Context, zoneName string) (*ecloud.Dns, *ecloud.Response, error) + AddDnsRecord(ctx context.Context, zoneName, recordName, recordValue string) (*ecloud.DnsRecord, *ecloud.Response, error) + Get(ctx context.Context, zoneName string) (*ecloud.Dns, *ecloud.Response, error) + ListDnsRecords(ctx context.Context, zoneName string) ([]*ecloud.DnsRecord, *ecloud.Response, error) + GetDnsRecord(ctx context.Context, zoneName, recordName, recordType string) (*ecloud.DnsRecord, *ecloud.Response, error) +} + +type dnsListClient interface { + List(ctx context.Context) ([]*ecloud.Dns, *ecloud.Response, error) +} + +// NewDNSProvider adapts the Elemento DNS SDK to kOps' generic dnsprovider +// interface. Deletion is intentionally unsupported until the SDK exposes a +// delete operation; Apply returns an explicit error for removals. +func NewDNSProvider(client ecloud.DnsClient, defaultZoneName string) (dnsprovider.Interface, error) { + dnsClient, ok := any(&client).(dnsClient) + if !ok { + return nil, fmt.Errorf("Elemento DNS SDK does not implement the read methods required by the kOps DNS provider") + } + + return &dnsProvider{ + client: dnsClient, + defaultZoneName: defaultZoneName, + }, nil +} + +type dnsProvider struct { + client dnsClient + defaultZoneName string +} + +var _ dnsprovider.Interface = &dnsProvider{} + +func (p *dnsProvider) Zones() (dnsprovider.Zones, bool) { + return &dnsZones{ + client: p.client, + defaultZoneName: p.defaultZoneName, + }, true +} + +type dnsZones struct { + client dnsClient + defaultZoneName string +} + +var _ dnsprovider.Zones = &dnsZones{} + +func (z *dnsZones) List() ([]dnsprovider.Zone, error) { + if listClient, ok := z.client.(dnsListClient); ok { + dnsServices, _, err := listClient.List(context.TODO()) + if err != nil { + return nil, fmt.Errorf("listing Elemento DNS zones: %w", err) + } + return zonesFromElementoDNS(z.client, dnsServices), nil + } + + if z.defaultZoneName == "" { + return nil, fmt.Errorf("listing Elemento DNS zones requires DnsClient.List or a default zone name") + } + + dnsService, _, err := z.client.Get(context.TODO(), z.defaultZoneName) + if err != nil { + return nil, fmt.Errorf("getting Elemento DNS zone %q: %w", z.defaultZoneName, err) + } + if dnsService == nil { + return nil, nil + } + + return zonesFromElementoDNS(z.client, []*ecloud.Dns{dnsService}), nil +} + +func (z *dnsZones) Add(newZone dnsprovider.Zone) (dnsprovider.Zone, error) { + dnsService, _, err := z.client.Create(context.TODO(), newZone.Name()) + if err != nil && !IsDNSAlreadyExists(err) { + return nil, fmt.Errorf("creating Elemento DNS zone %q: %w", newZone.Name(), err) + } + if dnsService == nil { + dnsService, _, err = z.client.Get(context.TODO(), newZone.Name()) + if err != nil { + return nil, fmt.Errorf("getting Elemento DNS zone %q after create: %w", newZone.Name(), err) + } + } + + return zoneFromElementoDNS(z.client, newZone.Name(), dnsService), nil +} + +func (z *dnsZones) Remove(zone dnsprovider.Zone) error { + return fmt.Errorf("deleting Elemento DNS zones is not supported yet") +} + +func (z *dnsZones) New(name string) (dnsprovider.Zone, error) { + return &dnsZone{ + client: z.client, + name: name, + id: name, + }, nil +} + +func zonesFromElementoDNS(client dnsClient, dnsServices []*ecloud.Dns) []dnsprovider.Zone { + var zones []dnsprovider.Zone + for _, dnsService := range dnsServices { + if dnsService == nil { + continue + } + zones = append(zones, zoneFromElementoDNS(client, dnsService.ZoneName, dnsService)) + } + return zones +} + +func zoneFromElementoDNS(client dnsClient, name string, dnsService *ecloud.Dns) dnsprovider.Zone { + id := name + if dnsService != nil { + if dnsService.ZoneName != "" { + name = dnsService.ZoneName + } + if dnsService.ID != "" { + id = dnsService.ID + } + } + + return &dnsZone{ + client: client, + name: name, + id: id, + } +} + +type dnsZone struct { + client dnsClient + name string + id string +} + +var _ dnsprovider.Zone = &dnsZone{} + +func (z *dnsZone) Name() string { + return z.name +} + +func (z *dnsZone) ID() string { + return z.id +} + +func (z *dnsZone) ResourceRecordSets() (dnsprovider.ResourceRecordSets, bool) { + return &dnsResourceRecordSets{ + client: z.client, + zone: z, + }, true +} + +type dnsResourceRecordSets struct { + client dnsClient + zone *dnsZone +} + +var _ dnsprovider.ResourceRecordSets = &dnsResourceRecordSets{} + +func (r *dnsResourceRecordSets) List() ([]dnsprovider.ResourceRecordSet, error) { + records, _, err := r.client.ListDnsRecords(context.TODO(), r.zone.Name()) + if err != nil { + return nil, fmt.Errorf("listing Elemento DNS records in zone %q: %w", r.zone.Name(), err) + } + + var rrsets []dnsprovider.ResourceRecordSet + for _, record := range records { + if record == nil { + continue + } + rrsets = append(rrsets, rrsetFromElementoRecord(record)) + } + + return rrsets, nil +} + +func (r *dnsResourceRecordSets) Get(name string) ([]dnsprovider.ResourceRecordSet, error) { + recordName := trimZoneSuffix(name, r.zone.Name()) + record, _, err := r.client.GetDnsRecord(context.TODO(), r.zone.Name(), recordName, string(rrstype.A)) + if err != nil { + if IsDNSMissing(err) { + return nil, nil + } + return nil, fmt.Errorf("getting Elemento DNS record %q in zone %q: %w", recordName, r.zone.Name(), err) + } + if record == nil { + return nil, nil + } + + return []dnsprovider.ResourceRecordSet{rrsetFromElementoRecord(record)}, nil +} + +func (r *dnsResourceRecordSets) New(name string, rrdatas []string, ttl int64, recordType rrstype.RrsType) dnsprovider.ResourceRecordSet { + if len(rrdatas) == 0 { + return nil + } + return &dnsResourceRecordSet{ + name: name, + rrdatas: rrdatas, + ttl: ttl, + recordType: recordType, + } +} + +func (r *dnsResourceRecordSets) StartChangeset() dnsprovider.ResourceRecordChangeset { + return &dnsResourceRecordChangeset{ + client: r.client, + rrsets: r, + zoneName: r.zone.Name(), + } +} + +func (r *dnsResourceRecordSets) Zone() dnsprovider.Zone { + return r.zone +} + +func rrsetFromElementoRecord(record *ecloud.DnsRecord) dnsprovider.ResourceRecordSet { + return &dnsResourceRecordSet{ + name: record.Name, + rrdatas: []string{record.Value}, + ttl: int64(record.TTL), + recordType: rrstype.RrsType(record.Type), + } +} + +type dnsResourceRecordSet struct { + name string + rrdatas []string + ttl int64 + recordType rrstype.RrsType +} + +var _ dnsprovider.ResourceRecordSet = &dnsResourceRecordSet{} + +func (r *dnsResourceRecordSet) Name() string { + return r.name +} + +func (r *dnsResourceRecordSet) Rrdatas() []string { + return r.rrdatas +} + +func (r *dnsResourceRecordSet) Ttl() int64 { + return r.ttl +} + +func (r *dnsResourceRecordSet) Type() rrstype.RrsType { + return r.recordType +} + +type dnsResourceRecordChangeset struct { + client dnsClient + rrsets dnsprovider.ResourceRecordSets + zoneName string + + additions []dnsprovider.ResourceRecordSet + removals []dnsprovider.ResourceRecordSet + upserts []dnsprovider.ResourceRecordSet +} + +var _ dnsprovider.ResourceRecordChangeset = &dnsResourceRecordChangeset{} + +func (c *dnsResourceRecordChangeset) Add(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset { + c.additions = append(c.additions, rrset) + return c +} + +func (c *dnsResourceRecordChangeset) Remove(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset { + c.removals = append(c.removals, rrset) + return c +} + +func (c *dnsResourceRecordChangeset) Upsert(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset { + c.upserts = append(c.upserts, rrset) + return c +} + +func (c *dnsResourceRecordChangeset) Apply(ctx context.Context) error { + if c.IsEmpty() { + return nil + } + if len(c.removals) != 0 { + return fmt.Errorf("deleting Elemento DNS records is not supported yet") + } + + if _, _, err := c.client.Create(ctx, c.zoneName); err != nil && !IsDNSAlreadyExists(err) { + return fmt.Errorf("creating Elemento DNS zone %q: %w", c.zoneName, err) + } + + for _, rrset := range append(c.additions, c.upserts...) { + if rrset == nil { + continue + } + if rrset.Type() != rrstype.A { + return fmt.Errorf("Elemento DNS currently supports only A records, got %q for %q", rrset.Type(), rrset.Name()) + } + + recordName := trimZoneSuffix(rrset.Name(), c.zoneName) + for _, rrdata := range rrset.Rrdatas() { + if _, _, err := c.client.AddDnsRecord(ctx, c.zoneName, recordName, rrdata); err != nil { + return fmt.Errorf("ensuring Elemento DNS record %q in zone %q: %w", recordName, c.zoneName, err) + } + } + } + + return nil +} + +func (c *dnsResourceRecordChangeset) IsEmpty() bool { + return len(c.additions) == 0 && len(c.upserts) == 0 && len(c.removals) == 0 +} + +func (c *dnsResourceRecordChangeset) ResourceRecordSets() dnsprovider.ResourceRecordSets { + return c.rrsets +} + +// IsDNSAlreadyExists reports whether an Elemento DNS create/upsert operation +// found an existing resource that is already compatible with the request. +func IsDNSAlreadyExists(err error) bool { + if ecloud.IsError(err, ecloud.ErrorCodeUniquenessError, ecloud.ErrorCodeConflict) { + return true + } + + message := strings.ToLower(err.Error()) + return strings.Contains(message, "already exists") || + strings.Contains(message, "already defined") || + strings.Contains(message, "uniqueness") +} + +// IsDNSMissing reports whether an Elemento DNS read operation did not find the +// requested zone or record. +func IsDNSMissing(err error) bool { + if ecloud.IsError(err, ecloud.ErrorCodeNotFound, ecloud.ErrorCodeDNSZoneNotFound) { + return true + } + + message := strings.ToLower(err.Error()) + return strings.Contains(message, "dns service not found") || + strings.Contains(message, "dns zone not found") || + strings.Contains(message, "api error: 404") +} + +func trimZoneSuffix(name, zone string) string { + zone = strings.TrimSuffix(zone, ".") + return strings.TrimSuffix(name, "."+zone) +} diff --git a/upup/pkg/fi/cloudup/elemento/verifier.go b/upup/pkg/fi/cloudup/elemento/verifier.go index af9abd844aca7..b2e320b845217 100644 --- a/upup/pkg/fi/cloudup/elemento/verifier.go +++ b/upup/pkg/fi/cloudup/elemento/verifier.go @@ -19,91 +19,83 @@ package elemento import ( "context" "fmt" - - // "net" + "net" "net/http" - // "strings" - // "strconv" + "strings" - "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" "k8s.io/kops/pkg/bootstrap" - // "k8s.io/kops/pkg/wellknownports" ) type ElementoVerifierOptions struct { } type elementoVerifier struct { - opt ElementoVerifierOptions - client *ecloud.Client + opt ElementoVerifierOptions } var _ bootstrap.Verifier = &elementoVerifier{} -func NewElementoVerifier(opt *ElementoVerifierOptions) (bootstrap.Verifier, error) { - elementoClient, err := ecloud.NewClient("kops-elemento", "1.0") - if err != nil { - return nil, fmt.Errorf("failed to get server info: %w", err) - } +type staticBootstrapNode struct { + NodeName string + InstanceGroupName string +} + +var staticBootstrapNodesByIP = map[string]staticBootstrapNode{ + "192.168.100.10": {NodeName: "control-plane-europe-1", InstanceGroupName: "control-plane-europe"}, + "192.168.100.11": {NodeName: "nodes-europe-1", InstanceGroupName: "nodes-europe"}, + "192.168.100.12": {NodeName: "nodes-europe-2", InstanceGroupName: "nodes-europe"}, +} +func NewElementoVerifier(opt *ElementoVerifierOptions) (bootstrap.Verifier, error) { return &elementoVerifier{ - opt: *opt, - client: elementoClient, + opt: *opt, }, nil } func (e elementoVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request, token string, body []byte) (*bootstrap.VerifyResult, error) { - // DISABLED: Comment out all verification checks for testing - /* - if !strings.HasPrefix(token, ElementoAuthenticationTokenPrefix) { - return nil, fmt.Errorf("invalid token format") - } - token = strings.TrimPrefix(token, ElementoAuthenticationTokenPrefix) - - server, _, err := e.client.Server.GetByID(ctx, token) - if err != nil || server == nil { - return nil, fmt.Errorf("failed to get server info: %w", err) - } + if !strings.HasPrefix(token, ElementoAuthenticationTokenPrefix) { + return nil, bootstrap.ErrNotThisVerifier + } + token = strings.TrimSpace(strings.TrimPrefix(token, ElementoAuthenticationTokenPrefix)) - var addrs []string - var challengeEndpoints []string - if server.PublicNet.IPv4 != "" { - // Don't challenge over the public network - addrs = append(addrs, server.PublicNet.IPv4) - } - for _, network := range server.PrivateNet { - if network.IP != nil { - addrs = append(addrs, network.IP.String()) - challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(network.IP.String(), strconv.Itoa(wellknownports.NodeupChallenge))) - } - } + remoteHost, _, err := net.SplitHostPort(rawRequest.RemoteAddr) + if err != nil { + remoteHost = rawRequest.RemoteAddr + } - if len(challengeEndpoints) == 0 { - return nil, fmt.Errorf("cannot determine challenge endpoint for server %q", server.ID) - } + nodeName := token + instanceGroupName := "" + if node, ok := staticBootstrapNodesByIP[remoteHost]; ok { + nodeName = node.NodeName + instanceGroupName = node.InstanceGroupName + } + if instanceGroupName == "" { + instanceGroupName = inferInstanceGroupName(nodeName) + } + if instanceGroupName == "" { + return nil, fmt.Errorf("failed to determine instance group for node %q", nodeName) + } + certificateNames := []string{nodeName} + if remoteHost != "" { + certificateNames = append(certificateNames, remoteHost) + } - result := &bootstrap.VerifyResult{ - NodeName: server.Name, - CertificateNames: addrs, - ChallengeEndpoint: challengeEndpoints[0], - } + return &bootstrap.VerifyResult{ + NodeName: nodeName, + CertificateNames: certificateNames, + InstanceGroupName: instanceGroupName, + }, nil +} - for key, value := range server.Labels { - if key == TagKubernetesInstanceGroup { - result.InstanceGroupName = value - } +func inferInstanceGroupName(serverName string) string { + i := strings.LastIndex(serverName, "-") + if i == -1 || i == len(serverName)-1 { + return serverName + } + for _, r := range serverName[i+1:] { + if r < '0' || r > '9' { + return serverName } - - return result, nil - */ - - // DISABLED: Return a dummy successful verification result - result := &bootstrap.VerifyResult{ - NodeName: "test-node", - CertificateNames: []string{"127.0.0.1"}, - ChallengeEndpoint: "127.0.0.1:10000", - InstanceGroupName: "nodes", } - - return result, nil + return serverName[:i] } diff --git a/upup/pkg/fi/cloudup/elementotasks/dns.go b/upup/pkg/fi/cloudup/elementotasks/dns.go new file mode 100644 index 0000000000000..4ca19d0c6a88a --- /dev/null +++ b/upup/pkg/fi/cloudup/elementotasks/dns.go @@ -0,0 +1,221 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elementotasks + +import ( + "context" + "fmt" + + "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" + "k8s.io/klog/v2" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" +) + +// +kops:fitask +type DNSZone struct { + Name *string + Lifecycle fi.Lifecycle +} + +var _ fi.CloudupTask = &DNSZone{} +var _ fi.HasLifecycle = &DNSZone{} +var _ fi.HasName = &DNSZone{} + +func (z *DNSZone) GetLifecycle() fi.Lifecycle { + return z.Lifecycle +} + +func (z *DNSZone) SetLifecycle(lifecycle fi.Lifecycle) { + z.Lifecycle = lifecycle +} + +func (z *DNSZone) GetName() *string { + return z.Name +} + +func (z *DNSZone) String() string { + return fi.CloudupTaskAsString(z) +} + +func (z *DNSZone) Find(c *fi.CloudupContext) (*DNSZone, error) { + cloud := c.T.Cloud.(elemento.ElementoCloud) + client := cloud.DnsClient() + zoneName := fi.ValueOf(z.Name) + + fmt.Printf("EKOPS: Finding Elemento DNS zone %q\n", zoneName) + + dns, _, err := client.Get(context.TODO(), zoneName) + if err != nil { + if elemento.IsDNSMissing(err) { + fmt.Printf("EKOPS: Elemento DNS zone %q not found\n", zoneName) + return nil, nil + } + return nil, fmt.Errorf("getting Elemento DNS zone %q: %w", zoneName, err) + } + if dns == nil { + fmt.Printf("EKOPS: Elemento DNS zone %q not found\n", zoneName) + return nil, nil + } + + fmt.Printf("EKOPS: Elemento DNS zone %q already exists\n", zoneName) + return &DNSZone{ + Name: fi.PtrTo(dns.ZoneName), + Lifecycle: z.Lifecycle, + }, nil +} + +func (z *DNSZone) Run(c *fi.CloudupContext) error { + return fi.CloudupDefaultDeltaRunMethod(z, c) +} + +func (_ *DNSZone) CheckChanges(actual, expected, changes *DNSZone) error { + if expected.Name == nil { + return fi.RequiredField("Name") + } + return nil +} + +func (_ *DNSZone) RenderElemento(t *elemento.ElementoAPITarget, actual, expected, changes *DNSZone) error { + zoneName := fi.ValueOf(expected.Name) + fmt.Printf("EKOPS: Ensuring Elemento DNS zone %q\n", zoneName) + if err := ensureElementoDNSZone(context.TODO(), t.Cloud.DnsClient(), zoneName); err != nil { + return err + } + fmt.Printf("EKOPS: Elemento DNS zone %q ensured\n", zoneName) + return nil +} + +// +kops:fitask +type DNSRecord struct { + Name *string + Data *string + DNSZone *string + DNSZoneTask *DNSZone + DependsOn *DNSRecord + Type *string + TTL *int64 + Lifecycle fi.Lifecycle + Comment *string +} + +var _ fi.CloudupTask = &DNSRecord{} +var _ fi.CloudupHasDependencies = &DNSRecord{} + +func (d *DNSRecord) GetDependencies(tasks map[string]fi.CloudupTask) []fi.CloudupTask { + var deps []fi.CloudupTask + if d.DNSZoneTask != nil { + deps = append(deps, d.DNSZoneTask) + } + if d.DependsOn != nil { + deps = append(deps, d.DependsOn) + } + return deps +} + +func (d *DNSRecord) Find(c *fi.CloudupContext) (*DNSRecord, error) { + cloud := c.T.Cloud.(elemento.ElementoCloud) + client := cloud.DnsClient() + + record, _, err := client.GetDnsRecord(context.TODO(), fi.ValueOf(d.DNSZone), fi.ValueOf(d.Name), fi.ValueOf(d.Type)) + if err != nil { + if elemento.IsDNSMissing(err) { + return nil, nil + } + return nil, fmt.Errorf("getting Elemento DNS record %q in zone %q: %w", fi.ValueOf(d.Name), fi.ValueOf(d.DNSZone), err) + } + if record == nil { + return nil, nil + } + + return &DNSRecord{ + Name: fi.PtrTo(record.Name), + Data: fi.PtrTo(record.Value), + DNSZone: d.DNSZone, + DNSZoneTask: d.DNSZoneTask, + DependsOn: d.DependsOn, + Type: fi.PtrTo(record.Type), + TTL: fi.PtrTo(int64(record.TTL)), + Lifecycle: d.Lifecycle, + Comment: d.Comment, + }, nil +} + +func (d *DNSRecord) Run(c *fi.CloudupContext) error { + return fi.CloudupDefaultDeltaRunMethod(d, c) +} + +func (_ *DNSRecord) CheckChanges(actual, expected, changes *DNSRecord) error { + if expected.Name == nil { + return fi.RequiredField("Name") + } + if expected.DNSZone == nil { + return fi.RequiredField("DNSZone") + } + if expected.Type == nil { + return fi.RequiredField("Type") + } + if fi.ValueOf(expected.Type) != "A" { + return fmt.Errorf("Elemento DNS currently supports only A records, got %q", fi.ValueOf(expected.Type)) + } + if expected.Data == nil { + return fi.RequiredField("Data") + } + + return nil +} + +func (_ *DNSRecord) RenderElemento(t *elemento.ElementoAPITarget, actual, expected, changes *DNSRecord) error { + client := t.Cloud.DnsClient() + zoneName := fi.ValueOf(expected.DNSZone) + recordName := fi.ValueOf(expected.Name) + recordValue := fi.ValueOf(expected.Data) + + if err := ensureElementoDNSRecord(context.TODO(), client, zoneName, recordName, recordValue); err != nil { + return err + } + + return nil +} + +func ensureElementoDNSZone(ctx context.Context, client ecloud.DnsClient, zoneName string) error { + _, _, err := client.Create(ctx, zoneName) + if err != nil { + if elemento.IsDNSAlreadyExists(err) { + klog.V(2).Infof("Elemento DNS zone %q already exists", zoneName) + return nil + } + return fmt.Errorf("creating Elemento DNS zone %q: %w", zoneName, err) + } + + klog.V(2).Infof("Created Elemento DNS zone %q", zoneName) + return nil +} + +func ensureElementoDNSRecord(ctx context.Context, client ecloud.DnsClient, zoneName, recordName, recordValue string) error { + record, _, err := client.AddDnsRecord(ctx, zoneName, recordName, recordValue) + if err != nil { + if elemento.IsDNSAlreadyExists(err) { + klog.V(2).Infof("Elemento DNS record %q in zone %q already exists", recordName, zoneName) + return nil + } + return fmt.Errorf("creating Elemento DNS record %q in zone %q: %w", recordName, zoneName, err) + } + + klog.V(2).Infof("Created Elemento DNS record %q in zone %q as %q", recordName, zoneName, record.Name) + return nil +} diff --git a/upup/pkg/fi/cloudup/elementotasks/dns_fitask.go b/upup/pkg/fi/cloudup/elementotasks/dns_fitask.go new file mode 100644 index 0000000000000..70ffa6d07a117 --- /dev/null +++ b/upup/pkg/fi/cloudup/elementotasks/dns_fitask.go @@ -0,0 +1,39 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elementotasks + +import "k8s.io/kops/upup/pkg/fi" + +var _ fi.HasLifecycle = &DNSRecord{} + +func (o *DNSRecord) GetLifecycle() fi.Lifecycle { + return o.Lifecycle +} + +func (o *DNSRecord) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = lifecycle +} + +var _ fi.HasName = &DNSRecord{} + +func (o *DNSRecord) GetName() *string { + return o.Name +} + +func (o *DNSRecord) String() string { + return fi.CloudupTaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/elementotasks/servergroup.go b/upup/pkg/fi/cloudup/elementotasks/servergroup.go index 62fe441cbc81b..bf56cede1fff4 100644 --- a/upup/pkg/fi/cloudup/elementotasks/servergroup.go +++ b/upup/pkg/fi/cloudup/elementotasks/servergroup.go @@ -21,7 +21,6 @@ import ( "crypto/sha256" "encoding/base64" "fmt" - "math/rand" "strings" "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" @@ -52,10 +51,37 @@ type ServerGroup struct { Labels map[string]string + DNSZoneTask *DNSZone + DNSRecordTasks []*DNSRecord + // RootVolumeSize is the size of the root volume in GB RootVolumeSize *int32 } +var _ fi.CloudupHasDependencies = &ServerGroup{} + +func (v *ServerGroup) GetDependencies(tasks map[string]fi.CloudupTask) []fi.CloudupTask { + var deps []fi.CloudupTask + + for _, sshKey := range v.SSHKeys { + deps = append(deps, sshKey) + } + if v.Network != nil { + deps = append(deps, v.Network) + } + if v.DNSZoneTask != nil { + deps = append(deps, v.DNSZoneTask) + } + for _, dnsRecordTask := range v.DNSRecordTasks { + deps = append(deps, dnsRecordTask) + } + if v.UserData != nil { + deps = append(deps, fi.FindDependencies(tasks, v.UserData)...) + } + + return deps +} + func (v *ServerGroup) Find(c *fi.CloudupContext) (*ServerGroup, error) { cloud := c.T.Cloud.(elemento.ElementoCloud) client := cloud.ServerClient() @@ -237,8 +263,9 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes fmt.Printf("EKOPS: UserData length: %d bytes, hash: %s\n", len(userData), userDataHash) for i := 1; i <= expectedCount-actualCount; i++ { - // Append a random/unique ID to the node name - name := fmt.Sprintf("%s-%x", fi.ValueOf(e.Name), rand.Int63()) + // Use deterministic names so manual DNS records can be created before VM creation. + ordinal := actualCount + i + name := fmt.Sprintf("%s-%d", fi.ValueOf(e.Name), ordinal) // Initialize labels if nil labels := e.Labels @@ -267,7 +294,7 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes // Add root volume configuration if specified if e.RootVolumeSize != nil { - opts.ServerType.Disk = int(fi.ValueOf(e.RootVolumeSize)) + opts.ServerType.Disk = float64(fi.ValueOf(e.RootVolumeSize)) } // Add the SSH keys. @@ -284,9 +311,10 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes fmt.Printf("EKOPS: Creating server %q with options: Location=%s, Size=%s, Image=%s\n", name, e.Location, e.Size, e.Image) + fmt.Printf("EKOPS: Calling client.Create() for server %q\n", name) - _, _, err = client.Create(context.TODO(), opts) + _, _, err := client.Create(context.TODO(), opts) if err != nil { fmt.Printf("EKOPS: ERROR creating server %q: %v\n", name, err) return err diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 2bc88ac846efc..71dcce3d3b0e5 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -259,7 +259,7 @@ func (c *populateClusterSpec) run(ctx context.Context, clientset simple.Clientse cluster.Spec.API.LoadBalancer.Class = kopsapi.LoadBalancerClassClassic } - if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() { + if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() && cluster.GetCloudProvider() != kopsapi.CloudProviderElemento { dns, err := cloud.DNS() if err != nil { return err @@ -284,7 +284,7 @@ func (c *populateClusterSpec) run(ctx context.Context, clientset simple.Clientse if cluster.Spec.DNSZone != "" && cluster.Spec.API.PublicName == "" { cluster.Spec.API.PublicName = "api." + cluster.Name } - if cluster.Spec.ExternalDNS == nil { + if cluster.Spec.ExternalDNS == nil && cluster.GetCloudProvider() != kopsapi.CloudProviderElemento { cluster.Spec.ExternalDNS = &kopsapi.ExternalDNSConfig{ Provider: kopsapi.ExternalDNSProviderDNSController, } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index c4d1ffe39d115..b766ce2946a5b 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -65,12 +65,12 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/do" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/gce" gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" - "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/util/pkg/env" "k8s.io/kops/util/pkg/maps" "sigs.k8s.io/yaml" @@ -641,7 +641,6 @@ func (tf *TemplateFunctions) DNSControllerArgv() ([]string, error) { argv = append(argv, "--dns=openstack-designate") case kops.CloudProviderScaleway: argv = append(argv, "--dns=scaleway") - default: return nil, fmt.Errorf("unhandled cloudprovider %q", cluster.GetCloudProvider()) } diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml index 37aef13e90b3c..3c4833df70615 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml index 37aef13e90b3c..3c4833df70615 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml index dee7eca1d1e5a..ea36d775bffa2 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml index 7ccb2666b3804..9ba9ea8522bfc 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml index b29751b1630f4..ea20e01d5099e 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml index 1a2921dd4ff50..a351116e3ac64 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -136,6 +136,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index a3c584e488b65..e47d7fa97a067 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml index 378192acc6551..96a421c5f1e7a 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml index 0c51821831965..eb5f8a81d816d 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml index 2aef52067feac..7dc2bc46fc854 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml index 1a2921dd4ff50..a351116e3ac64 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -136,6 +136,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml index d9717082bac51..1bb6faf9266b9 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml index 1a2921dd4ff50..a351116e3ac64 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -136,6 +136,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index 5e7def30fc939..674fd54fe8ef2 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/executor.go b/upup/pkg/fi/executor.go index ccd71a292f0ba..a72a68671e133 100644 --- a/upup/pkg/fi/executor.go +++ b/upup/pkg/fi/executor.go @@ -204,9 +204,10 @@ func (e *executor[T]) RunTasks(ctx context.Context, taskMap map[string]Task[T]) } // Execute final ecloud task after all other tasks are completed - if err := e.executeFinalECloudTask(ctx); err != nil { - return fmt.Errorf("failed to execute final ecloud task: %v", err) - } + /* + if err := e.executeFinalECloudTask(ctx); err != nil { + return fmt.Errorf("failed to execute final ecloud task: %v", err) + }*/ return nil }