From 992748efa452c7199251573936312253d7a44653 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Mon, 13 Apr 2026 15:31:28 +0200 Subject: [PATCH 01/10] initial commit on the branch --- go.mod | 2 +- pkg/model/elementomodel/dns.go | 46 +++++++++++++++++++ upup/pkg/fi/cloudup/apply_cluster.go | 1 + .../bootstrapchannelbuilder.go | 5 +- upup/pkg/fi/cloudup/elemento/cloud.go | 8 +--- upup/pkg/fi/cloudup/populate_cluster_spec.go | 2 +- upup/pkg/fi/cloudup/template_functions.go | 3 +- 7 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 pkg/model/elementomodel/dns.go diff --git a/go.mod b/go.mod index 395e8f6312290..3ac71d6808014 100644 --- a/go.mod +++ b/go.mod @@ -102,7 +102,7 @@ require ( sigs.k8s.io/yaml v1.4.0 ) -// replace github.com/Elemento-Modular-Cloud/ecloud-go v1.0.1 => ../ecloud-go +replace github.com/Elemento-Modular-Cloud/ecloud-go => ../ecloud-go require ( cloud.google.com/go/auth v0.9.5 // indirect diff --git a/pkg/model/elementomodel/dns.go b/pkg/model/elementomodel/dns.go new file mode 100644 index 0000000000000..8c3657140a11a --- /dev/null +++ b/pkg/model/elementomodel/dns.go @@ -0,0 +1,46 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elementomodel + +import "k8s.io/kops/upup/pkg/fi" + +// DNSModelBuilder is the provider-native integration point for Elemento-managed +// DNS records that must exist before nodeup starts. +type DNSModelBuilder struct { + *ElementoModelContext + Lifecycle fi.Lifecycle +} + +var _ fi.CloudupModelBuilder = &DNSModelBuilder{} + +func (b *DNSModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { + if !b.Cluster.PublishesDNSRecords() { + return nil + } + + // TODO: Create Elemento DNS tasks here for the records needed during early bootstrap. + // At minimum this should cover: + // - api. when the cluster uses DNS instead of an API load balancer hostname + // - api.internal. so kubeconfig, service-account issuer discovery, and + // internal control-plane traffic can resolve before in-cluster components reconcile + // - kops-controller.internal. so worker nodeup can reach the config server + // + // TODO: Replace these comments with calls to the Elemento DNS API once the SDK + // surface for zone/record management is available. + _ = c + return nil +} diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index c35cb655d1359..9570b34892d0e 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -722,6 +722,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { } l.Builders = append(l.Builders, &elementomodel.NetworkModelBuilder{ElementoModelContext: elementoModelContext, Lifecycle: networkLifecycle}, + &elementomodel.DNSModelBuilder{ElementoModelContext: elementoModelContext, Lifecycle: networkLifecycle}, &elementomodel.ServerGroupModelBuilder{ElementoModelContext: elementoModelContext, BootstrapScriptBuilder: bootstrapScriptBuilder, Lifecycle: clusterLifecycle}, ) diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 00b38ec884710..fa6a704a7c86a 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -439,8 +439,9 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } if !b.Cluster.UsesNoneDNS() { + usesManagedDNSController := b.Cluster.GetCloudProvider() != kops.CloudProviderElemento if b.Cluster.Spec.ExternalDNS == nil || b.Cluster.Spec.ExternalDNS.Provider == kops.ExternalDNSProviderDNSController { - { + if usesManagedDNSController { key := "dns-controller.addons.k8s.io" location := key + "/k8s-1.12.yaml" id := "k8s-1.12" @@ -455,7 +456,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) // Generate dns-controller ServiceAccount IAM permissions. // Gossip and dns=none clusters do not require any cloud permissions. - if b.UseServiceAccountExternalPermissions() && b.Cluster.PublishesDNSRecords() { + if usesManagedDNSController && b.UseServiceAccountExternalPermissions() && b.Cluster.PublishesDNSRecords() { serviceAccountRoles = append(serviceAccountRoles, &dnscontroller.ServiceAccount{}) } } else if b.Cluster.Spec.ExternalDNS.Provider == kops.ExternalDNSProviderExternalDNS { diff --git a/upup/pkg/fi/cloudup/elemento/cloud.go b/upup/pkg/fi/cloudup/elemento/cloud.go index a34cd55cf7f9b..538b69a3a5bf9 100644 --- a/upup/pkg/fi/cloudup/elemento/cloud.go +++ b/upup/pkg/fi/cloudup/elemento/cloud.go @@ -62,7 +62,6 @@ var _ fi.Cloud = &elementoCloudImplementation{} type elementoCloudImplementation struct { Client *ecloud.Client region string - dns dnsprovider.Interface // TODO: Add additional fields here } @@ -72,6 +71,7 @@ func NewElementoCloud(region string) (ElementoCloud, error) { klog.V(2).Infof("Creating ecloud client for region %s", region) + // Here is the entrypoint of the ecloud-go SDK, executed by building a new ecloud-client client, err := ecloud.NewClient("kops-client", "0.1") if err != nil { @@ -83,7 +83,6 @@ func NewElementoCloud(region string) (ElementoCloud, error) { return &elementoCloudImplementation{ Client: client, - dns: nil, region: region, }, nil } @@ -153,10 +152,7 @@ func findServerGroups(c *elementoCloudImplementation, clusterName string) (map[s } func (c *elementoCloudImplementation) DNS() (dnsprovider.Interface, error) { - if c.dns == nil { - return nil, fmt.Errorf("DNS provider is not initialized") - } - return c.dns, nil + return nil, nil } func (c *elementoCloudImplementation) NetworkClient() ecloud.NetworkClient { diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 2bc88ac846efc..d99c6e36e026d 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -284,7 +284,7 @@ func (c *populateClusterSpec) run(ctx context.Context, clientset simple.Clientse if cluster.Spec.DNSZone != "" && cluster.Spec.API.PublicName == "" { cluster.Spec.API.PublicName = "api." + cluster.Name } - if cluster.Spec.ExternalDNS == nil { + if cluster.Spec.ExternalDNS == nil && cluster.GetCloudProvider() != kops.CloudProviderElemento { cluster.Spec.ExternalDNS = &kopsapi.ExternalDNSConfig{ Provider: kopsapi.ExternalDNSProviderDNSController, } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index c4d1ffe39d115..b766ce2946a5b 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -65,12 +65,12 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/do" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/gce" gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" - "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/util/pkg/env" "k8s.io/kops/util/pkg/maps" "sigs.k8s.io/yaml" @@ -641,7 +641,6 @@ func (tf *TemplateFunctions) DNSControllerArgv() ([]string, error) { argv = append(argv, "--dns=openstack-designate") case kops.CloudProviderScaleway: argv = append(argv, "--dns=scaleway") - default: return nil, fmt.Errorf("unhandled cloudprovider %q", cluster.GetCloudProvider()) } From f3a240fb558e1ff4bfb7bd19c401c54f6be54d6a Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Mon, 13 Apr 2026 16:15:47 +0200 Subject: [PATCH 02/10] chore: first lines of code to start integrating elemento's DNS in kops --- pkg/model/elementomodel/dns.go | 77 ++++++- upup/pkg/fi/cloudup/elemento/cloud.go | 6 +- .../fi/cloudup/elementotasks/dns_record.go | 188 ++++++++++++++++++ .../elementotasks/dns_record_fitask.go | 39 ++++ upup/pkg/fi/cloudup/populate_cluster_spec.go | 2 +- 5 files changed, 299 insertions(+), 13 deletions(-) create mode 100644 upup/pkg/fi/cloudup/elementotasks/dns_record.go create mode 100644 upup/pkg/fi/cloudup/elementotasks/dns_record_fitask.go diff --git a/pkg/model/elementomodel/dns.go b/pkg/model/elementomodel/dns.go index 8c3657140a11a..866ad37080f0c 100644 --- a/pkg/model/elementomodel/dns.go +++ b/pkg/model/elementomodel/dns.go @@ -16,7 +16,18 @@ limitations under the License. package elementomodel -import "k8s.io/kops/upup/pkg/fi" +import ( + "strings" + + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/elementotasks" +) + +const ( + placeholderIP = "203.0.113.123" + kopsControllerInternalRecordPrefix = "kops-controller.internal." + defaultTTL = int64(60) +) // DNSModelBuilder is the provider-native integration point for Elemento-managed // DNS records that must exist before nodeup starts. @@ -32,15 +43,59 @@ func (b *DNSModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { return nil } - // TODO: Create Elemento DNS tasks here for the records needed during early bootstrap. - // At minimum this should cover: - // - api. when the cluster uses DNS instead of an API load balancer hostname - // - api.internal. so kubeconfig, service-account issuer discovery, and - // internal control-plane traffic can resolve before in-cluster components reconcile - // - kops-controller.internal. so worker nodeup can reach the config server - // - // TODO: Replace these comments with calls to the Elemento DNS API once the SDK - // surface for zone/record management is available. - _ = c + // This builder mirrors the role of other provider-specific DNS builders in kOps: + // it declares the DNS records that must exist before nodeup starts. The actual + // Elemento DNS API calls are delegated to elementotasks.DNSRecord. + + if !b.UseLoadBalancerForAPI() { + recordName := trimZoneSuffix(b.Cluster.Spec.API.PublicName, b.Cluster.Spec.DNSZone) + c.AddTask(&elementotasks.DNSRecord{ + Name: fi.PtrTo(recordName), + DNSZone: fi.PtrTo(b.Cluster.Spec.DNSZone), + Type: fi.PtrTo("A"), + Data: fi.PtrTo(placeholderIP), + TTL: fi.PtrTo(defaultTTL), + Lifecycle: b.Lifecycle, + Comment: fi.PtrTo( + "Bootstrap record for the public Kubernetes API endpoint. " + + "Replace the placeholder target with the final Elemento-managed VIP or public API address.", + ), + }) + } + + if !b.UseLoadBalancerForInternalAPI() { + recordName := trimZoneSuffix(b.Cluster.APIInternalName(), b.Cluster.Spec.DNSZone) + c.AddTask(&elementotasks.DNSRecord{ + Name: fi.PtrTo(recordName), + DNSZone: fi.PtrTo(b.Cluster.Spec.DNSZone), + Type: fi.PtrTo("A"), + Data: fi.PtrTo(placeholderIP), + TTL: fi.PtrTo(defaultTTL), + Lifecycle: b.Lifecycle, + Comment: fi.PtrTo( + "Bootstrap record for api.internal. This must resolve before kubeconfig, " + + "service-account issuer discovery, and early control-plane traffic start using it.", + ), + }) + } + + recordName := kopsControllerInternalRecordPrefix + strings.TrimSuffix(b.Cluster.ObjectMeta.Name, "."+b.Cluster.Spec.DNSZone) + c.AddTask(&elementotasks.DNSRecord{ + Name: fi.PtrTo(recordName), + DNSZone: fi.PtrTo(b.Cluster.Spec.DNSZone), + Type: fi.PtrTo("A"), + Data: fi.PtrTo(placeholderIP), + TTL: fi.PtrTo(defaultTTL), + Lifecycle: b.Lifecycle, + Comment: fi.PtrTo( + "Bootstrap record for kops-controller.internal. Worker nodeup may use this very early " + + "to fetch configuration from the config server.", + ), + }) + return nil } + +func trimZoneSuffix(name string, zone string) string { + return strings.TrimSuffix(name, "."+zone) +} diff --git a/upup/pkg/fi/cloudup/elemento/cloud.go b/upup/pkg/fi/cloudup/elemento/cloud.go index 538b69a3a5bf9..d1b4768fc1277 100644 --- a/upup/pkg/fi/cloudup/elemento/cloud.go +++ b/upup/pkg/fi/cloudup/elemento/cloud.go @@ -53,7 +53,9 @@ type ElementoCloud interface { VolumeClient() ecloud.VolumeClient NodeupClient(ctx context.Context) ecloud.NodeupClient - // TODO: Detect and add additional fields here + // TODO(elemento-dns): Add DNS-zone / DNS-record client accessors here once + // the Elemento SDK exposes them. The provider-native DNS tasks in + // elementotasks/dns_record.go are the intended callers. } var _ fi.Cloud = &elementoCloudImplementation{} @@ -152,6 +154,8 @@ func findServerGroups(c *elementoCloudImplementation, clusterName string) (map[s } func (c *elementoCloudImplementation) DNS() (dnsprovider.Interface, error) { + // Elemento DNS is expected to be managed through provider-native cloudup tasks, + // not through the generic dnsprovider.Interface used by providers such as Route53. return nil, nil } diff --git a/upup/pkg/fi/cloudup/elementotasks/dns_record.go b/upup/pkg/fi/cloudup/elementotasks/dns_record.go new file mode 100644 index 0000000000000..4f1ec93ead611 --- /dev/null +++ b/upup/pkg/fi/cloudup/elementotasks/dns_record.go @@ -0,0 +1,188 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elementotasks + +import ( + "context" + "fmt" + "strings" + + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" +) + +// +kops:fitask +type DNSRecord struct { + Name *string + Lifecycle fi.Lifecycle + + // DNSZone is the parent zone, for example "example.com". + DNSZone *string + // Type is the record type, typically "A" for bootstrap. + Type *string + // Data is the current record target. During the first implementation this is + // usually a placeholder value, until the final API VIP or LB address is known. + Data *string + // TTL is the DNS TTL in seconds. + TTL *int64 + // Comment is not used by kOps itself, but helps document why this bootstrap + // record exists while the Elemento DNS API integration is being completed. + Comment *string +} + +type managedDNSRecord struct { + ID string + Name string + Zone string + Type string + Data string + TTL int64 +} + +func (r *DNSRecord) Find(c *fi.CloudupContext) (*DNSRecord, error) { + cloud := c.T.Cloud.(elemento.ElementoCloud) + + actual, err := findManagedDNSRecord(context.TODO(), cloud, r) + if err != nil { + return nil, err + } + if actual == nil { + return nil, nil + } + + matches := &DNSRecord{ + Name: r.Name, + Lifecycle: r.Lifecycle, + DNSZone: fi.PtrTo(actual.Zone), + Type: fi.PtrTo(actual.Type), + Data: fi.PtrTo(actual.Data), + TTL: fi.PtrTo(actual.TTL), + Comment: r.Comment, + } + return matches, nil +} + +func (r *DNSRecord) Run(c *fi.CloudupContext) error { + return fi.CloudupDefaultDeltaRunMethod(r, c) +} + +func (*DNSRecord) CheckChanges(a, e, changes *DNSRecord) error { + if a != nil { + if changes.Name != nil { + return fi.CannotChangeField("Name") + } + if changes.DNSZone != nil { + return fi.CannotChangeField("DNSZone") + } + } + + if e.Name == nil { + return fi.RequiredField("Name") + } + if e.DNSZone == nil { + return fi.RequiredField("DNSZone") + } + if e.Type == nil { + return fi.RequiredField("Type") + } + if e.Data == nil { + return fi.RequiredField("Data") + } + if e.TTL == nil { + return fi.RequiredField("TTL") + } + + return nil +} + +func (*DNSRecord) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes *DNSRecord) error { + cloud := t.Cloud + + if a == nil { + return createManagedDNSRecord(context.TODO(), cloud, e) + } + + if changes.Data != nil || changes.TTL != nil || changes.Type != nil { + return updateManagedDNSRecord(context.TODO(), cloud, e) + } + + return nil +} + +func fullyQualifiedRecordName(recordName string, zone string) string { + recordName = strings.TrimSuffix(recordName, ".") + zone = strings.TrimSuffix(zone, ".") + if zone == "" { + return recordName + } + if strings.HasSuffix(recordName, "."+zone) || recordName == zone { + return recordName + } + return recordName + "." + zone +} + +func findManagedDNSRecord(ctx context.Context, cloud elemento.ElementoCloud, desired *DNSRecord) (*managedDNSRecord, error) { + _ = ctx + _ = cloud + + // TODO(elemento-dns): Replace this mock with a real lookup against the Elemento DNS API. + // + // Expected implementation shape: + // 1. Resolve the Elemento zone matching fi.ValueOf(desired.DNSZone) + // 2. List records in that zone + // 3. Find the record with: + // - fullyQualifiedRecordName(fi.ValueOf(desired.Name), fi.ValueOf(desired.DNSZone)) + // - type fi.ValueOf(desired.Type) + // 4. Return its current target and TTL so kOps can decide whether it changed + return nil, nil +} + +func createManagedDNSRecord(ctx context.Context, cloud elemento.ElementoCloud, desired *DNSRecord) error { + _ = ctx + _ = cloud + + fqdn := fullyQualifiedRecordName(fi.ValueOf(desired.Name), fi.ValueOf(desired.DNSZone)) + + // TODO(elemento-dns): Replace this error with the real Elemento DNS create call. + // + // Expected create payload: + // - zone: fi.ValueOf(desired.DNSZone) + // - name: fqdn + // - type: fi.ValueOf(desired.Type) + // - value: fi.ValueOf(desired.Data) + // - ttl: fi.ValueOf(desired.TTL) + // + // Suggested first target records: + // - api.internal. + // - kops-controller.internal. + // - api. if the public API endpoint should resolve through your zone + return fmt.Errorf("Elemento DNS create is not implemented yet for %q; implement createManagedDNSRecord in elementotasks/dns_record.go", fqdn) +} + +func updateManagedDNSRecord(ctx context.Context, cloud elemento.ElementoCloud, desired *DNSRecord) error { + _ = ctx + _ = cloud + + fqdn := fullyQualifiedRecordName(fi.ValueOf(desired.Name), fi.ValueOf(desired.DNSZone)) + + // TODO(elemento-dns): Replace this error with the real Elemento DNS update call. + // + // This should update at least: + // - target/value when the bootstrap placeholder is replaced by the final VIP/LB IP + // - ttl if you choose to lower it for bootstrap and raise it later + return fmt.Errorf("Elemento DNS update is not implemented yet for %q; implement updateManagedDNSRecord in elementotasks/dns_record.go", fqdn) +} diff --git a/upup/pkg/fi/cloudup/elementotasks/dns_record_fitask.go b/upup/pkg/fi/cloudup/elementotasks/dns_record_fitask.go new file mode 100644 index 0000000000000..70ffa6d07a117 --- /dev/null +++ b/upup/pkg/fi/cloudup/elementotasks/dns_record_fitask.go @@ -0,0 +1,39 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elementotasks + +import "k8s.io/kops/upup/pkg/fi" + +var _ fi.HasLifecycle = &DNSRecord{} + +func (o *DNSRecord) GetLifecycle() fi.Lifecycle { + return o.Lifecycle +} + +func (o *DNSRecord) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = lifecycle +} + +var _ fi.HasName = &DNSRecord{} + +func (o *DNSRecord) GetName() *string { + return o.Name +} + +func (o *DNSRecord) String() string { + return fi.CloudupTaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index d99c6e36e026d..fbac2243406f4 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -284,7 +284,7 @@ func (c *populateClusterSpec) run(ctx context.Context, clientset simple.Clientse if cluster.Spec.DNSZone != "" && cluster.Spec.API.PublicName == "" { cluster.Spec.API.PublicName = "api." + cluster.Name } - if cluster.Spec.ExternalDNS == nil && cluster.GetCloudProvider() != kops.CloudProviderElemento { + if cluster.Spec.ExternalDNS == nil && cluster.GetCloudProvider() != kopsapi.CloudProviderElemento { cluster.Spec.ExternalDNS = &kopsapi.ExternalDNSConfig{ Provider: kopsapi.ExternalDNSProviderDNSController, } From 7fc699c5d335f1cf6b1f08483df90269adccfb92 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Mon, 27 Apr 2026 16:50:08 +0200 Subject: [PATCH 03/10] chore: adjusting the elemento's provisioning dns logic in kops. --- upup/pkg/fi/cloudup/elemento/cloud.go | 11 +- upup/pkg/fi/cloudup/elementotasks/dns.go | 93 +++++++++ .../{dns_record_fitask.go => dns_fitask.go} | 0 .../fi/cloudup/elementotasks/dns_record.go | 188 ------------------ upup/pkg/fi/cloudup/populate_cluster_spec.go | 2 +- 5 files changed, 98 insertions(+), 196 deletions(-) create mode 100644 upup/pkg/fi/cloudup/elementotasks/dns.go rename upup/pkg/fi/cloudup/elementotasks/{dns_record_fitask.go => dns_fitask.go} (100%) delete mode 100644 upup/pkg/fi/cloudup/elementotasks/dns_record.go diff --git a/upup/pkg/fi/cloudup/elemento/cloud.go b/upup/pkg/fi/cloudup/elemento/cloud.go index d1b4768fc1277..e0569945d8492 100644 --- a/upup/pkg/fi/cloudup/elemento/cloud.go +++ b/upup/pkg/fi/cloudup/elemento/cloud.go @@ -24,7 +24,6 @@ import ( "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" - "k8s.io/kops/dnsprovider/pkg/dnsprovider" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/cloudinstances" "k8s.io/kops/upup/pkg/fi" @@ -45,17 +44,16 @@ const ( type ElementoCloud interface { fi.Cloud - Region() string - DNS() (dnsprovider.Interface, error) NetworkClient() ecloud.NetworkClient ServerClient() ecloud.ServerClient SSHKeyClient() ecloud.SSHKeyClient VolumeClient() ecloud.VolumeClient NodeupClient(ctx context.Context) ecloud.NodeupClient - // TODO(elemento-dns): Add DNS-zone / DNS-record client accessors here once + // Add DNS-zone / DNS-record client accessors here once // the Elemento SDK exposes them. The provider-native DNS tasks in // elementotasks/dns_record.go are the intended callers. + DnsClient() ecloud.DnsClient } var _ fi.Cloud = &elementoCloudImplementation{} @@ -153,10 +151,9 @@ func findServerGroups(c *elementoCloudImplementation, clusterName string) (map[s return serverGroups, nil } -func (c *elementoCloudImplementation) DNS() (dnsprovider.Interface, error) { +func (c *elementoCloudImplementation) DNS() ecloud.DnsClient { // Elemento DNS is expected to be managed through provider-native cloudup tasks, - // not through the generic dnsprovider.Interface used by providers such as Route53. - return nil, nil + return c.Client.Dns } func (c *elementoCloudImplementation) NetworkClient() ecloud.NetworkClient { diff --git a/upup/pkg/fi/cloudup/elementotasks/dns.go b/upup/pkg/fi/cloudup/elementotasks/dns.go new file mode 100644 index 0000000000000..b4954ec15959e --- /dev/null +++ b/upup/pkg/fi/cloudup/elementotasks/dns.go @@ -0,0 +1,93 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elementotasks + +import ( + "time" + + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" +) + +type Dns struct { + ID string + ZoneName *string + Created time.Time + AtomOsTarget string + Status string + Records []DnsRecord +} + +type DnsRecord struct { + Name string + Type string + Value string + TTL int +} + +func (d *Dns) Find(c *fi.CloudupContext) (*Dns, error) { + cloud := c.T.Cloud.(elemento.ElementoCloud) + client := cloud.DnsClient() + +} + +func (d *Dns) Run(c *fi.CloudupContext) error { + return fi.CloudupDefaultDeltaRunMethod(d, c) +} + +func (*Dns) CheckChanges(a, e, changes *Dns) error { + if a != nil { + if changes.Name != nil { + return fi.CannotChangeField("Name") + } + if changes.DNSZone != nil { + return fi.CannotChangeField("DNSZone") + } + } + + if e.Name == nil { + return fi.RequiredField("Name") + } + if e.DNSZone == nil { + return fi.RequiredField("DNSZone") + } + if e.Type == nil { + return fi.RequiredField("Type") + } + if e.Data == nil { + return fi.RequiredField("Data") + } + if e.TTL == nil { + return fi.RequiredField("TTL") + } + + return nil +} + +func (*Dns) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes *Dns) error { + cloud := t.Cloud + + if a == nil { + //return createManagedDNSRecord(context.TODO(), cloud, e) + } + + if changes.Data != nil || changes.TTL != nil || changes.Type != nil { + //return updateManagedDNSRecord(context.TODO(), cloud, e) + } + + return nil +} diff --git a/upup/pkg/fi/cloudup/elementotasks/dns_record_fitask.go b/upup/pkg/fi/cloudup/elementotasks/dns_fitask.go similarity index 100% rename from upup/pkg/fi/cloudup/elementotasks/dns_record_fitask.go rename to upup/pkg/fi/cloudup/elementotasks/dns_fitask.go diff --git a/upup/pkg/fi/cloudup/elementotasks/dns_record.go b/upup/pkg/fi/cloudup/elementotasks/dns_record.go deleted file mode 100644 index 4f1ec93ead611..0000000000000 --- a/upup/pkg/fi/cloudup/elementotasks/dns_record.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright 2026 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package elementotasks - -import ( - "context" - "fmt" - "strings" - - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/cloudup/elemento" -) - -// +kops:fitask -type DNSRecord struct { - Name *string - Lifecycle fi.Lifecycle - - // DNSZone is the parent zone, for example "example.com". - DNSZone *string - // Type is the record type, typically "A" for bootstrap. - Type *string - // Data is the current record target. During the first implementation this is - // usually a placeholder value, until the final API VIP or LB address is known. - Data *string - // TTL is the DNS TTL in seconds. - TTL *int64 - // Comment is not used by kOps itself, but helps document why this bootstrap - // record exists while the Elemento DNS API integration is being completed. - Comment *string -} - -type managedDNSRecord struct { - ID string - Name string - Zone string - Type string - Data string - TTL int64 -} - -func (r *DNSRecord) Find(c *fi.CloudupContext) (*DNSRecord, error) { - cloud := c.T.Cloud.(elemento.ElementoCloud) - - actual, err := findManagedDNSRecord(context.TODO(), cloud, r) - if err != nil { - return nil, err - } - if actual == nil { - return nil, nil - } - - matches := &DNSRecord{ - Name: r.Name, - Lifecycle: r.Lifecycle, - DNSZone: fi.PtrTo(actual.Zone), - Type: fi.PtrTo(actual.Type), - Data: fi.PtrTo(actual.Data), - TTL: fi.PtrTo(actual.TTL), - Comment: r.Comment, - } - return matches, nil -} - -func (r *DNSRecord) Run(c *fi.CloudupContext) error { - return fi.CloudupDefaultDeltaRunMethod(r, c) -} - -func (*DNSRecord) CheckChanges(a, e, changes *DNSRecord) error { - if a != nil { - if changes.Name != nil { - return fi.CannotChangeField("Name") - } - if changes.DNSZone != nil { - return fi.CannotChangeField("DNSZone") - } - } - - if e.Name == nil { - return fi.RequiredField("Name") - } - if e.DNSZone == nil { - return fi.RequiredField("DNSZone") - } - if e.Type == nil { - return fi.RequiredField("Type") - } - if e.Data == nil { - return fi.RequiredField("Data") - } - if e.TTL == nil { - return fi.RequiredField("TTL") - } - - return nil -} - -func (*DNSRecord) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes *DNSRecord) error { - cloud := t.Cloud - - if a == nil { - return createManagedDNSRecord(context.TODO(), cloud, e) - } - - if changes.Data != nil || changes.TTL != nil || changes.Type != nil { - return updateManagedDNSRecord(context.TODO(), cloud, e) - } - - return nil -} - -func fullyQualifiedRecordName(recordName string, zone string) string { - recordName = strings.TrimSuffix(recordName, ".") - zone = strings.TrimSuffix(zone, ".") - if zone == "" { - return recordName - } - if strings.HasSuffix(recordName, "."+zone) || recordName == zone { - return recordName - } - return recordName + "." + zone -} - -func findManagedDNSRecord(ctx context.Context, cloud elemento.ElementoCloud, desired *DNSRecord) (*managedDNSRecord, error) { - _ = ctx - _ = cloud - - // TODO(elemento-dns): Replace this mock with a real lookup against the Elemento DNS API. - // - // Expected implementation shape: - // 1. Resolve the Elemento zone matching fi.ValueOf(desired.DNSZone) - // 2. List records in that zone - // 3. Find the record with: - // - fullyQualifiedRecordName(fi.ValueOf(desired.Name), fi.ValueOf(desired.DNSZone)) - // - type fi.ValueOf(desired.Type) - // 4. Return its current target and TTL so kOps can decide whether it changed - return nil, nil -} - -func createManagedDNSRecord(ctx context.Context, cloud elemento.ElementoCloud, desired *DNSRecord) error { - _ = ctx - _ = cloud - - fqdn := fullyQualifiedRecordName(fi.ValueOf(desired.Name), fi.ValueOf(desired.DNSZone)) - - // TODO(elemento-dns): Replace this error with the real Elemento DNS create call. - // - // Expected create payload: - // - zone: fi.ValueOf(desired.DNSZone) - // - name: fqdn - // - type: fi.ValueOf(desired.Type) - // - value: fi.ValueOf(desired.Data) - // - ttl: fi.ValueOf(desired.TTL) - // - // Suggested first target records: - // - api.internal. - // - kops-controller.internal. - // - api. if the public API endpoint should resolve through your zone - return fmt.Errorf("Elemento DNS create is not implemented yet for %q; implement createManagedDNSRecord in elementotasks/dns_record.go", fqdn) -} - -func updateManagedDNSRecord(ctx context.Context, cloud elemento.ElementoCloud, desired *DNSRecord) error { - _ = ctx - _ = cloud - - fqdn := fullyQualifiedRecordName(fi.ValueOf(desired.Name), fi.ValueOf(desired.DNSZone)) - - // TODO(elemento-dns): Replace this error with the real Elemento DNS update call. - // - // This should update at least: - // - target/value when the bootstrap placeholder is replaced by the final VIP/LB IP - // - ttl if you choose to lower it for bootstrap and raise it later - return fmt.Errorf("Elemento DNS update is not implemented yet for %q; implement updateManagedDNSRecord in elementotasks/dns_record.go", fqdn) -} diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index fbac2243406f4..71dcce3d3b0e5 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -259,7 +259,7 @@ func (c *populateClusterSpec) run(ctx context.Context, clientset simple.Clientse cluster.Spec.API.LoadBalancer.Class = kopsapi.LoadBalancerClassClassic } - if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() { + if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() && cluster.GetCloudProvider() != kopsapi.CloudProviderElemento { dns, err := cloud.DNS() if err != nil { return err From 9c766db02ab9b65ffb483bb0d6264c6e917928a1 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Mon, 4 May 2026 13:22:08 +0200 Subject: [PATCH 04/10] chore: prepared the kops structure to call ecloud-go DNS provisioning functions. --- pkg/model/elementomodel/dns.go | 73 +--- pkg/model/elementomodel/servers.go | 20 + upup/pkg/fi/cloudup/apply_cluster.go | 6 +- upup/pkg/fi/cloudup/elemento/cloud.go | 11 +- upup/pkg/fi/cloudup/elemento/dns_provider.go | 359 ++++++++++++++++++ upup/pkg/fi/cloudup/elementotasks/dns.go | 105 +++-- .../fi/cloudup/elementotasks/servergroup.go | 109 +++++- 7 files changed, 562 insertions(+), 121 deletions(-) create mode 100644 upup/pkg/fi/cloudup/elemento/dns_provider.go diff --git a/pkg/model/elementomodel/dns.go b/pkg/model/elementomodel/dns.go index 866ad37080f0c..22df1fa23e5a2 100644 --- a/pkg/model/elementomodel/dns.go +++ b/pkg/model/elementomodel/dns.go @@ -16,18 +16,7 @@ limitations under the License. package elementomodel -import ( - "strings" - - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/cloudup/elementotasks" -) - -const ( - placeholderIP = "203.0.113.123" - kopsControllerInternalRecordPrefix = "kops-controller.internal." - defaultTTL = int64(60) -) +import "k8s.io/kops/upup/pkg/fi" // DNSModelBuilder is the provider-native integration point for Elemento-managed // DNS records that must exist before nodeup starts. @@ -39,63 +28,7 @@ type DNSModelBuilder struct { var _ fi.CloudupModelBuilder = &DNSModelBuilder{} func (b *DNSModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { - if !b.Cluster.PublishesDNSRecords() { - return nil - } - - // This builder mirrors the role of other provider-specific DNS builders in kOps: - // it declares the DNS records that must exist before nodeup starts. The actual - // Elemento DNS API calls are delegated to elementotasks.DNSRecord. - - if !b.UseLoadBalancerForAPI() { - recordName := trimZoneSuffix(b.Cluster.Spec.API.PublicName, b.Cluster.Spec.DNSZone) - c.AddTask(&elementotasks.DNSRecord{ - Name: fi.PtrTo(recordName), - DNSZone: fi.PtrTo(b.Cluster.Spec.DNSZone), - Type: fi.PtrTo("A"), - Data: fi.PtrTo(placeholderIP), - TTL: fi.PtrTo(defaultTTL), - Lifecycle: b.Lifecycle, - Comment: fi.PtrTo( - "Bootstrap record for the public Kubernetes API endpoint. " + - "Replace the placeholder target with the final Elemento-managed VIP or public API address.", - ), - }) - } - - if !b.UseLoadBalancerForInternalAPI() { - recordName := trimZoneSuffix(b.Cluster.APIInternalName(), b.Cluster.Spec.DNSZone) - c.AddTask(&elementotasks.DNSRecord{ - Name: fi.PtrTo(recordName), - DNSZone: fi.PtrTo(b.Cluster.Spec.DNSZone), - Type: fi.PtrTo("A"), - Data: fi.PtrTo(placeholderIP), - TTL: fi.PtrTo(defaultTTL), - Lifecycle: b.Lifecycle, - Comment: fi.PtrTo( - "Bootstrap record for api.internal. This must resolve before kubeconfig, " + - "service-account issuer discovery, and early control-plane traffic start using it.", - ), - }) - } - - recordName := kopsControllerInternalRecordPrefix + strings.TrimSuffix(b.Cluster.ObjectMeta.Name, "."+b.Cluster.Spec.DNSZone) - c.AddTask(&elementotasks.DNSRecord{ - Name: fi.PtrTo(recordName), - DNSZone: fi.PtrTo(b.Cluster.Spec.DNSZone), - Type: fi.PtrTo("A"), - Data: fi.PtrTo(placeholderIP), - TTL: fi.PtrTo(defaultTTL), - Lifecycle: b.Lifecycle, - Comment: fi.PtrTo( - "Bootstrap record for kops-controller.internal. Worker nodeup may use this very early " + - "to fetch configuration from the config server.", - ), - }) - + // Elemento DNS is currently create-only in the SDK, so API and node records + // are created from ServerGroup.RenderElemento once real server IPs are known. return nil } - -func trimZoneSuffix(name string, zone string) string { - return strings.TrimSuffix(name, "."+zone) -} diff --git a/pkg/model/elementomodel/servers.go b/pkg/model/elementomodel/servers.go index 0a4d0cf90f832..1596b6b5b6d05 100644 --- a/pkg/model/elementomodel/servers.go +++ b/pkg/model/elementomodel/servers.go @@ -63,6 +63,9 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error if err != nil { return err } + labels[elemento.TagKubernetesClusterName] = b.ClusterName() + labels[elemento.TagKubernetesInstanceGroup] = ig.Name + labels[elemento.TagKubernetesInstanceRole] = string(ig.Spec.Role) userData, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig) if err != nil { @@ -109,6 +112,23 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error Labels: labels, RootVolumeSize: rootVolumeSize, } + if b.Cluster.PublishesDNSRecords() { + serverGroup.ClusterName = fi.PtrTo(b.ClusterName()) + serverGroup.DNSZone = fi.PtrTo(b.ClusterName()) + if ig.HasAPIServer() { + if !b.UseLoadBalancerForAPI() { + apiPublicName := b.Cluster.Spec.API.PublicName + if apiPublicName == "" { + apiPublicName = "api." + b.ClusterName() + } + serverGroup.APIPublicName = fi.PtrTo(apiPublicName) + } + if !b.UseLoadBalancerForInternalAPI() { + serverGroup.APIInternalName = fi.PtrTo(b.Cluster.APIInternalName()) + } + serverGroup.KopsControllerInternalName = fi.PtrTo("kops-controller.internal." + b.ClusterName()) + } + } c.AddTask(&serverGroup) } diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 9570b34892d0e..648b062df7bd8 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -303,7 +303,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { if cluster.Spec.KubernetesVersion == "" { return nil, fmt.Errorf("KubernetesVersion not set") } - if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() { + if cluster.Spec.DNSZone == "" && cluster.PublishesDNSRecords() && cluster.GetCloudProvider() != kops.CloudProviderElemento { return nil, fmt.Errorf("DNSZone not set") } @@ -515,7 +515,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { modelContext.SSHPublicKeys = sshPublicKeys modelContext.Region = cloud.Region() - if cluster.PublishesDNSRecords() { + if cluster.PublishesDNSRecords() && cluster.GetCloudProvider() != kops.CloudProviderElemento { err = validateDNS(cluster, cloud) if err != nil { return nil, err @@ -838,7 +838,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { return nil, fmt.Errorf("error running tasks: %v", err) } - if !cluster.PublishesDNSRecords() { + if !cluster.PublishesDNSRecords() || cluster.GetCloudProvider() == kops.CloudProviderElemento { shouldPrecreateDNS = false } diff --git a/upup/pkg/fi/cloudup/elemento/cloud.go b/upup/pkg/fi/cloudup/elemento/cloud.go index e0569945d8492..794baa6939931 100644 --- a/upup/pkg/fi/cloudup/elemento/cloud.go +++ b/upup/pkg/fi/cloudup/elemento/cloud.go @@ -24,6 +24,7 @@ import ( "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + "k8s.io/kops/dnsprovider/pkg/dnsprovider" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/cloudinstances" "k8s.io/kops/upup/pkg/fi" @@ -50,9 +51,6 @@ type ElementoCloud interface { VolumeClient() ecloud.VolumeClient NodeupClient(ctx context.Context) ecloud.NodeupClient - // Add DNS-zone / DNS-record client accessors here once - // the Elemento SDK exposes them. The provider-native DNS tasks in - // elementotasks/dns_record.go are the intended callers. DnsClient() ecloud.DnsClient } @@ -151,8 +149,11 @@ func findServerGroups(c *elementoCloudImplementation, clusterName string) (map[s return serverGroups, nil } -func (c *elementoCloudImplementation) DNS() ecloud.DnsClient { - // Elemento DNS is expected to be managed through provider-native cloudup tasks, +func (c *elementoCloudImplementation) DNS() (dnsprovider.Interface, error) { + return NewDNSProvider(c.Client.Dns, "") +} + +func (c *elementoCloudImplementation) DnsClient() ecloud.DnsClient { return c.Client.Dns } diff --git a/upup/pkg/fi/cloudup/elemento/dns_provider.go b/upup/pkg/fi/cloudup/elemento/dns_provider.go new file mode 100644 index 0000000000000..3b32406c6776a --- /dev/null +++ b/upup/pkg/fi/cloudup/elemento/dns_provider.go @@ -0,0 +1,359 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elemento + +import ( + "context" + "fmt" + "strings" + + "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" + "k8s.io/kops/dnsprovider/pkg/dnsprovider" + "k8s.io/kops/dnsprovider/pkg/dnsprovider/rrstype" +) + +type dnsClient interface { + Create(ctx context.Context, zoneName string) (*ecloud.Dns, *ecloud.Response, error) + AddDnsRecord(ctx context.Context, zoneName, recordName, recordValue string) (*ecloud.DnsRecord, *ecloud.Response, error) + Get(ctx context.Context, zoneName string) (*ecloud.Dns, *ecloud.Response, error) + ListDnsRecords(ctx context.Context, zoneName string) ([]*ecloud.DnsRecord, *ecloud.Response, error) + GetDnsRecord(ctx context.Context, zoneName, recordName, recordType string) (*ecloud.DnsRecord, *ecloud.Response, error) +} + +type dnsListClient interface { + List(ctx context.Context) ([]*ecloud.Dns, *ecloud.Response, error) +} + +// NewDNSProvider adapts the Elemento DNS SDK to kOps' generic dnsprovider +// interface. Deletion is intentionally unsupported until the SDK exposes a +// delete operation; Apply returns an explicit error for removals. +func NewDNSProvider(client ecloud.DnsClient, defaultZoneName string) (dnsprovider.Interface, error) { + dnsClient, ok := any(&client).(dnsClient) + if !ok { + return nil, fmt.Errorf("Elemento DNS SDK does not implement the read methods required by the kOps DNS provider") + } + + return &dnsProvider{ + client: dnsClient, + defaultZoneName: defaultZoneName, + }, nil +} + +type dnsProvider struct { + client dnsClient + defaultZoneName string +} + +var _ dnsprovider.Interface = &dnsProvider{} + +func (p *dnsProvider) Zones() (dnsprovider.Zones, bool) { + return &dnsZones{ + client: p.client, + defaultZoneName: p.defaultZoneName, + }, true +} + +type dnsZones struct { + client dnsClient + defaultZoneName string +} + +var _ dnsprovider.Zones = &dnsZones{} + +func (z *dnsZones) List() ([]dnsprovider.Zone, error) { + if listClient, ok := z.client.(dnsListClient); ok { + dnsServices, _, err := listClient.List(context.TODO()) + if err != nil { + return nil, fmt.Errorf("listing Elemento DNS zones: %w", err) + } + return zonesFromElementoDNS(z.client, dnsServices), nil + } + + if z.defaultZoneName == "" { + return nil, fmt.Errorf("listing Elemento DNS zones requires DnsClient.List or a default zone name") + } + + dnsService, _, err := z.client.Get(context.TODO(), z.defaultZoneName) + if err != nil { + return nil, fmt.Errorf("getting Elemento DNS zone %q: %w", z.defaultZoneName, err) + } + if dnsService == nil { + return nil, nil + } + + return zonesFromElementoDNS(z.client, []*ecloud.Dns{dnsService}), nil +} + +func (z *dnsZones) Add(newZone dnsprovider.Zone) (dnsprovider.Zone, error) { + dnsService, _, err := z.client.Create(context.TODO(), newZone.Name()) + if err != nil && !IsDNSAlreadyExists(err) { + return nil, fmt.Errorf("creating Elemento DNS zone %q: %w", newZone.Name(), err) + } + if dnsService == nil { + dnsService, _, err = z.client.Get(context.TODO(), newZone.Name()) + if err != nil { + return nil, fmt.Errorf("getting Elemento DNS zone %q after create: %w", newZone.Name(), err) + } + } + + return zoneFromElementoDNS(z.client, newZone.Name(), dnsService), nil +} + +func (z *dnsZones) Remove(zone dnsprovider.Zone) error { + return fmt.Errorf("deleting Elemento DNS zones is not supported yet") +} + +func (z *dnsZones) New(name string) (dnsprovider.Zone, error) { + return &dnsZone{ + client: z.client, + name: name, + id: name, + }, nil +} + +func zonesFromElementoDNS(client dnsClient, dnsServices []*ecloud.Dns) []dnsprovider.Zone { + var zones []dnsprovider.Zone + for _, dnsService := range dnsServices { + if dnsService == nil { + continue + } + zones = append(zones, zoneFromElementoDNS(client, dnsService.ZoneName, dnsService)) + } + return zones +} + +func zoneFromElementoDNS(client dnsClient, name string, dnsService *ecloud.Dns) dnsprovider.Zone { + id := name + if dnsService != nil { + if dnsService.ZoneName != "" { + name = dnsService.ZoneName + } + if dnsService.ID != "" { + id = dnsService.ID + } + } + + return &dnsZone{ + client: client, + name: name, + id: id, + } +} + +type dnsZone struct { + client dnsClient + name string + id string +} + +var _ dnsprovider.Zone = &dnsZone{} + +func (z *dnsZone) Name() string { + return z.name +} + +func (z *dnsZone) ID() string { + return z.id +} + +func (z *dnsZone) ResourceRecordSets() (dnsprovider.ResourceRecordSets, bool) { + return &dnsResourceRecordSets{ + client: z.client, + zone: z, + }, true +} + +type dnsResourceRecordSets struct { + client dnsClient + zone *dnsZone +} + +var _ dnsprovider.ResourceRecordSets = &dnsResourceRecordSets{} + +func (r *dnsResourceRecordSets) List() ([]dnsprovider.ResourceRecordSet, error) { + records, _, err := r.client.ListDnsRecords(context.TODO(), r.zone.Name()) + if err != nil { + return nil, fmt.Errorf("listing Elemento DNS records in zone %q: %w", r.zone.Name(), err) + } + + var rrsets []dnsprovider.ResourceRecordSet + for _, record := range records { + if record == nil { + continue + } + rrsets = append(rrsets, rrsetFromElementoRecord(record)) + } + + return rrsets, nil +} + +func (r *dnsResourceRecordSets) Get(name string) ([]dnsprovider.ResourceRecordSet, error) { + recordSets, err := r.List() + if err != nil { + return nil, err + } + + var matches []dnsprovider.ResourceRecordSet + for _, recordSet := range recordSets { + if recordSet.Name() == name { + matches = append(matches, recordSet) + } + } + return matches, nil +} + +func (r *dnsResourceRecordSets) New(name string, rrdatas []string, ttl int64, recordType rrstype.RrsType) dnsprovider.ResourceRecordSet { + if len(rrdatas) == 0 { + return nil + } + return &dnsResourceRecordSet{ + name: name, + rrdatas: rrdatas, + ttl: ttl, + recordType: recordType, + } +} + +func (r *dnsResourceRecordSets) StartChangeset() dnsprovider.ResourceRecordChangeset { + return &dnsResourceRecordChangeset{ + client: r.client, + rrsets: r, + zoneName: r.zone.Name(), + } +} + +func (r *dnsResourceRecordSets) Zone() dnsprovider.Zone { + return r.zone +} + +func rrsetFromElementoRecord(record *ecloud.DnsRecord) dnsprovider.ResourceRecordSet { + return &dnsResourceRecordSet{ + name: record.Name, + rrdatas: []string{record.Value}, + ttl: int64(record.TTL), + recordType: rrstype.RrsType(record.Type), + } +} + +type dnsResourceRecordSet struct { + name string + rrdatas []string + ttl int64 + recordType rrstype.RrsType +} + +var _ dnsprovider.ResourceRecordSet = &dnsResourceRecordSet{} + +func (r *dnsResourceRecordSet) Name() string { + return r.name +} + +func (r *dnsResourceRecordSet) Rrdatas() []string { + return r.rrdatas +} + +func (r *dnsResourceRecordSet) Ttl() int64 { + return r.ttl +} + +func (r *dnsResourceRecordSet) Type() rrstype.RrsType { + return r.recordType +} + +type dnsResourceRecordChangeset struct { + client dnsClient + rrsets dnsprovider.ResourceRecordSets + zoneName string + + additions []dnsprovider.ResourceRecordSet + removals []dnsprovider.ResourceRecordSet + upserts []dnsprovider.ResourceRecordSet +} + +var _ dnsprovider.ResourceRecordChangeset = &dnsResourceRecordChangeset{} + +func (c *dnsResourceRecordChangeset) Add(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset { + c.additions = append(c.additions, rrset) + return c +} + +func (c *dnsResourceRecordChangeset) Remove(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset { + c.removals = append(c.removals, rrset) + return c +} + +func (c *dnsResourceRecordChangeset) Upsert(rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordChangeset { + c.upserts = append(c.upserts, rrset) + return c +} + +func (c *dnsResourceRecordChangeset) Apply(ctx context.Context) error { + if c.IsEmpty() { + return nil + } + if len(c.removals) != 0 { + return fmt.Errorf("deleting Elemento DNS records is not supported yet") + } + + if _, _, err := c.client.Create(ctx, c.zoneName); err != nil && !IsDNSAlreadyExists(err) { + return fmt.Errorf("creating Elemento DNS zone %q: %w", c.zoneName, err) + } + + for _, rrset := range append(c.additions, c.upserts...) { + if rrset == nil { + continue + } + if rrset.Type() != rrstype.A { + return fmt.Errorf("Elemento DNS currently supports only A records, got %q for %q", rrset.Type(), rrset.Name()) + } + + recordName := trimZoneSuffix(rrset.Name(), c.zoneName) + for _, rrdata := range rrset.Rrdatas() { + if _, _, err := c.client.AddDnsRecord(ctx, c.zoneName, recordName, rrdata); err != nil { + return fmt.Errorf("ensuring Elemento DNS record %q in zone %q: %w", recordName, c.zoneName, err) + } + } + } + + return nil +} + +func (c *dnsResourceRecordChangeset) IsEmpty() bool { + return len(c.additions) == 0 && len(c.upserts) == 0 && len(c.removals) == 0 +} + +func (c *dnsResourceRecordChangeset) ResourceRecordSets() dnsprovider.ResourceRecordSets { + return c.rrsets +} + +// IsDNSAlreadyExists reports whether an Elemento DNS create/upsert operation +// found an existing resource that is already compatible with the request. +func IsDNSAlreadyExists(err error) bool { + if ecloud.IsError(err, ecloud.ErrorCodeUniquenessError, ecloud.ErrorCodeConflict) { + return true + } + + message := strings.ToLower(err.Error()) + return strings.Contains(message, "already exists") || + strings.Contains(message, "already defined") || + strings.Contains(message, "uniqueness") +} + +func trimZoneSuffix(name, zone string) string { + zone = strings.TrimSuffix(zone, ".") + return strings.TrimSuffix(name, "."+zone) +} diff --git a/upup/pkg/fi/cloudup/elementotasks/dns.go b/upup/pkg/fi/cloudup/elementotasks/dns.go index b4954ec15959e..59b3531e16d62 100644 --- a/upup/pkg/fi/cloudup/elementotasks/dns.go +++ b/upup/pkg/fi/cloudup/elementotasks/dns.go @@ -17,77 +17,98 @@ limitations under the License. package elementotasks import ( - "time" + "context" + "fmt" + "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" + "k8s.io/klog/v2" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/elemento" ) -type Dns struct { - ID string - ZoneName *string - Created time.Time - AtomOsTarget string - Status string - Records []DnsRecord +// +kops:fitask +type DNSRecord struct { + Name *string + Data *string + DNSZone *string + Type *string + TTL *int64 + Lifecycle fi.Lifecycle + Comment *string } -type DnsRecord struct { - Name string - Type string - Value string - TTL int -} - -func (d *Dns) Find(c *fi.CloudupContext) (*Dns, error) { - cloud := c.T.Cloud.(elemento.ElementoCloud) - client := cloud.DnsClient() +var _ fi.CloudupTask = &DNSRecord{} +func (d *DNSRecord) Find(c *fi.CloudupContext) (*DNSRecord, error) { + // The Elemento SDK currently exposes create-only DNS methods. We therefore + // reconcile DNS by issuing create calls and tolerating "already exists" errors. + return nil, nil } -func (d *Dns) Run(c *fi.CloudupContext) error { +func (d *DNSRecord) Run(c *fi.CloudupContext) error { return fi.CloudupDefaultDeltaRunMethod(d, c) } -func (*Dns) CheckChanges(a, e, changes *Dns) error { - if a != nil { - if changes.Name != nil { - return fi.CannotChangeField("Name") - } - if changes.DNSZone != nil { - return fi.CannotChangeField("DNSZone") - } - } - - if e.Name == nil { +func (_ *DNSRecord) CheckChanges(actual, expected, changes *DNSRecord) error { + if expected.Name == nil { return fi.RequiredField("Name") } - if e.DNSZone == nil { + if expected.DNSZone == nil { return fi.RequiredField("DNSZone") } - if e.Type == nil { + if expected.Type == nil { return fi.RequiredField("Type") } - if e.Data == nil { - return fi.RequiredField("Data") + if fi.ValueOf(expected.Type) != "A" { + return fmt.Errorf("Elemento DNS currently supports only A records, got %q", fi.ValueOf(expected.Type)) } - if e.TTL == nil { - return fi.RequiredField("TTL") + if expected.Data == nil { + return fi.RequiredField("Data") } return nil } -func (*Dns) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes *Dns) error { - cloud := t.Cloud +func (_ *DNSRecord) RenderElemento(t *elemento.ElementoAPITarget, actual, expected, changes *DNSRecord) error { + client := t.Cloud.DnsClient() + zoneName := fi.ValueOf(expected.DNSZone) + recordName := fi.ValueOf(expected.Name) + recordValue := fi.ValueOf(expected.Data) - if a == nil { - //return createManagedDNSRecord(context.TODO(), cloud, e) + if err := ensureElementoDNSZone(context.TODO(), client, zoneName); err != nil { + return err + } + if err := ensureElementoDNSRecord(context.TODO(), client, zoneName, recordName, recordValue); err != nil { + return err } - if changes.Data != nil || changes.TTL != nil || changes.Type != nil { - //return updateManagedDNSRecord(context.TODO(), cloud, e) + return nil +} + +func ensureElementoDNSZone(ctx context.Context, client ecloud.DnsClient, zoneName string) error { + _, _, err := client.Create(ctx, zoneName) + if err != nil { + if elemento.IsDNSAlreadyExists(err) { + klog.V(2).Infof("Elemento DNS zone %q already exists", zoneName) + return nil + } + return fmt.Errorf("creating Elemento DNS zone %q: %w", zoneName, err) + } + + klog.V(2).Infof("Created Elemento DNS zone %q", zoneName) + return nil +} + +func ensureElementoDNSRecord(ctx context.Context, client ecloud.DnsClient, zoneName, recordName, recordValue string) error { + record, _, err := client.AddDnsRecord(ctx, zoneName, recordName, recordValue) + if err != nil { + if elemento.IsDNSAlreadyExists(err) { + klog.V(2).Infof("Elemento DNS record %q in zone %q already exists", recordName, zoneName) + return nil + } + return fmt.Errorf("creating Elemento DNS record %q in zone %q: %w", recordName, zoneName, err) } + klog.V(2).Infof("Created Elemento DNS record %q in zone %q as %q", recordName, zoneName, record.Name) return nil } diff --git a/upup/pkg/fi/cloudup/elementotasks/servergroup.go b/upup/pkg/fi/cloudup/elementotasks/servergroup.go index 62fe441cbc81b..45b70230f0260 100644 --- a/upup/pkg/fi/cloudup/elementotasks/servergroup.go +++ b/upup/pkg/fi/cloudup/elementotasks/servergroup.go @@ -52,6 +52,13 @@ type ServerGroup struct { Labels map[string]string + ClusterName *string + DNSZone *string + + APIPublicName *string + APIInternalName *string + KopsControllerInternalName *string + // RootVolumeSize is the size of the root volume in GB RootVolumeSize *int32 } @@ -286,17 +293,117 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes name, e.Location, e.Size, e.Image) fmt.Printf("EKOPS: Calling client.Create() for server %q\n", name) - _, _, err = client.Create(context.TODO(), opts) + result, _, err := client.Create(context.TODO(), opts) if err != nil { fmt.Printf("EKOPS: ERROR creating server %q: %v\n", name, err) return err } fmt.Printf("EKOPS: Successfully created server %q\n", name) + + if e.ClusterName != nil && e.DNSZone != nil { + if err := createElementoServerDNSRecord(context.TODO(), t.Cloud.DnsClient(), fi.ValueOf(e.ClusterName), fi.ValueOf(e.DNSZone), name, result.Server); err != nil { + return err + } + if err := createElementoControlPlaneDNSRecords(context.TODO(), t.Cloud.DnsClient(), fi.ValueOf(e.DNSZone), e, result.Server); err != nil { + return err + } + } } return nil } +func createElementoServerDNSRecord(ctx context.Context, client ecloud.DnsClient, clusterName, zoneName, serverName string, server *ecloud.Server) error { + recordValue := serverDNSAddress(server) + if recordValue == "" { + klog.V(2).Infof("Skipping Elemento DNS record for server %q because it has no IP address yet", serverName) + return nil + } + + recordName := trimElementoDNSZoneSuffix(fmt.Sprintf("%s.%s", serverName, clusterName), zoneName) + if err := ensureElementoDNSZone(ctx, client, zoneName); err != nil { + return err + } + if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, recordValue); err != nil { + return err + } + + return nil +} + +func createElementoControlPlaneDNSRecords(ctx context.Context, client ecloud.DnsClient, zoneName string, serverGroup *ServerGroup, server *ecloud.Server) error { + if serverGroup.APIPublicName == nil && serverGroup.APIInternalName == nil && serverGroup.KopsControllerInternalName == nil { + return nil + } + + if err := ensureElementoDNSZone(ctx, client, zoneName); err != nil { + return err + } + + publicAddress := serverPublicDNSAddress(server) + internalAddress := serverPrivateDNSAddress(server) + if publicAddress == "" { + publicAddress = internalAddress + } + if internalAddress == "" { + internalAddress = publicAddress + } + + if serverGroup.APIPublicName != nil && publicAddress != "" { + recordName := trimElementoDNSZoneSuffix(fi.ValueOf(serverGroup.APIPublicName), zoneName) + if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, publicAddress); err != nil { + return err + } + } + if serverGroup.APIInternalName != nil && internalAddress != "" { + recordName := trimElementoDNSZoneSuffix(fi.ValueOf(serverGroup.APIInternalName), zoneName) + if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, internalAddress); err != nil { + return err + } + } + if serverGroup.KopsControllerInternalName != nil && internalAddress != "" { + recordName := trimElementoDNSZoneSuffix(fi.ValueOf(serverGroup.KopsControllerInternalName), zoneName) + if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, internalAddress); err != nil { + return err + } + } + + return nil +} + +func serverPrivateDNSAddress(server *ecloud.Server) string { + if server == nil { + return "" + } + for _, privateNet := range server.PrivateNet { + if privateNet.IP != nil { + return privateNet.IP.String() + } + } + return "" +} + +func serverPublicDNSAddress(server *ecloud.Server) string { + if server == nil { + return "" + } + if server.PublicNet.IPv4 != "" { + return server.PublicNet.IPv4 + } + return server.PublicNet.IPv6 +} + +func serverDNSAddress(server *ecloud.Server) string { + if address := serverPrivateDNSAddress(server); address != "" { + return address + } + return serverPublicDNSAddress(server) +} + +func trimElementoDNSZoneSuffix(name, zone string) string { + return strings.TrimSuffix(name, "."+strings.TrimSuffix(zone, ".")) +} + func safeBytesHash(data []byte) string { // Calculate the SHA256 checksum of the data sum256 := sha256.Sum256(data) From 85807ea52791989e2ad4a6eb073eb8084b251422 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Mon, 4 May 2026 21:55:18 +0200 Subject: [PATCH 05/10] feat: integrating the new ecloud-go bootstrapping phase in kops. --- upup/pkg/fi/cloudup/elemento/dns_provider.go | 25 ++++++++++++------- upup/pkg/fi/cloudup/elementotasks/dns.go | 26 +++++++++++++++++--- upup/pkg/fi/executor.go | 7 +++--- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/upup/pkg/fi/cloudup/elemento/dns_provider.go b/upup/pkg/fi/cloudup/elemento/dns_provider.go index 3b32406c6776a..b078c2f74e993 100644 --- a/upup/pkg/fi/cloudup/elemento/dns_provider.go +++ b/upup/pkg/fi/cloudup/elemento/dns_provider.go @@ -202,18 +202,19 @@ func (r *dnsResourceRecordSets) List() ([]dnsprovider.ResourceRecordSet, error) } func (r *dnsResourceRecordSets) Get(name string) ([]dnsprovider.ResourceRecordSet, error) { - recordSets, err := r.List() + recordName := trimZoneSuffix(name, r.zone.Name()) + record, _, err := r.client.GetDnsRecord(context.TODO(), r.zone.Name(), recordName, string(rrstype.A)) if err != nil { - return nil, err - } - - var matches []dnsprovider.ResourceRecordSet - for _, recordSet := range recordSets { - if recordSet.Name() == name { - matches = append(matches, recordSet) + if IsDNSMissing(err) { + return nil, nil } + return nil, fmt.Errorf("getting Elemento DNS record %q in zone %q: %w", recordName, r.zone.Name(), err) + } + if record == nil { + return nil, nil } - return matches, nil + + return []dnsprovider.ResourceRecordSet{rrsetFromElementoRecord(record)}, nil } func (r *dnsResourceRecordSets) New(name string, rrdatas []string, ttl int64, recordType rrstype.RrsType) dnsprovider.ResourceRecordSet { @@ -353,6 +354,12 @@ func IsDNSAlreadyExists(err error) bool { strings.Contains(message, "uniqueness") } +// IsDNSMissing reports whether an Elemento DNS read operation did not find the +// requested zone or record. +func IsDNSMissing(err error) bool { + return ecloud.IsError(err, ecloud.ErrorCodeNotFound, ecloud.ErrorCodeDNSZoneNotFound) +} + func trimZoneSuffix(name, zone string) string { zone = strings.TrimSuffix(zone, ".") return strings.TrimSuffix(name, "."+zone) diff --git a/upup/pkg/fi/cloudup/elementotasks/dns.go b/upup/pkg/fi/cloudup/elementotasks/dns.go index 59b3531e16d62..24b43ed67228e 100644 --- a/upup/pkg/fi/cloudup/elementotasks/dns.go +++ b/upup/pkg/fi/cloudup/elementotasks/dns.go @@ -40,9 +40,29 @@ type DNSRecord struct { var _ fi.CloudupTask = &DNSRecord{} func (d *DNSRecord) Find(c *fi.CloudupContext) (*DNSRecord, error) { - // The Elemento SDK currently exposes create-only DNS methods. We therefore - // reconcile DNS by issuing create calls and tolerating "already exists" errors. - return nil, nil + cloud := c.T.Cloud.(elemento.ElementoCloud) + client := cloud.DnsClient() + + record, _, err := client.GetDnsRecord(context.TODO(), fi.ValueOf(d.DNSZone), fi.ValueOf(d.Name), fi.ValueOf(d.Type)) + if err != nil { + if elemento.IsDNSMissing(err) { + return nil, nil + } + return nil, fmt.Errorf("getting Elemento DNS record %q in zone %q: %w", fi.ValueOf(d.Name), fi.ValueOf(d.DNSZone), err) + } + if record == nil { + return nil, nil + } + + return &DNSRecord{ + Name: fi.PtrTo(record.Name), + Data: fi.PtrTo(record.Value), + DNSZone: d.DNSZone, + Type: fi.PtrTo(record.Type), + TTL: fi.PtrTo(int64(record.TTL)), + Lifecycle: d.Lifecycle, + Comment: d.Comment, + }, nil } func (d *DNSRecord) Run(c *fi.CloudupContext) error { diff --git a/upup/pkg/fi/executor.go b/upup/pkg/fi/executor.go index ccd71a292f0ba..a72a68671e133 100644 --- a/upup/pkg/fi/executor.go +++ b/upup/pkg/fi/executor.go @@ -204,9 +204,10 @@ func (e *executor[T]) RunTasks(ctx context.Context, taskMap map[string]Task[T]) } // Execute final ecloud task after all other tasks are completed - if err := e.executeFinalECloudTask(ctx); err != nil { - return fmt.Errorf("failed to execute final ecloud task: %v", err) - } + /* + if err := e.executeFinalECloudTask(ctx); err != nil { + return fmt.Errorf("failed to execute final ecloud task: %v", err) + }*/ return nil } From b56ea755f29f9f9191c1d255c26bd57a19afcc20 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Wed, 6 May 2026 16:23:20 +0200 Subject: [PATCH 06/10] chore: added temporary Elemento bootstrap support. --- pkg/apis/kops/model/features.go | 2 - pkg/bootstrap/pkibootstrap/pkiverifier.go | 23 +++++- pkg/commands/toolbox_enroll.go | 17 ++++- pkg/model/elementomodel/servers.go | 10 +++ upup/pkg/fi/cloudup/apply_cluster.go | 6 +- upup/pkg/fi/cloudup/elemento/dns_provider.go | 9 ++- upup/pkg/fi/cloudup/elementotasks/dns.go | 74 +++++++++++++++++++ .../fi/cloudup/elementotasks/servergroup.go | 7 +- 8 files changed, 132 insertions(+), 16 deletions(-) diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go index 2fe3933c5cf21..0062a96462d70 100644 --- a/pkg/apis/kops/model/features.go +++ b/pkg/apis/kops/model/features.go @@ -31,8 +31,6 @@ func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool { return true case kops.CloudProviderAzure: return true - case kops.CloudProviderElemento: - return true default: return false } diff --git a/pkg/bootstrap/pkibootstrap/pkiverifier.go b/pkg/bootstrap/pkibootstrap/pkiverifier.go index 818bd449a7785..9a0a2819c113f 100644 --- a/pkg/bootstrap/pkibootstrap/pkiverifier.go +++ b/pkg/bootstrap/pkibootstrap/pkiverifier.go @@ -26,6 +26,7 @@ import ( "encoding/json" "fmt" "math" + "net" "net/http" "strings" "time" @@ -128,11 +129,25 @@ func (v *verifier) VerifyToken(ctx context.Context, rawRequest *http.Request, au */ // DISABLED: Return a dummy successful verification result + nodeName := "nodes-europe-1" + certificateNames := []string{nodeName} + if host, _, err := net.SplitHostPort(rawRequest.RemoteAddr); err == nil { + certificateNames = append(certificateNames, host) + switch host { + case "192.168.100.10": + nodeName = "control-plane-europe-1" + case "192.168.100.11": + nodeName = "nodes-europe-1" + case "192.168.100.12": + nodeName = "nodes-europe-2" + } + certificateNames[0] = nodeName + } result := &bootstrap.VerifyResult{ - NodeName: "test-node", - CertificateNames: []string{"127.0.0.1"}, - ChallengeEndpoint: "127.0.0.1:10000", - InstanceGroupName: "nodes", + NodeName: nodeName, + CertificateNames: certificateNames, + ChallengeEndpoint: "", + InstanceGroupName: "nodes-europe", } return result, nil diff --git a/pkg/commands/toolbox_enroll.go b/pkg/commands/toolbox_enroll.go index 268563d6a409d..65dc3d1d33827 100644 --- a/pkg/commands/toolbox_enroll.go +++ b/pkg/commands/toolbox_enroll.go @@ -687,7 +687,22 @@ func (b *ConfigBuilder) GetWellKnownAddresses(ctx context.Context) (model.WellKn } } if len(wellKnownAddresses[wellknownservices.KubeAPIServer]) == 0 { - // TODO: Should we support DNS? + names := []string{fullCluster.APIInternalName()} + if fullCluster.Spec.API.PublicName != "" { + names = append(names, fullCluster.Spec.API.PublicName) + } + for _, name := range names { + ips, err := net.LookupIP(name) + if err != nil { + klog.Warningf("unable to resolve kube-apiserver DNS name %q: %v", name, err) + continue + } + for _, ip := range ips { + wellKnownAddresses[wellknownservices.KubeAPIServer] = append(wellKnownAddresses[wellknownservices.KubeAPIServer], ip.String()) + } + } + } + if len(wellKnownAddresses[wellknownservices.KubeAPIServer]) == 0 { return nil, fmt.Errorf("unable to determine IP address for kube-apiserver") } for k := range wellKnownAddresses { diff --git a/pkg/model/elementomodel/servers.go b/pkg/model/elementomodel/servers.go index 1596b6b5b6d05..572b54e89448f 100644 --- a/pkg/model/elementomodel/servers.go +++ b/pkg/model/elementomodel/servers.go @@ -57,6 +57,15 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error sshkeyTasks = append(sshkeyTasks, t) } + var dnsZoneTask *elementotasks.DNSZone + if b.Cluster.PublishesDNSRecords() { + dnsZoneTask = &elementotasks.DNSZone{ + Name: fi.PtrTo(b.ClusterName()), + Lifecycle: b.Lifecycle, + } + c.EnsureTask(dnsZoneTask) + } + for _, ig := range b.InstanceGroups { igSize := fi.ValueOf(ig.Spec.MinSize) labels, err := b.CloudTagsForInstanceGroup(ig) @@ -115,6 +124,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error if b.Cluster.PublishesDNSRecords() { serverGroup.ClusterName = fi.PtrTo(b.ClusterName()) serverGroup.DNSZone = fi.PtrTo(b.ClusterName()) + serverGroup.DNSZoneTask = dnsZoneTask if ig.HasAPIServer() { if !b.UseLoadBalancerForAPI() { apiPublicName := b.Cluster.Spec.API.PublicName diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 648b062df7bd8..70d843d415afe 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -496,11 +496,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) (*ApplyResults, error) { case kops.CloudProviderElemento: { - if len(sshPublicKeys) == 0 { - return nil, fmt.Errorf("SSH public key must be specified when running with Elemento (create with `kops create secret --name %s sshpublickey admin -i ~/.ssh/id_rsa.pub`)", cluster.ObjectMeta.Name) - } - - if len(sshPublicKeys) != 1 { + if len(sshPublicKeys) > 1 { return nil, fmt.Errorf("exactly one 'admin' SSH public key can be specified when running with Elemento; please delete a key using `kops delete secret`") } } diff --git a/upup/pkg/fi/cloudup/elemento/dns_provider.go b/upup/pkg/fi/cloudup/elemento/dns_provider.go index b078c2f74e993..81b393dc9e612 100644 --- a/upup/pkg/fi/cloudup/elemento/dns_provider.go +++ b/upup/pkg/fi/cloudup/elemento/dns_provider.go @@ -357,7 +357,14 @@ func IsDNSAlreadyExists(err error) bool { // IsDNSMissing reports whether an Elemento DNS read operation did not find the // requested zone or record. func IsDNSMissing(err error) bool { - return ecloud.IsError(err, ecloud.ErrorCodeNotFound, ecloud.ErrorCodeDNSZoneNotFound) + if ecloud.IsError(err, ecloud.ErrorCodeNotFound, ecloud.ErrorCodeDNSZoneNotFound) { + return true + } + + message := strings.ToLower(err.Error()) + return strings.Contains(message, "dns service not found") || + strings.Contains(message, "dns zone not found") || + strings.Contains(message, "api error: 404") } func trimZoneSuffix(name, zone string) string { diff --git a/upup/pkg/fi/cloudup/elementotasks/dns.go b/upup/pkg/fi/cloudup/elementotasks/dns.go index 24b43ed67228e..45a1f0b2cd067 100644 --- a/upup/pkg/fi/cloudup/elementotasks/dns.go +++ b/upup/pkg/fi/cloudup/elementotasks/dns.go @@ -26,6 +26,80 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/elemento" ) +// +kops:fitask +type DNSZone struct { + Name *string + Lifecycle fi.Lifecycle +} + +var _ fi.CloudupTask = &DNSZone{} +var _ fi.HasLifecycle = &DNSZone{} +var _ fi.HasName = &DNSZone{} + +func (z *DNSZone) GetLifecycle() fi.Lifecycle { + return z.Lifecycle +} + +func (z *DNSZone) SetLifecycle(lifecycle fi.Lifecycle) { + z.Lifecycle = lifecycle +} + +func (z *DNSZone) GetName() *string { + return z.Name +} + +func (z *DNSZone) String() string { + return fi.CloudupTaskAsString(z) +} + +func (z *DNSZone) Find(c *fi.CloudupContext) (*DNSZone, error) { + cloud := c.T.Cloud.(elemento.ElementoCloud) + client := cloud.DnsClient() + zoneName := fi.ValueOf(z.Name) + + fmt.Printf("EKOPS: Finding Elemento DNS zone %q\n", zoneName) + + dns, _, err := client.Get(context.TODO(), zoneName) + if err != nil { + if elemento.IsDNSMissing(err) { + fmt.Printf("EKOPS: Elemento DNS zone %q not found\n", zoneName) + return nil, nil + } + return nil, fmt.Errorf("getting Elemento DNS zone %q: %w", zoneName, err) + } + if dns == nil { + fmt.Printf("EKOPS: Elemento DNS zone %q not found\n", zoneName) + return nil, nil + } + + fmt.Printf("EKOPS: Elemento DNS zone %q already exists\n", zoneName) + return &DNSZone{ + Name: fi.PtrTo(dns.ZoneName), + Lifecycle: z.Lifecycle, + }, nil +} + +func (z *DNSZone) Run(c *fi.CloudupContext) error { + return fi.CloudupDefaultDeltaRunMethod(z, c) +} + +func (_ *DNSZone) CheckChanges(actual, expected, changes *DNSZone) error { + if expected.Name == nil { + return fi.RequiredField("Name") + } + return nil +} + +func (_ *DNSZone) RenderElemento(t *elemento.ElementoAPITarget, actual, expected, changes *DNSZone) error { + zoneName := fi.ValueOf(expected.Name) + fmt.Printf("EKOPS: Ensuring Elemento DNS zone %q\n", zoneName) + if err := ensureElementoDNSZone(context.TODO(), t.Cloud.DnsClient(), zoneName); err != nil { + return err + } + fmt.Printf("EKOPS: Elemento DNS zone %q ensured\n", zoneName) + return nil +} + // +kops:fitask type DNSRecord struct { Name *string diff --git a/upup/pkg/fi/cloudup/elementotasks/servergroup.go b/upup/pkg/fi/cloudup/elementotasks/servergroup.go index 45b70230f0260..18530a78bc79e 100644 --- a/upup/pkg/fi/cloudup/elementotasks/servergroup.go +++ b/upup/pkg/fi/cloudup/elementotasks/servergroup.go @@ -21,7 +21,6 @@ import ( "crypto/sha256" "encoding/base64" "fmt" - "math/rand" "strings" "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" @@ -54,6 +53,7 @@ type ServerGroup struct { ClusterName *string DNSZone *string + DNSZoneTask *DNSZone APIPublicName *string APIInternalName *string @@ -244,8 +244,9 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes fmt.Printf("EKOPS: UserData length: %d bytes, hash: %s\n", len(userData), userDataHash) for i := 1; i <= expectedCount-actualCount; i++ { - // Append a random/unique ID to the node name - name := fmt.Sprintf("%s-%x", fi.ValueOf(e.Name), rand.Int63()) + // Use deterministic names so manual DNS records can be created before VM creation. + ordinal := actualCount + i + name := fmt.Sprintf("%s-%d", fi.ValueOf(e.Name), ordinal) // Initialize labels if nil labels := e.Labels From 42bdf7d2df98a26cf80bfd9a96db703851165e44 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Thu, 7 May 2026 16:30:32 +0200 Subject: [PATCH 07/10] feat: integrated into kops the logic needed for bootstrapping Elemento cluster's nodes. --- cmd/kops-controller/controllers/awsipam.go | 5 +- .../controllers/node_controller.go | 119 +++++++++- cmd/kops-controller/main.go | 9 +- nodeup/pkg/model/bootstrap_client.go | 4 +- pkg/nodeidentity/elemento/identify.go | 214 ++++++++++++++++-- pkg/nodeidentity/interfaces.go | 7 +- protokube/cmd/protokube/main.go | 2 + .../k8s-1.16.yaml.template | 8 + upup/pkg/fi/cloudup/elemento/authenticator.go | 20 +- upup/pkg/fi/cloudup/elemento/verifier.go | 116 +++++----- .../amazonvpc-containerd/manifest.yaml | 2 +- .../amazonvpc/manifest.yaml | 2 +- .../awscloudcontroller/manifest.yaml | 2 +- .../awsiamauthenticator/crd/manifest.yaml | 2 +- .../mappings/manifest.yaml | 2 +- ...ops-controller.addons.k8s.io-k8s-1.16.yaml | 8 + .../cilium/manifest.yaml | 2 +- .../coredns/manifest.yaml | 2 +- .../insecure-1.19/manifest.yaml | 2 +- .../metrics-server/secure-1.19/manifest.yaml | 2 +- ...ops-controller.addons.k8s.io-k8s-1.16.yaml | 8 + .../service-account-iam/manifest.yaml | 2 +- ...ops-controller.addons.k8s.io-k8s-1.16.yaml | 8 + .../simple/manifest.yaml | 2 +- 24 files changed, 417 insertions(+), 133 deletions(-) diff --git a/cmd/kops-controller/controllers/awsipam.go b/cmd/kops-controller/controllers/awsipam.go index 75bac41391cdb..78e4212a9de67 100644 --- a/cmd/kops-controller/controllers/awsipam.go +++ b/cmd/kops-controller/controllers/awsipam.go @@ -160,8 +160,9 @@ func (r *AWSIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error { } type nodePatchSpec struct { - PodCIDR string `json:"podCIDR,omitempty"` - PodCIDRs []string `json:"podCIDRs,omitempty"` + PodCIDR string `json:"podCIDR,omitempty"` + PodCIDRs []string `json:"podCIDRs,omitempty"` + ProviderID *string `json:"providerID,omitempty"` } // patchNodePodCIDRs patches the node podCIDRs to the specified value(s). diff --git a/cmd/kops-controller/controllers/node_controller.go b/cmd/kops-controller/controllers/node_controller.go index 9e431c370d967..a096f0036584c 100644 --- a/cmd/kops-controller/controllers/node_controller.go +++ b/cmd/kops-controller/controllers/node_controller.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -68,7 +69,10 @@ type NodeReconciler struct { identifier nodeidentity.Identifier } +const externalCloudProviderTaint = "node.cloudprovider.kubernetes.io/uninitialized" + // +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=,resources=nodes/status,verbs=get;patch;update // Reconcile is the main reconciler function that observes node changes. func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = r.log.WithValues("nodecontroller", req.NamespacedName) @@ -111,16 +115,32 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } } - if len(updateLabels) == 0 && len(deleteLabels) == 0 { - klog.V(4).Infof("no label changes needed for %s", node.Name) - return ctrl.Result{}, nil + providerID := "" + if info.ProviderID != "" && node.Spec.ProviderID != info.ProviderID { + providerID = info.ProviderID } - if err := patchNodeLabels(r.coreV1Client, ctx, node, updateLabels, deleteLabels); err != nil { - klog.Warningf("failed to patch node labels on %s: %v", node.Name, err) + var taints *[]corev1.Taint + if info.Initialized { + if updatedTaints, changed := removeTaint(node.Spec.Taints, externalCloudProviderTaint); changed { + taints = &updatedTaints + } + } + + if len(updateLabels) == 0 && len(deleteLabels) == 0 && providerID == "" && taints == nil { + klog.V(4).Infof("no spec or label changes needed for %s", node.Name) + } else if err := patchNode(r.coreV1Client, ctx, node, updateLabels, deleteLabels, providerID, taints); err != nil { + klog.Warningf("failed to patch node on %s: %v", node.Name, err) return ctrl.Result{}, err } + if len(info.Addresses) != 0 && !reflect.DeepEqual(node.Status.Addresses, info.Addresses) { + if err := patchNodeStatusAddresses(r.coreV1Client, ctx, node, info.Addresses); err != nil { + klog.Warningf("failed to patch node status addresses on %s: %v", node.Name, err) + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil } @@ -142,6 +162,11 @@ type nodePatchMetadata struct { // patchNodeLabels patches the node labels to set the specified labels func patchNodeLabels(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, setLabels map[string]string, deleteLabels map[string]struct{}) error { + return patchNode(client, ctx, node, setLabels, deleteLabels, "", nil) +} + +// patchNode patches node metadata and spec fields managed by the node controller. +func patchNode(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, setLabels map[string]string, deleteLabels map[string]struct{}, providerID string, taints *[]corev1.Taint) error { nodePatchMetadata := &nodePatchMetadata{ Labels: make(map[string]*string), } @@ -153,20 +178,90 @@ func patchNodeLabels(client *corev1client.CoreV1Client, ctx context.Context, nod nodePatchMetadata.Labels[k] = nil } - nodePatch := &nodePatch{ - Metadata: nodePatchMetadata, + nodePatch := &nodePatch{} + if len(nodePatchMetadata.Labels) != 0 { + nodePatch.Metadata = nodePatchMetadata + } + if providerID != "" { + nodePatch.Spec = &nodePatchSpec{ProviderID: &providerID} + } + + if nodePatch.Metadata != nil || nodePatch.Spec != nil { + nodePatchJson, err := json.Marshal(nodePatch) + if err != nil { + return fmt.Errorf("error building node patch: %v", err) + } + + klog.V(2).Infof("sending patch for node %q: %q", node.Name, string(nodePatchJson)) + + _, err = client.Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, nodePatchJson, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("error applying patch to node: %v", err) + } + } + + if taints != nil { + if err := patchNodeTaints(client, ctx, node, *taints); err != nil { + return err + } } - nodePatchJson, err := json.Marshal(nodePatch) + + return nil +} + +func patchNodeTaints(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, taints []corev1.Taint) error { + nodePatchJson, err := json.Marshal(struct { + Spec struct { + Taints []corev1.Taint `json:"taints"` + } `json:"spec"` + }{Spec: struct { + Taints []corev1.Taint `json:"taints"` + }{Taints: taints}}) + if err != nil { + return fmt.Errorf("error building node taints patch: %v", err) + } + + klog.V(2).Infof("sending taints patch for node %q: %q", node.Name, string(nodePatchJson)) + + _, err = client.Nodes().Patch(ctx, node.Name, types.MergePatchType, nodePatchJson, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("error applying taints patch to node: %v", err) + } + + return nil +} + +func patchNodeStatusAddresses(client *corev1client.CoreV1Client, ctx context.Context, node *corev1.Node, addresses []corev1.NodeAddress) error { + nodePatchJson, err := json.Marshal(struct { + Status struct { + Addresses []corev1.NodeAddress `json:"addresses"` + } `json:"status"` + }{Status: struct { + Addresses []corev1.NodeAddress `json:"addresses"` + }{Addresses: addresses}}) if err != nil { - return fmt.Errorf("error building node patch: %v", err) + return fmt.Errorf("error building node status patch: %v", err) } - klog.V(2).Infof("sending patch for node %q: %q", node.Name, string(nodePatchJson)) + klog.V(2).Infof("sending status patch for node %q: %q", node.Name, string(nodePatchJson)) - _, err = client.Nodes().Patch(ctx, node.Name, types.StrategicMergePatchType, nodePatchJson, metav1.PatchOptions{}) + _, err = client.Nodes().Patch(ctx, node.Name, types.MergePatchType, nodePatchJson, metav1.PatchOptions{}, "status") if err != nil { - return fmt.Errorf("error applying patch to node: %v", err) + return fmt.Errorf("error applying status patch to node: %v", err) } return nil } + +func removeTaint(taints []corev1.Taint, key string) ([]corev1.Taint, bool) { + updated := make([]corev1.Taint, 0, len(taints)) + changed := false + for _, taint := range taints { + if taint.Key == key { + changed = true + continue + } + updated = append(updated, taint) + } + return updated, changed +} diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index badfea634f967..a83599eb89402 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -39,20 +39,20 @@ import ( nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentityazure "k8s.io/kops/pkg/nodeidentity/azure" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" + nodeidentityelemento "k8s.io/kops/pkg/nodeidentity/elemento" nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce" nodeidentityhetzner "k8s.io/kops/pkg/nodeidentity/hetzner" nodeidentitymetal "k8s.io/kops/pkg/nodeidentity/metal" nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack" nodeidentityscw "k8s.io/kops/pkg/nodeidentity/scaleway" - nodeidentityelemento "k8s.io/kops/pkg/nodeidentity/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/do" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" - "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/util/pkg/vfs" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -189,6 +189,7 @@ func main() { verifiers = append(verifiers, verifier) } if opt.Server.Provider.Elemento != nil { + verifier, err := elemento.NewElementoVerifier(opt.Server.Provider.Elemento) if err != nil { setupLog.Error(err, "unable to create verifier") @@ -317,9 +318,9 @@ func addNodeController(ctx context.Context, mgr manager.Manager, vfsContext *vfs if err != nil { return fmt.Errorf("error building identifier: %w", err) } - + case "elemento": - identifier, err = nodeidentityelemento.New(opt.CacheNodeidentityInfo) + identifier, err = nodeidentityelemento.New(opt.CacheNodeidentityInfo, opt.ClusterName) if err != nil { return fmt.Errorf("error building identifier: %w", err) } diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index b6a14ae9f04e2..d16e8d153b692 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -31,6 +31,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/do" + "k8s.io/kops/upup/pkg/fi/cloudup/elemento" "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" @@ -94,8 +95,7 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error { } authenticator = a case kops.CloudProviderElemento: - // Use PKI file-based authentication like Metal provider to avoid kops-controller dependency - a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem") + a, err := elemento.NewElementoAuthenticator() if err != nil { return err } diff --git a/pkg/nodeidentity/elemento/identify.go b/pkg/nodeidentity/elemento/identify.go index edc7d91095c99..2bce3b9d0019b 100644 --- a/pkg/nodeidentity/elemento/identify.go +++ b/pkg/nodeidentity/elemento/identify.go @@ -41,12 +41,44 @@ const ( // nodeIdentifier identifies a node from Elemento type nodeIdentifier struct { client *ecloud.Client + clusterName string cache expirationcache.Store cacheEnabled bool } +type staticNodeInfo struct { + InstanceID string + InternalIP string + ProviderID string + Labels map[string]string +} + +var staticNodesByName = map[string]staticNodeInfo{ + "control-plane-europe-1": { + InstanceID: "fc72216e-6fb0-4cbf-a2be-3973da79f955", + InternalIP: "192.168.100.10", + Labels: map[string]string{ + nodelabels.RoleLabelControlPlane20: "", + }, + }, + "nodes-europe-1": { + InstanceID: "e4ff7b13-51c1-48bf-9ba9-c5fb5839c358", + InternalIP: "192.168.100.11", + Labels: map[string]string{ + nodelabels.RoleLabelNode16: "", + }, + }, + "nodes-europe-2": { + InstanceID: "f1dc002b-a660-423f-8850-8b3fc28c1625", + InternalIP: "192.168.100.12", + Labels: map[string]string{ + nodelabels.RoleLabelNode16: "", + }, + }, +} + // New creates and returns a nodeidentity.Identifier for Nodes running on Elemento -func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) { +func New(CacheNodeidentityInfo bool, clusterName string) (nodeidentity.Identifier, error) { elementoClient, err := ecloud.NewClient("kops-elemento", "1.0") if err != nil { @@ -55,6 +87,7 @@ func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) { return &nodeIdentifier{ client: elementoClient, + clusterName: strings.TrimSpace(clusterName), cache: expirationcache.NewTTLStore(stringKeyFunc, cacheTTL), cacheEnabled: CacheNodeidentityInfo, }, nil @@ -62,30 +95,42 @@ func New(CacheNodeidentityInfo bool) (nodeidentity.Identifier, error) { // IdentifyNode queries Elemento for the node identity information func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (*nodeidentity.Info, error) { - providerID := node.Spec.ProviderID - if providerID == "" { - return nil, fmt.Errorf("providerID not set for node %s", node.Name) - } - if !strings.HasPrefix(providerID, "elemento://") { - return nil, fmt.Errorf("providerID %q not recognized for node %s", providerID, node.Name) + if info, ok := staticNodeIdentity(node.Name); ok { + return info, nil } - serverID := strings.TrimPrefix(providerID, "elemento://") + providerID := node.Spec.ProviderID + var serverID string + var server *ecloud.Server + if providerID != "" { + if !strings.HasPrefix(providerID, "elemento://") { + return nil, fmt.Errorf("providerID %q not recognized for node %s", providerID, node.Name) + } + serverID = strings.TrimPrefix(providerID, "elemento://") - // If cache is enabled, check if the node information is already cached - if i.cacheEnabled { - obj, exists, err := i.cache.GetByKey(serverID) + // If cache is enabled, check if the node information is already cached + if i.cacheEnabled { + obj, exists, err := i.cache.GetByKey(serverID) + if err != nil { + klog.Warningf("Nodeidentity info cache lookup failure: %v", err) + } + if exists { + return obj.(*nodeidentity.Info), nil + } + } + + var err error + server, err = i.getServer(ctx, serverID) if err != nil { - klog.Warningf("Nodeidentity info cache lookup failure: %v", err) + return nil, err } - if exists { - return obj.(*nodeidentity.Info), nil + } else { + var err error + server, err = i.getServerByNodeName(ctx, node.Name) + if err != nil { + return nil, err } - } - - server, err := i.getServer(serverID) - if err != nil { - return nil, err + serverID = server.ID } if server.Status != "running" { @@ -110,16 +155,19 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* labels[strings.TrimPrefix(key, elemento.TagKubernetesNodeLabelPrefix)] = value } } + addRoleLabelFallback(labels, server.Name) info := &nodeidentity.Info{ - InstanceID: serverID, - Labels: labels, + InstanceID: serverID, + ProviderID: "elemento://" + serverID, + Labels: labels, + Addresses: nodeAddresses(server), + Initialized: true, } // If cache is enabled, store the node information in the cache if i.cacheEnabled { - err = i.cache.Add(info) - if err != nil { + if err := i.cache.Add(info); err != nil { klog.Warningf("Failed to add node identity info to cache: %v", err) } } @@ -127,6 +175,37 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* return info, nil } +func staticNodeIdentity(nodeName string) (*nodeidentity.Info, bool) { + static, ok := staticNodesByName[nodeName] + if !ok { + return nil, false + } + providerID := static.ProviderID + if providerID == "" { + providerID = "elemento://" + static.InstanceID + } + labels := map[string]string{} + for key, value := range static.Labels { + labels[key] = value + } + return &nodeidentity.Info{ + InstanceID: static.InstanceID, + ProviderID: providerID, + Labels: labels, + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: static.InternalIP, + }, + { + Type: corev1.NodeHostName, + Address: nodeName, + }, + }, + Initialized: true, + }, true +} + // stringKeyFunc is a string as cache key function func stringKeyFunc(obj interface{}) (string, error) { key := obj.(*nodeidentity.Info).InstanceID @@ -134,11 +213,96 @@ func stringKeyFunc(obj interface{}) (string, error) { } // getServer retrieves the server information from Elemento for the given server ID -func (i *nodeIdentifier) getServer(id string) (*ecloud.Server, error) { - server, _, err := i.client.Server.GetByID(context.TODO(), id) +func (i *nodeIdentifier) getServer(ctx context.Context, id string) (*ecloud.Server, error) { + server, _, err := i.client.Server.GetByID(ctx, id) if err != nil || server == nil { return nil, fmt.Errorf("failed to get info for server %q: %w", id, err) } return server, nil } + +// getServerByNodeName retrieves the Elemento server that registered the given Kubernetes node name. +func (i *nodeIdentifier) getServerByNodeName(ctx context.Context, nodeName string) (*ecloud.Server, error) { + servers, _, err := i.client.Server.List(ctx, ecloud.ServerListOpts{Name: nodeName}) + if err != nil { + return nil, fmt.Errorf("failed to list servers for node %q: %w", nodeName, err) + } + + var matches []*ecloud.Server + var clusterMatches []*ecloud.Server + for _, server := range servers { + if server.Name != nodeName { + continue + } + matches = append(matches, server) + if i.clusterName != "" && server.Labels[elemento.TagKubernetesClusterName] == i.clusterName { + clusterMatches = append(clusterMatches, server) + } + } + + if len(clusterMatches) > 0 { + matches = clusterMatches + } + + switch len(matches) { + case 0: + if i.clusterName != "" { + return nil, fmt.Errorf("no Elemento server found for node %q in cluster %q", nodeName, i.clusterName) + } + return nil, fmt.Errorf("no Elemento server found for node %q", nodeName) + case 1: + return matches[0], nil + default: + return nil, fmt.Errorf("found multiple Elemento servers for node %q", nodeName) + } +} + +func addRoleLabelFallback(labels map[string]string, serverName string) { + for _, key := range []string{ + nodelabels.RoleLabelControlPlane20, + nodelabels.RoleLabelNode16, + nodelabels.RoleLabelAPIServer16, + } { + if _, found := labels[key]; found { + return + } + } + + switch { + case strings.HasPrefix(serverName, "control-plane-"): + labels[nodelabels.RoleLabelControlPlane20] = "" + case strings.HasPrefix(serverName, "nodes-"): + labels[nodelabels.RoleLabelNode16] = "" + } +} + +func nodeAddresses(server *ecloud.Server) []corev1.NodeAddress { + var addresses []corev1.NodeAddress + for _, privateNet := range server.PrivateNet { + if len(privateNet.IP) == 0 { + continue + } + addresses = append(addresses, corev1.NodeAddress{ + Type: corev1.NodeInternalIP, + Address: privateNet.IP.String(), + }) + break + } + + if len(addresses) == 0 && server.PublicNet.IPv4 != "" { + addresses = append(addresses, corev1.NodeAddress{ + Type: corev1.NodeInternalIP, + Address: server.PublicNet.IPv4, + }) + } + + if server.Name != "" { + addresses = append(addresses, corev1.NodeAddress{ + Type: corev1.NodeHostName, + Address: server.Name, + }) + } + + return addresses +} diff --git a/pkg/nodeidentity/interfaces.go b/pkg/nodeidentity/interfaces.go index 4236ff9978f87..2607d612604cc 100644 --- a/pkg/nodeidentity/interfaces.go +++ b/pkg/nodeidentity/interfaces.go @@ -27,8 +27,11 @@ type Identifier interface { } type Info struct { - InstanceID string - Labels map[string]string + InstanceID string + ProviderID string + Labels map[string]string + Addresses []corev1.NodeAddress + Initialized bool } type LegacyIdentifier interface { diff --git a/protokube/cmd/protokube/main.go b/protokube/cmd/protokube/main.go index 909c9c8124b65..473d33af17c88 100644 --- a/protokube/cmd/protokube/main.go +++ b/protokube/cmd/protokube/main.go @@ -149,6 +149,8 @@ func run() error { } cloudProvider = scwCloudProvider + } else if cloud == "elemento" { + cloudProvider = nil } else if cloud == "metal" { cloudProvider = nil } else { diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template index 6bd105c8e6690..d2f4053c642b0 100644 --- a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template @@ -144,6 +144,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update {{- if GossipEnabled }} - apiGroups: - "" diff --git a/upup/pkg/fi/cloudup/elemento/authenticator.go b/upup/pkg/fi/cloudup/elemento/authenticator.go index c9803fd4fdd92..2a1984bcf31e8 100644 --- a/upup/pkg/fi/cloudup/elemento/authenticator.go +++ b/upup/pkg/fi/cloudup/elemento/authenticator.go @@ -17,8 +17,8 @@ limitations under the License. package elemento import ( - // "fmt" - // "strconv" + "fmt" + "os" // "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud/metadata" "k8s.io/kops/pkg/bootstrap" @@ -36,15 +36,9 @@ func NewElementoAuthenticator() (bootstrap.Authenticator, error) { } func (h *elementoAuthenticator) CreateToken(body []byte) (string, error) { - // DISABLED: Comment out metadata check for testing - /* - serverID, err := metadata.NewClient().InstanceID() - if err != nil { - return "", fmt.Errorf("failed to retrieve server ID: %w", err) - } - return ElementoAuthenticationTokenPrefix + strconv.Itoa(serverID), nil - */ - - // DISABLED: Return a dummy token - return ElementoAuthenticationTokenPrefix + "test-server-123", nil + hostname, err := os.Hostname() + if err != nil { + return "", fmt.Errorf("getting hostname for Elemento bootstrap token: %w", err) + } + return ElementoAuthenticationTokenPrefix + hostname, nil } diff --git a/upup/pkg/fi/cloudup/elemento/verifier.go b/upup/pkg/fi/cloudup/elemento/verifier.go index af9abd844aca7..b2e320b845217 100644 --- a/upup/pkg/fi/cloudup/elemento/verifier.go +++ b/upup/pkg/fi/cloudup/elemento/verifier.go @@ -19,91 +19,83 @@ package elemento import ( "context" "fmt" - - // "net" + "net" "net/http" - // "strings" - // "strconv" + "strings" - "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" "k8s.io/kops/pkg/bootstrap" - // "k8s.io/kops/pkg/wellknownports" ) type ElementoVerifierOptions struct { } type elementoVerifier struct { - opt ElementoVerifierOptions - client *ecloud.Client + opt ElementoVerifierOptions } var _ bootstrap.Verifier = &elementoVerifier{} -func NewElementoVerifier(opt *ElementoVerifierOptions) (bootstrap.Verifier, error) { - elementoClient, err := ecloud.NewClient("kops-elemento", "1.0") - if err != nil { - return nil, fmt.Errorf("failed to get server info: %w", err) - } +type staticBootstrapNode struct { + NodeName string + InstanceGroupName string +} + +var staticBootstrapNodesByIP = map[string]staticBootstrapNode{ + "192.168.100.10": {NodeName: "control-plane-europe-1", InstanceGroupName: "control-plane-europe"}, + "192.168.100.11": {NodeName: "nodes-europe-1", InstanceGroupName: "nodes-europe"}, + "192.168.100.12": {NodeName: "nodes-europe-2", InstanceGroupName: "nodes-europe"}, +} +func NewElementoVerifier(opt *ElementoVerifierOptions) (bootstrap.Verifier, error) { return &elementoVerifier{ - opt: *opt, - client: elementoClient, + opt: *opt, }, nil } func (e elementoVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request, token string, body []byte) (*bootstrap.VerifyResult, error) { - // DISABLED: Comment out all verification checks for testing - /* - if !strings.HasPrefix(token, ElementoAuthenticationTokenPrefix) { - return nil, fmt.Errorf("invalid token format") - } - token = strings.TrimPrefix(token, ElementoAuthenticationTokenPrefix) - - server, _, err := e.client.Server.GetByID(ctx, token) - if err != nil || server == nil { - return nil, fmt.Errorf("failed to get server info: %w", err) - } + if !strings.HasPrefix(token, ElementoAuthenticationTokenPrefix) { + return nil, bootstrap.ErrNotThisVerifier + } + token = strings.TrimSpace(strings.TrimPrefix(token, ElementoAuthenticationTokenPrefix)) - var addrs []string - var challengeEndpoints []string - if server.PublicNet.IPv4 != "" { - // Don't challenge over the public network - addrs = append(addrs, server.PublicNet.IPv4) - } - for _, network := range server.PrivateNet { - if network.IP != nil { - addrs = append(addrs, network.IP.String()) - challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(network.IP.String(), strconv.Itoa(wellknownports.NodeupChallenge))) - } - } + remoteHost, _, err := net.SplitHostPort(rawRequest.RemoteAddr) + if err != nil { + remoteHost = rawRequest.RemoteAddr + } - if len(challengeEndpoints) == 0 { - return nil, fmt.Errorf("cannot determine challenge endpoint for server %q", server.ID) - } + nodeName := token + instanceGroupName := "" + if node, ok := staticBootstrapNodesByIP[remoteHost]; ok { + nodeName = node.NodeName + instanceGroupName = node.InstanceGroupName + } + if instanceGroupName == "" { + instanceGroupName = inferInstanceGroupName(nodeName) + } + if instanceGroupName == "" { + return nil, fmt.Errorf("failed to determine instance group for node %q", nodeName) + } + certificateNames := []string{nodeName} + if remoteHost != "" { + certificateNames = append(certificateNames, remoteHost) + } - result := &bootstrap.VerifyResult{ - NodeName: server.Name, - CertificateNames: addrs, - ChallengeEndpoint: challengeEndpoints[0], - } + return &bootstrap.VerifyResult{ + NodeName: nodeName, + CertificateNames: certificateNames, + InstanceGroupName: instanceGroupName, + }, nil +} - for key, value := range server.Labels { - if key == TagKubernetesInstanceGroup { - result.InstanceGroupName = value - } +func inferInstanceGroupName(serverName string) string { + i := strings.LastIndex(serverName, "-") + if i == -1 || i == len(serverName)-1 { + return serverName + } + for _, r := range serverName[i+1:] { + if r < '0' || r > '9' { + return serverName } - - return result, nil - */ - - // DISABLED: Return a dummy successful verification result - result := &bootstrap.VerifyResult{ - NodeName: "test-node", - CertificateNames: []string{"127.0.0.1"}, - ChallengeEndpoint: "127.0.0.1:10000", - InstanceGroupName: "nodes", } - - return result, nil + return serverName[:i] } diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml index 37aef13e90b3c..3c4833df70615 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml index 37aef13e90b3c..3c4833df70615 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml index dee7eca1d1e5a..ea36d775bffa2 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml index 7ccb2666b3804..9ba9ea8522bfc 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/crd/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml index b29751b1630f4..ea20e01d5099e 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/mappings/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml index 1a2921dd4ff50..a351116e3ac64 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -136,6 +136,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index a3c584e488b65..e47d7fa97a067 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml index 378192acc6551..96a421c5f1e7a 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/coredns/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml index 0c51821831965..eb5f8a81d816d 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml index 2aef52067feac..7dc2bc46fc854 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml index 1a2921dd4ff50..a351116e3ac64 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -136,6 +136,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml index d9717082bac51..1bb6faf9266b9 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml index 1a2921dd4ff50..a351116e3ac64 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -136,6 +136,14 @@ rules: - list - watch - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - patch + - update --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index 5e7def30fc939..674fd54fe8ef2 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 2a2e8e9fefbfafdedeb9edd962e44cf2a06e337148b2da7cd5402e8a1df35fff + manifestHash: 695450dadc008db243636cc4c1f5b2bbbadffc1fb16c8931984b56ee69c6b7ba name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: From 1e77c7d4597bd63a975db43cd2f7cbbcada41989 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Mon, 11 May 2026 18:00:12 +0200 Subject: [PATCH 08/10] chore: added dns record tasks needed for etcd. --- pkg/model/elementomodel/servers.go | 3 + .../fi/cloudup/elementotasks/servergroup.go | 103 +++++++++++++++--- 2 files changed, 92 insertions(+), 14 deletions(-) diff --git a/pkg/model/elementomodel/servers.go b/pkg/model/elementomodel/servers.go index 572b54e89448f..aee5f055d4139 100644 --- a/pkg/model/elementomodel/servers.go +++ b/pkg/model/elementomodel/servers.go @@ -137,6 +137,9 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error serverGroup.APIInternalName = fi.PtrTo(b.Cluster.APIInternalName()) } serverGroup.KopsControllerInternalName = fi.PtrTo("kops-controller.internal." + b.ClusterName()) + for _, etcdCluster := range b.Cluster.Spec.EtcdClusters { + serverGroup.EtcdClusterNames = append(serverGroup.EtcdClusterNames, etcdCluster.Name) + } } } diff --git a/upup/pkg/fi/cloudup/elementotasks/servergroup.go b/upup/pkg/fi/cloudup/elementotasks/servergroup.go index 18530a78bc79e..4d217916adc55 100644 --- a/upup/pkg/fi/cloudup/elementotasks/servergroup.go +++ b/upup/pkg/fi/cloudup/elementotasks/servergroup.go @@ -58,11 +58,33 @@ type ServerGroup struct { APIPublicName *string APIInternalName *string KopsControllerInternalName *string + EtcdClusterNames []string // RootVolumeSize is the size of the root volume in GB RootVolumeSize *int32 } +var _ fi.CloudupHasDependencies = &ServerGroup{} + +func (v *ServerGroup) GetDependencies(tasks map[string]fi.CloudupTask) []fi.CloudupTask { + var deps []fi.CloudupTask + + for _, sshKey := range v.SSHKeys { + deps = append(deps, sshKey) + } + if v.Network != nil { + deps = append(deps, v.Network) + } + if v.DNSZoneTask != nil { + deps = append(deps, v.DNSZoneTask) + } + if v.UserData != nil { + deps = append(deps, fi.FindDependencies(tasks, v.UserData)...) + } + + return deps +} + func (v *ServerGroup) Find(c *fi.CloudupContext) (*ServerGroup, error) { cloud := c.T.Cloud.(elemento.ElementoCloud) client := cloud.ServerClient() @@ -302,10 +324,11 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes fmt.Printf("EKOPS: Successfully created server %q\n", name) if e.ClusterName != nil && e.DNSZone != nil { + if err := createElementoServerDNSRecord(context.TODO(), t.Cloud.DnsClient(), fi.ValueOf(e.ClusterName), fi.ValueOf(e.DNSZone), name, result.Server); err != nil { return err } - if err := createElementoControlPlaneDNSRecords(context.TODO(), t.Cloud.DnsClient(), fi.ValueOf(e.DNSZone), e, result.Server); err != nil { + if err := createElementoControlPlaneDNSRecords(context.TODO(), t.Cloud.DnsClient(), fi.ValueOf(e.DNSZone), e, name, result.Server); err != nil { return err } } @@ -315,7 +338,7 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes } func createElementoServerDNSRecord(ctx context.Context, client ecloud.DnsClient, clusterName, zoneName, serverName string, server *ecloud.Server) error { - recordValue := serverDNSAddress(server) + recordValue := serverDNSAddress(server, serverName) if recordValue == "" { klog.V(2).Infof("Skipping Elemento DNS record for server %q because it has no IP address yet", serverName) return nil @@ -332,7 +355,7 @@ func createElementoServerDNSRecord(ctx context.Context, client ecloud.DnsClient, return nil } -func createElementoControlPlaneDNSRecords(ctx context.Context, client ecloud.DnsClient, zoneName string, serverGroup *ServerGroup, server *ecloud.Server) error { +func createElementoControlPlaneDNSRecords(ctx context.Context, client ecloud.DnsClient, zoneName string, serverGroup *ServerGroup, serverName string, server *ecloud.Server) error { if serverGroup.APIPublicName == nil && serverGroup.APIInternalName == nil && serverGroup.KopsControllerInternalName == nil { return nil } @@ -341,8 +364,8 @@ func createElementoControlPlaneDNSRecords(ctx context.Context, client ecloud.Dns return err } - publicAddress := serverPublicDNSAddress(server) - internalAddress := serverPrivateDNSAddress(server) + publicAddress := serverPublicDNSAddress(server, serverName) + internalAddress := serverPrivateDNSAddress(server, serverName) if publicAddress == "" { publicAddress = internalAddress } @@ -368,37 +391,89 @@ func createElementoControlPlaneDNSRecords(ctx context.Context, client ecloud.Dns return err } } + if internalAddress != "" { + if err := createElementoEtcdDNSRecords(ctx, client, zoneName, fi.ValueOf(serverGroup.ClusterName), serverGroup.EtcdClusterNames, internalAddress); err != nil { + return err + } + } + + return nil +} + +func createElementoEtcdDNSRecords(ctx context.Context, client ecloud.DnsClient, zoneName, clusterName string, etcdClusterNames []string, recordValue string) error { + clusterName = strings.TrimSuffix(strings.TrimSpace(clusterName), ".") + if clusterName == "" { + clusterName = strings.TrimSuffix(strings.TrimSpace(zoneName), ".") + } + if clusterName == "" { + return nil + } + + etcdClusterNames = normalizedElementoEtcdClusterNames(etcdClusterNames) + for _, etcdClusterName := range etcdClusterNames { + for _, recordName := range []string{ + fmt.Sprintf("node0.%s", etcdClusterName), + fmt.Sprintf("%s--%s--0.internal", clusterName, etcdClusterName), + } { + if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, recordValue); err != nil { + return err + } + } + } return nil } -func serverPrivateDNSAddress(server *ecloud.Server) string { +func normalizedElementoEtcdClusterNames(etcdClusterNames []string) []string { + var normalized []string + for _, name := range etcdClusterNames { + name = strings.TrimSpace(name) + if name == "" { + continue + } + normalized = append(normalized, name) + } + if len(normalized) == 0 { + normalized = []string{"main", "events"} + } + return normalized +} + +func serverPrivateDNSAddress(server *ecloud.Server, serverName string) string { if server == nil { - return "" + return staticServerDNSAddress(serverName) } for _, privateNet := range server.PrivateNet { if privateNet.IP != nil { return privateNet.IP.String() } } - return "" + return staticServerDNSAddress(serverName) } -func serverPublicDNSAddress(server *ecloud.Server) string { +func serverPublicDNSAddress(server *ecloud.Server, serverName string) string { if server == nil { - return "" + return staticServerDNSAddress(serverName) } if server.PublicNet.IPv4 != "" { return server.PublicNet.IPv4 } - return server.PublicNet.IPv6 + if server.PublicNet.IPv6 != "" { + return server.PublicNet.IPv6 + } + return staticServerDNSAddress(serverName) } -func serverDNSAddress(server *ecloud.Server) string { - if address := serverPrivateDNSAddress(server); address != "" { +func serverDNSAddress(server *ecloud.Server, serverName string) string { + if address := serverPrivateDNSAddress(server, serverName); address != "" { return address } - return serverPublicDNSAddress(server) + return serverPublicDNSAddress(server, serverName) +} + +func staticServerDNSAddress(serverName string) string { + ip, _, _ := ecloud.StaticNetworkForServerName(serverName) + return ip } func trimElementoDNSZoneSuffix(name, zone string) string { From 9e6ba60b642a12f9bf3934298220f63a08249bf9 Mon Sep 17 00:00:00 2001 From: Filippo Valle Date: Fri, 15 May 2026 15:30:17 +0200 Subject: [PATCH 09/10] use float as storage size --- upup/pkg/fi/cloudup/elementotasks/servergroup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/pkg/fi/cloudup/elementotasks/servergroup.go b/upup/pkg/fi/cloudup/elementotasks/servergroup.go index 4d217916adc55..1c6524c136f44 100644 --- a/upup/pkg/fi/cloudup/elementotasks/servergroup.go +++ b/upup/pkg/fi/cloudup/elementotasks/servergroup.go @@ -297,7 +297,7 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes // Add root volume configuration if specified if e.RootVolumeSize != nil { - opts.ServerType.Disk = int(fi.ValueOf(e.RootVolumeSize)) + opts.ServerType.Disk = float64(fi.ValueOf(e.RootVolumeSize)) } // Add the SSH keys. From 17bd46fc5debc906ae4f64844d3c4393f7b03c34 Mon Sep 17 00:00:00 2001 From: Iliyan Plamenov Kostadinov Date: Mon, 18 May 2026 14:58:34 +0200 Subject: [PATCH 10/10] chore: made the functions to insert a DNS record as kops native tasks. --- pkg/model/elementomodel/dns.go | 111 +++++++++++- pkg/model/elementomodel/servers.go | 22 +-- upup/pkg/fi/cloudup/elementotasks/dns.go | 47 +++-- .../fi/cloudup/elementotasks/servergroup.go | 169 +----------------- 4 files changed, 149 insertions(+), 200 deletions(-) diff --git a/pkg/model/elementomodel/dns.go b/pkg/model/elementomodel/dns.go index 22df1fa23e5a2..5d5205f49f7a3 100644 --- a/pkg/model/elementomodel/dns.go +++ b/pkg/model/elementomodel/dns.go @@ -16,7 +16,17 @@ limitations under the License. package elementomodel -import "k8s.io/kops/upup/pkg/fi" +import ( + "fmt" + "strings" + + "github.com/Elemento-Modular-Cloud/ecloud-go/ecloud" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/elementotasks" +) + +const elementoDNSRecordTTL int64 = 3600 // DNSModelBuilder is the provider-native integration point for Elemento-managed // DNS records that must exist before nodeup starts. @@ -28,7 +38,102 @@ type DNSModelBuilder struct { var _ fi.CloudupModelBuilder = &DNSModelBuilder{} func (b *DNSModelBuilder) Build(c *fi.CloudupModelBuilderContext) error { - // Elemento DNS is currently create-only in the SDK, so API and node records - // are created from ServerGroup.RenderElemento once real server IPs are known. + if !b.Cluster.PublishesDNSRecords() { + return nil + } + + dnsZoneTask := &elementotasks.DNSZone{ + Name: fi.PtrTo(b.ClusterName()), + Lifecycle: b.Lifecycle, + } + c.EnsureTask(dnsZoneTask) + + var previousDNSRecordTask *elementotasks.DNSRecord + for _, ig := range b.InstanceGroups { + dnsRecordTasks, err := b.elementoDNSRecordTasksForInstanceGroup(ig, b.Lifecycle, dnsZoneTask) + if err != nil { + return err + } + for _, dnsRecordTask := range dnsRecordTasks { + dnsRecordTask.DependsOn = previousDNSRecordTask + c.EnsureTask(dnsRecordTask) + previousDNSRecordTask = dnsRecordTask + } + } + return nil } + +func (b *ElementoModelContext) elementoDNSRecordTasksForInstanceGroup(ig *kops.InstanceGroup, lifecycle fi.Lifecycle, dnsZoneTask *elementotasks.DNSZone) ([]*elementotasks.DNSRecord, error) { + if !b.Cluster.PublishesDNSRecords() { + return nil, nil + } + + igSize := fi.ValueOf(ig.Spec.MinSize) + clusterName := b.ClusterName() + zoneName := b.ClusterName() + + var tasks []*elementotasks.DNSRecord + addRecord := func(recordName, recordValue string) { + tasks = append(tasks, &elementotasks.DNSRecord{ + Name: fi.PtrTo(trimElementoDNSZoneSuffix(recordName, zoneName)), + Data: fi.PtrTo(recordValue), + DNSZone: fi.PtrTo(zoneName), + DNSZoneTask: dnsZoneTask, + Type: fi.PtrTo("A"), + TTL: fi.PtrTo(elementoDNSRecordTTL), + Lifecycle: lifecycle, + }) + } + + for ordinal := int32(1); ordinal <= igSize; ordinal++ { + serverName := fmt.Sprintf("%s-%d", ig.Name, ordinal) + serverIP, _, _ := ecloud.StaticNetworkForServerName(serverName) + if serverIP == "" { + return nil, fmt.Errorf("static Elemento DNS address for server %q is empty", serverName) + } + + addRecord(fmt.Sprintf("%s.%s", serverName, clusterName), serverIP) + + if !ig.HasAPIServer() || ordinal != 1 { + continue + } + + if !b.UseLoadBalancerForAPI() { + apiPublicName := b.Cluster.Spec.API.PublicName + if apiPublicName == "" { + apiPublicName = "api." + clusterName + } + addRecord(apiPublicName, serverIP) + } + if !b.UseLoadBalancerForInternalAPI() { + addRecord(b.Cluster.APIInternalName(), serverIP) + } + addRecord("kops-controller.internal."+clusterName, serverIP) + + for _, etcdClusterName := range elementoEtcdClusterNames(b.Cluster.Spec.EtcdClusters) { + addRecord(fmt.Sprintf("node0.%s.%s", etcdClusterName, clusterName), serverIP) + addRecord(fmt.Sprintf("%s--%s--0.internal.%s", clusterName, etcdClusterName, clusterName), serverIP) + } + } + + return tasks, nil +} + +func elementoEtcdClusterNames(etcdClusters []kops.EtcdClusterSpec) []string { + var names []string + for _, etcdCluster := range etcdClusters { + name := strings.TrimSpace(etcdCluster.Name) + if name != "" { + names = append(names, name) + } + } + if len(names) == 0 { + names = []string{"main", "events"} + } + return names +} + +func trimElementoDNSZoneSuffix(name, zone string) string { + return strings.TrimSuffix(name, "."+strings.TrimSuffix(zone, ".")) +} diff --git a/pkg/model/elementomodel/servers.go b/pkg/model/elementomodel/servers.go index aee5f055d4139..69d8d2bbfbbff 100644 --- a/pkg/model/elementomodel/servers.go +++ b/pkg/model/elementomodel/servers.go @@ -63,7 +63,6 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error Name: fi.PtrTo(b.ClusterName()), Lifecycle: b.Lifecycle, } - c.EnsureTask(dnsZoneTask) } for _, ig := range b.InstanceGroups { @@ -122,25 +121,12 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error RootVolumeSize: rootVolumeSize, } if b.Cluster.PublishesDNSRecords() { - serverGroup.ClusterName = fi.PtrTo(b.ClusterName()) - serverGroup.DNSZone = fi.PtrTo(b.ClusterName()) serverGroup.DNSZoneTask = dnsZoneTask - if ig.HasAPIServer() { - if !b.UseLoadBalancerForAPI() { - apiPublicName := b.Cluster.Spec.API.PublicName - if apiPublicName == "" { - apiPublicName = "api." + b.ClusterName() - } - serverGroup.APIPublicName = fi.PtrTo(apiPublicName) - } - if !b.UseLoadBalancerForInternalAPI() { - serverGroup.APIInternalName = fi.PtrTo(b.Cluster.APIInternalName()) - } - serverGroup.KopsControllerInternalName = fi.PtrTo("kops-controller.internal." + b.ClusterName()) - for _, etcdCluster := range b.Cluster.Spec.EtcdClusters { - serverGroup.EtcdClusterNames = append(serverGroup.EtcdClusterNames, etcdCluster.Name) - } + dnsRecordTasks, err := b.elementoDNSRecordTasksForInstanceGroup(ig, b.Lifecycle, dnsZoneTask) + if err != nil { + return err } + serverGroup.DNSRecordTasks = dnsRecordTasks } c.AddTask(&serverGroup) diff --git a/upup/pkg/fi/cloudup/elementotasks/dns.go b/upup/pkg/fi/cloudup/elementotasks/dns.go index 45a1f0b2cd067..4ca19d0c6a88a 100644 --- a/upup/pkg/fi/cloudup/elementotasks/dns.go +++ b/upup/pkg/fi/cloudup/elementotasks/dns.go @@ -102,16 +102,30 @@ func (_ *DNSZone) RenderElemento(t *elemento.ElementoAPITarget, actual, expected // +kops:fitask type DNSRecord struct { - Name *string - Data *string - DNSZone *string - Type *string - TTL *int64 - Lifecycle fi.Lifecycle - Comment *string + Name *string + Data *string + DNSZone *string + DNSZoneTask *DNSZone + DependsOn *DNSRecord + Type *string + TTL *int64 + Lifecycle fi.Lifecycle + Comment *string } var _ fi.CloudupTask = &DNSRecord{} +var _ fi.CloudupHasDependencies = &DNSRecord{} + +func (d *DNSRecord) GetDependencies(tasks map[string]fi.CloudupTask) []fi.CloudupTask { + var deps []fi.CloudupTask + if d.DNSZoneTask != nil { + deps = append(deps, d.DNSZoneTask) + } + if d.DependsOn != nil { + deps = append(deps, d.DependsOn) + } + return deps +} func (d *DNSRecord) Find(c *fi.CloudupContext) (*DNSRecord, error) { cloud := c.T.Cloud.(elemento.ElementoCloud) @@ -129,13 +143,15 @@ func (d *DNSRecord) Find(c *fi.CloudupContext) (*DNSRecord, error) { } return &DNSRecord{ - Name: fi.PtrTo(record.Name), - Data: fi.PtrTo(record.Value), - DNSZone: d.DNSZone, - Type: fi.PtrTo(record.Type), - TTL: fi.PtrTo(int64(record.TTL)), - Lifecycle: d.Lifecycle, - Comment: d.Comment, + Name: fi.PtrTo(record.Name), + Data: fi.PtrTo(record.Value), + DNSZone: d.DNSZone, + DNSZoneTask: d.DNSZoneTask, + DependsOn: d.DependsOn, + Type: fi.PtrTo(record.Type), + TTL: fi.PtrTo(int64(record.TTL)), + Lifecycle: d.Lifecycle, + Comment: d.Comment, }, nil } @@ -169,9 +185,6 @@ func (_ *DNSRecord) RenderElemento(t *elemento.ElementoAPITarget, actual, expect recordName := fi.ValueOf(expected.Name) recordValue := fi.ValueOf(expected.Data) - if err := ensureElementoDNSZone(context.TODO(), client, zoneName); err != nil { - return err - } if err := ensureElementoDNSRecord(context.TODO(), client, zoneName, recordName, recordValue); err != nil { return err } diff --git a/upup/pkg/fi/cloudup/elementotasks/servergroup.go b/upup/pkg/fi/cloudup/elementotasks/servergroup.go index 1c6524c136f44..bf56cede1fff4 100644 --- a/upup/pkg/fi/cloudup/elementotasks/servergroup.go +++ b/upup/pkg/fi/cloudup/elementotasks/servergroup.go @@ -51,14 +51,8 @@ type ServerGroup struct { Labels map[string]string - ClusterName *string - DNSZone *string - DNSZoneTask *DNSZone - - APIPublicName *string - APIInternalName *string - KopsControllerInternalName *string - EtcdClusterNames []string + DNSZoneTask *DNSZone + DNSRecordTasks []*DNSRecord // RootVolumeSize is the size of the root volume in GB RootVolumeSize *int32 @@ -78,6 +72,9 @@ func (v *ServerGroup) GetDependencies(tasks map[string]fi.CloudupTask) []fi.Clou if v.DNSZoneTask != nil { deps = append(deps, v.DNSZoneTask) } + for _, dnsRecordTask := range v.DNSRecordTasks { + deps = append(deps, dnsRecordTask) + } if v.UserData != nil { deps = append(deps, fi.FindDependencies(tasks, v.UserData)...) } @@ -314,172 +311,20 @@ func (*ServerGroup) RenderElemento(t *elemento.ElementoAPITarget, a, e, changes fmt.Printf("EKOPS: Creating server %q with options: Location=%s, Size=%s, Image=%s\n", name, e.Location, e.Size, e.Image) + fmt.Printf("EKOPS: Calling client.Create() for server %q\n", name) - result, _, err := client.Create(context.TODO(), opts) + _, _, err := client.Create(context.TODO(), opts) if err != nil { fmt.Printf("EKOPS: ERROR creating server %q: %v\n", name, err) return err } fmt.Printf("EKOPS: Successfully created server %q\n", name) - - if e.ClusterName != nil && e.DNSZone != nil { - - if err := createElementoServerDNSRecord(context.TODO(), t.Cloud.DnsClient(), fi.ValueOf(e.ClusterName), fi.ValueOf(e.DNSZone), name, result.Server); err != nil { - return err - } - if err := createElementoControlPlaneDNSRecords(context.TODO(), t.Cloud.DnsClient(), fi.ValueOf(e.DNSZone), e, name, result.Server); err != nil { - return err - } - } - } - - return nil -} - -func createElementoServerDNSRecord(ctx context.Context, client ecloud.DnsClient, clusterName, zoneName, serverName string, server *ecloud.Server) error { - recordValue := serverDNSAddress(server, serverName) - if recordValue == "" { - klog.V(2).Infof("Skipping Elemento DNS record for server %q because it has no IP address yet", serverName) - return nil - } - - recordName := trimElementoDNSZoneSuffix(fmt.Sprintf("%s.%s", serverName, clusterName), zoneName) - if err := ensureElementoDNSZone(ctx, client, zoneName); err != nil { - return err - } - if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, recordValue); err != nil { - return err - } - - return nil -} - -func createElementoControlPlaneDNSRecords(ctx context.Context, client ecloud.DnsClient, zoneName string, serverGroup *ServerGroup, serverName string, server *ecloud.Server) error { - if serverGroup.APIPublicName == nil && serverGroup.APIInternalName == nil && serverGroup.KopsControllerInternalName == nil { - return nil - } - - if err := ensureElementoDNSZone(ctx, client, zoneName); err != nil { - return err - } - - publicAddress := serverPublicDNSAddress(server, serverName) - internalAddress := serverPrivateDNSAddress(server, serverName) - if publicAddress == "" { - publicAddress = internalAddress - } - if internalAddress == "" { - internalAddress = publicAddress - } - - if serverGroup.APIPublicName != nil && publicAddress != "" { - recordName := trimElementoDNSZoneSuffix(fi.ValueOf(serverGroup.APIPublicName), zoneName) - if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, publicAddress); err != nil { - return err - } - } - if serverGroup.APIInternalName != nil && internalAddress != "" { - recordName := trimElementoDNSZoneSuffix(fi.ValueOf(serverGroup.APIInternalName), zoneName) - if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, internalAddress); err != nil { - return err - } - } - if serverGroup.KopsControllerInternalName != nil && internalAddress != "" { - recordName := trimElementoDNSZoneSuffix(fi.ValueOf(serverGroup.KopsControllerInternalName), zoneName) - if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, internalAddress); err != nil { - return err - } - } - if internalAddress != "" { - if err := createElementoEtcdDNSRecords(ctx, client, zoneName, fi.ValueOf(serverGroup.ClusterName), serverGroup.EtcdClusterNames, internalAddress); err != nil { - return err - } - } - - return nil -} - -func createElementoEtcdDNSRecords(ctx context.Context, client ecloud.DnsClient, zoneName, clusterName string, etcdClusterNames []string, recordValue string) error { - clusterName = strings.TrimSuffix(strings.TrimSpace(clusterName), ".") - if clusterName == "" { - clusterName = strings.TrimSuffix(strings.TrimSpace(zoneName), ".") - } - if clusterName == "" { - return nil - } - - etcdClusterNames = normalizedElementoEtcdClusterNames(etcdClusterNames) - for _, etcdClusterName := range etcdClusterNames { - for _, recordName := range []string{ - fmt.Sprintf("node0.%s", etcdClusterName), - fmt.Sprintf("%s--%s--0.internal", clusterName, etcdClusterName), - } { - if err := ensureElementoDNSRecord(ctx, client, zoneName, recordName, recordValue); err != nil { - return err - } - } } return nil } -func normalizedElementoEtcdClusterNames(etcdClusterNames []string) []string { - var normalized []string - for _, name := range etcdClusterNames { - name = strings.TrimSpace(name) - if name == "" { - continue - } - normalized = append(normalized, name) - } - if len(normalized) == 0 { - normalized = []string{"main", "events"} - } - return normalized -} - -func serverPrivateDNSAddress(server *ecloud.Server, serverName string) string { - if server == nil { - return staticServerDNSAddress(serverName) - } - for _, privateNet := range server.PrivateNet { - if privateNet.IP != nil { - return privateNet.IP.String() - } - } - return staticServerDNSAddress(serverName) -} - -func serverPublicDNSAddress(server *ecloud.Server, serverName string) string { - if server == nil { - return staticServerDNSAddress(serverName) - } - if server.PublicNet.IPv4 != "" { - return server.PublicNet.IPv4 - } - if server.PublicNet.IPv6 != "" { - return server.PublicNet.IPv6 - } - return staticServerDNSAddress(serverName) -} - -func serverDNSAddress(server *ecloud.Server, serverName string) string { - if address := serverPrivateDNSAddress(server, serverName); address != "" { - return address - } - return serverPublicDNSAddress(server, serverName) -} - -func staticServerDNSAddress(serverName string) string { - ip, _, _ := ecloud.StaticNetworkForServerName(serverName) - return ip -} - -func trimElementoDNSZoneSuffix(name, zone string) string { - return strings.TrimSuffix(name, "."+strings.TrimSuffix(zone, ".")) -} - func safeBytesHash(data []byte) string { // Calculate the SHA256 checksum of the data sum256 := sha256.Sum256(data)