ignition: add vpc IPv6 CIDR to the cluster-config-v1 ConfigMap

The etcd operator requires IPv6 machine networks in dualstack
networking. Since we may not know the VPC IPV6 at install time, we need
to add it in later on once the infrastructure is ready.

Author: tthvo

pkg/infrastructure/aws/clusterapi/aws.go

@@ -92,7 +92,10 @@ func (*Provider) PreProvision(ctx context.Context, in clusterapi.PreProvisionInp
 // infrastructure CR. The infrastructure CR is updated and added to the ignition files. CAPA creates a
 // bucket for ignition, and this ignition data will be placed in the bucket.
 func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]*corev1.Secret, error) {
-	ignOutput, err := editIgnition(ctx, in)
+	ignOutput, err := clusterapi.ApplyIgnitionEdits(ctx, in,
+		editIgnitionForCustomDNS,
+		editIgnitionForDualStack,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to edit bootstrap master or worker ignition: %w", err)
 	}

pkg/infrastructure/aws/clusterapi/ignition.go

@@ -14,11 +14,14 @@ import (
 	awsconfig "github.com/openshift/installer/pkg/asset/installconfig/aws"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
+	"github.com/openshift/installer/pkg/ipnet"
+	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 	"github.com/openshift/installer/pkg/types/dns"
+	"github.com/openshift/installer/pkg/types/network"
 )
 
-func editIgnition(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi.IgnitionOutput, error) {
+func editIgnitionForCustomDNS(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi.IgnitionOutput, error) {
 	if in.InstallConfig.Config.AWS.UserProvisionedDNS != dns.UserProvisionedDNSEnabled {
 		return &clusterapi.IgnitionOutput{
 			UpdatedBootstrapIgn: in.BootstrapIgnData,
@@ -83,5 +86,48 @@ func editIgnition(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi
 		publicIPAddresses = privateIPAddresses
 	}
 	logrus.Debugf("AWS: Editing Ignition files to start in-cluster DNS when UserProvisionedDNS is enabled")
-	return clusterapi.EditIgnition(in, awstypes.Name, publicIPAddresses, privateIPAddresses)
+	return clusterapi.EditIgnitionForCustomDNS(in, awstypes.Name, publicIPAddresses, privateIPAddresses)
+}
+
+func editIgnitionForDualStack(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi.IgnitionOutput, error) {
+	ic := in.InstallConfig.Config
+	machineCIDRs := capiutils.MachineCIDRsFromInstallConfig(in.InstallConfig)
+
+	// If the machine network entries contain IPv6 CIDRs, the users must have added in manually for BYO subnets.
+	// In this case, those CIDRs are already passed to the AWSCluster node port ingress rule spec
+	if !ic.AWS.IPFamily.DualStackEnabled() || len(capiutils.GetIPv6CIDRs(machineCIDRs)) > 0 {
+		return &clusterapi.IgnitionOutput{
+			UpdatedBootstrapIgn: in.BootstrapIgnData,
+			UpdatedMasterIgn:    in.MasterIgnData,
+			UpdatedWorkerIgn:    in.WorkerIgnData}, nil
+	}
+
+	awsCluster := &capa.AWSCluster{}
+	key := k8sClient.ObjectKey{
+		Name:      in.InfraID,
+		Namespace: capiutils.Namespace,
+	}
+	if err := in.Client.Get(ctx, key, awsCluster); err != nil {
+		return nil, fmt.Errorf("failed to get AWSCluster: %w", err)
+	}
+
+	vpcSpec := awsCluster.Spec.NetworkSpec.VPC
+	if vpcSpec.IPv6 == nil || vpcSpec.IPv6.CidrBlock == "" {
+		return nil, fmt.Errorf("dualstack networking is enabled, but VPC does not have IPV6 CIDR")
+	}
+
+	machineNetworks := ic.MachineNetwork
+	ipv6Entry := []types.MachineNetworkEntry{
+		{
+			CIDR: *ipnet.MustParseCIDR(vpcSpec.IPv6.CidrBlock),
+		},
+	}
+
+	if ic.AWS.IPFamily == network.DualStackIPv6Primary {
+		machineNetworks = append(ipv6Entry, machineNetworks...)
+	} else {
+		machineNetworks = append(machineNetworks, ipv6Entry...)
+	}
+
+	return clusterapi.EditIgnitionForDualStack(in, awstypes.Name, machineNetworks)
 }

pkg/infrastructure/azure/ignition.go

@@ -50,5 +50,5 @@ func editIgnition(ctx context.Context, in clusterapi.IgnitionInput, publicIP str
 		apiLBIP = publicIP
 	}
 	logrus.Debugf("Azure: Editing Ignition files with API LB IP: %s and API Int LB IP: %s", apiLBIP, apiIntLBIP)
-	return clusterapi.EditIgnition(in, azure.Name, []string{apiLBIP}, []string{apiIntLBIP})
+	return clusterapi.EditIgnitionForCustomDNS(in, azure.Name, []string{apiLBIP}, []string{apiIntLBIP})
 }

pkg/infrastructure/clusterapi/ignition.go

@@ -20,6 +20,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/lbconfig"
 	"github.com/openshift/installer/pkg/asset/machines"
 	"github.com/openshift/installer/pkg/asset/tls"
+	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 	azuretypes "github.com/openshift/installer/pkg/types/azure"
 	gcptypes "github.com/openshift/installer/pkg/types/gcp"
@@ -32,20 +33,24 @@ const (
 	mcsCertFile            = "/opt/openshift/tls/machine-config-server.crt"
 	masterUserDataFile     = "/opt/openshift/openshift/99_openshift-cluster-api_master-user-data-secret.yaml"
 	workerUserDataFile     = "/opt/openshift/openshift/99_openshift-cluster-api_worker-user-data-secret.yaml"
+	clusterConfigDataFile  = "/opt/openshift/manifests/cluster-config.yaml"
 
 	// header is the string that precedes the encoded data in the ignition data.
 	// The data must be replaced before decoding the string, and the string must be
 	// prepended to the encoded data.
 	header = "data:text/plain;charset=utf-8;base64,"
 
+	// The key in the cluster-config-v1 ConfigMap to extract the install-config.
+	clusterConfigCMKey = "install-config"
+
 	masterRole = "master"
 	workerRole = "worker"
 )
 
-// EditIgnition attempts to edit the contents of the bootstrap ignition when the user has selected
+// EditIgnitionForCustomDNS attempts to edit the contents of the bootstrap ignition when the user has selected
 // a custom DNS configuration. Find the public and private load balancer addresses and fill in the
 // infrastructure file within the ignition struct.
-func EditIgnition(in IgnitionInput, platform string, publicIPAddresses, privateIPAddresses []string) (*IgnitionOutput, error) {
+func EditIgnitionForCustomDNS(in IgnitionInput, platform string, publicIPAddresses, privateIPAddresses []string) (*IgnitionOutput, error) {
 	ignData := &igntypes.Config{}
 	err := json.Unmarshal(in.BootstrapIgnData, ignData)
 	if err != nil {
@@ -292,3 +297,77 @@ func updateUserDataSecret(in IgnitionInput, role string, config *igntypes.Config
 	}
 	return nil
 }
+
+// EditIgnitionForDualStack attempts to edit the contents of the bootstrap ignition when the cluster is in dualstack.
+func EditIgnitionForDualStack(in IgnitionInput, platform string, machineNetworks []types.MachineNetworkEntry) (*IgnitionOutput, error) {
+	ignData := &igntypes.Config{}
+	err := json.Unmarshal(in.BootstrapIgnData, ignData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal bootstrap ignition: %w", err)
+	}
+
+	err = updateMachineNetworks(in, ignData, machineNetworks)
+	if err != nil {
+		return nil, fmt.Errorf("failed to update machine networks in ignition config: %w", err)
+	}
+	logrus.Debugf("Successfully updated the install-config machine networks")
+
+	editedIgnBytes, err := json.Marshal(ignData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert ignition data to json: %w", err)
+	}
+	logrus.Debugf("Successfully updated bootstrap ignition with updated manifests for dualstack networking")
+
+	return &IgnitionOutput{
+		UpdatedBootstrapIgn: editedIgnBytes,
+		UpdatedMasterIgn:    in.MasterIgnData,
+		UpdatedWorkerIgn:    in.WorkerIgnData,
+	}, nil
+}
+
+func updateMachineNetworks(in IgnitionInput, config *igntypes.Config, machineNetworks []types.MachineNetworkEntry) error {
+	for i, fileData := range config.Storage.Files {
+		if fileData.Path != clusterConfigDataFile {
+			continue
+		}
+
+		contents := strings.Split(*config.Storage.Files[i].Contents.Source, ",")
+		rawDecodedText, err := base64.StdEncoding.DecodeString(contents[1])
+		if err != nil {
+			return fmt.Errorf("failed to decode contents of ignition file: %w", err)
+		}
+
+		configCM := &corev1.ConfigMap{}
+		if err := yaml.Unmarshal(rawDecodedText, configCM); err != nil {
+			return fmt.Errorf("failed to unmarshal cluster-config ConfigMap: %w", err)
+		}
+
+		installConfig := &types.InstallConfig{}
+		if err := yaml.Unmarshal([]byte(configCM.Data[clusterConfigCMKey]), installConfig); err != nil {
+			return fmt.Errorf("failed to unmarshal install-config content: %w", err)
+		}
+
+		// Update the machine network field
+		installConfig.MachineNetwork = machineNetworks
+
+		// Convert the installconfig back to string and save it to the configmap
+		icContents, err := yaml.Marshal(installConfig)
+		if err != nil {
+			return fmt.Errorf("failed to marshal install-config: %w", err)
+		}
+		configCM.Data[clusterConfigCMKey] = string(icContents)
+
+		// convert the infrastructure back to an encoded string
+		configCMContents, err := yaml.Marshal(configCM)
+		if err != nil {
+			return fmt.Errorf("failed to marshal cluster-config ConfigMap: %w", err)
+		}
+
+		encoded := fmt.Sprintf("%s%s", header, base64.StdEncoding.EncodeToString(configCMContents))
+		// replace the contents with the edited information
+		config.Storage.Files[i].Contents.Source = &encoded
+
+		break
+	}
+	return nil
+}

pkg/infrastructure/clusterapi/types.go

@@ -72,6 +72,42 @@ type IgnitionInput struct {
 	RootCA           *tls.RootCA
 }
 
+// WithOutput returns a new IgnitionInput with ignition data from the output.
+// This allows chaining multiple ignition edits.
+func (in IgnitionInput) WithOutput(output *IgnitionOutput) IgnitionInput {
+	if output == nil {
+		return in
+	}
+	in.BootstrapIgnData = output.UpdatedBootstrapIgn
+	in.MasterIgnData = output.UpdatedMasterIgn
+	in.WorkerIgnData = output.UpdatedWorkerIgn
+	return in
+}
+
+// IgnitionEditFunc is a function that edits ignition data.
+type IgnitionEditFunc func(context.Context, IgnitionInput) (*IgnitionOutput, error)
+
+// ApplyIgnitionEdits applies multiple ignition edit functions in sequence, passing the ignition output
+// of each as input to the next. Returns the final output or the first error encountered.
+func ApplyIgnitionEdits(ctx context.Context, in IgnitionInput, edits ...IgnitionEditFunc) (*IgnitionOutput, error) {
+	output := &IgnitionOutput{
+		UpdatedBootstrapIgn: in.BootstrapIgnData,
+		UpdatedMasterIgn:    in.MasterIgnData,
+		UpdatedWorkerIgn:    in.WorkerIgnData,
+	}
+
+	for _, edit := range edits {
+		result, err := edit(ctx, in)
+		if err != nil {
+			return nil, err
+		}
+		output = result
+		in = in.WithOutput(result)
+	}
+
+	return output, nil
+}
+
 // IgnitionOutput collects updated Ignition Data for Bootstrap, Master and Worker nodes.
 type IgnitionOutput struct {
 	UpdatedBootstrapIgn []byte

pkg/infrastructure/gcp/clusterapi/ignition.go

@@ -76,5 +76,5 @@ func editIgnition(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi
 	}
 
 	logrus.Debugf("GCP: Editing Ignition files to start in-cluster DNS when UserProvisionedDNS is enabled")
-	return clusterapi.EditIgnition(in, gcp.Name, []string{computeAddress}, []string{computeIntAddress})
+	return clusterapi.EditIgnitionForCustomDNS(in, gcp.Name, []string{computeAddress}, []string{computeIntAddress})
 }