diff --git a/cmd/generate-config/config/config-openapi-spec.json b/cmd/generate-config/config/config-openapi-spec.json index d303b11e96..803f737ccf 100755 --- a/cmd/generate-config/config/config-openapi-spec.json +++ b/cmd/generate-config/config/config-openapi-spec.json @@ -232,8 +232,11 @@ "type": "string" }, "nextHop": { - "description": "IP address of the remote cluster's node, used as next-hop for routing.", - "type": "string" + "description": "IP addresses of the remote cluster's node, used as next-hop for routing.\nAt most one IPv4 and one IPv6 address. Dual-stack clusters need both.", + "type": "array", + "items": { + "type": "string" + } }, "serviceNetwork": { "description": "Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes.", diff --git a/docs/user/howto_config.md b/docs/user/howto_config.md index 1e7fd62732..2ae85bb83e 100644 --- a/docs/user/howto_config.md +++ b/docs/user/howto_config.md @@ -38,7 +38,7 @@ clusterToCluster: remoteClusters: - clusterNetwork: [] domain: "" - nextHop: "" + nextHop: [] serviceNetwork: [] routing: routeTableID: 0 @@ -207,7 +207,7 @@ clusterToCluster: remoteClusters: - clusterNetwork: [] domain: "" - nextHop: "" + nextHop: [] serviceNetwork: [] routing: routeTableID: 200 diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go index 6d7ee786fb..2ceca76e72 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go @@ -68,8 +68,9 @@ type C2CC struct { } type RemoteCluster struct { - // IP address of the remote cluster's node, used as next-hop for routing. - NextHop string `json:"nextHop"` + // IP addresses of the remote cluster's node, used as next-hop for routing. + // At most one IPv4 and one IPv6 address. Dual-stack clusters need both. + NextHop []string `json:"nextHop"` // Pod CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. ClusterNetwork []string `json:"clusterNetwork"` // Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. @@ -80,8 +81,9 @@ type RemoteCluster struct { Domain string `json:"domain,omitempty"` } + type ResolvedRemoteCluster struct { - NextHop net.IP + NextHops map[int]net.IP // key: netlink.FAMILY_V4 or netlink.FAMILY_V6 ClusterNetwork []*net.IPNet ServiceNetwork []*net.IPNet Domain string @@ -89,6 +91,32 @@ type ResolvedRemoteCluster struct { ProbeIP string // 11th IP of ServiceNetwork[0], deterministic probe service ClusterIP } +func (rc *ResolvedRemoteCluster) NextHopForFamily(family int) (net.IP, bool) { + ip, ok := rc.NextHops[family] + return ip, ok +} + +func (rc *ResolvedRemoteCluster) PrimaryNextHop() net.IP { + if len(rc.ClusterNetwork) == 0 { + return nil + } + return rc.NextHops[ipFamilyOfIPNet(rc.ClusterNetwork[0])] +} + +func ipFamilyOfIPNet(ipNet *net.IPNet) int { + if ipNet.IP.To4() != nil { + return netlink.FAMILY_V4 + } + return netlink.FAMILY_V6 +} + +func familyName(family int) string { + if family == netlink.FAMILY_V4 { + return "IPv4" + } + return "IPv6" +} + func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet { all := make([]*net.IPNet, 0, len(rc.ClusterNetwork)+len(rc.ServiceNetwork)) all = append(all, rc.ClusterNetwork...) @@ -114,7 +142,7 @@ func (c *C2CC) AllRemoteCIDRStrings() []string { } func (rc *RemoteCluster) isEmpty() bool { - return rc.NextHop == "" && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" + return len(rc.NextHop) == 0 && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" } func (c *C2CC) stripEmptyRemoteClusters() { @@ -155,11 +183,26 @@ func (c *C2CC) parseRemoteClusters() ([]ResolvedRemoteCluster, []error) { rc := &c.RemoteClusters[i] label := fmt.Sprintf("remoteClusters[%d]", i) - ip := net.ParseIP(rc.NextHop) - if ip == nil { - errs = append(errs, fmt.Errorf("%s.nextHop %q is not a valid IP address", label, rc.NextHop)) + if len(rc.NextHop) == 0 { + errs = append(errs, fmt.Errorf("%s.nextHop must not be empty", label)) + } + resolved[i].NextHops = make(map[int]net.IP, len(rc.NextHop)) + for j, hopStr := range rc.NextHop { + ip := net.ParseIP(hopStr) + if ip == nil { + errs = append(errs, fmt.Errorf("%s.nextHop[%d] %q is not a valid IP address", label, j, hopStr)) + continue + } + family := netlink.FAMILY_V4 + if ip.To4() == nil { + family = netlink.FAMILY_V6 + } + if _, dup := resolved[i].NextHops[family]; dup { + errs = append(errs, fmt.Errorf("%s.nextHop has multiple %s addresses (max 1 per family)", label, familyName(family))) + continue + } + resolved[i].NextHops[family] = ip } - resolved[i].NextHop = ip if len(rc.ClusterNetwork) == 0 { errs = append(errs, fmt.Errorf("%s.clusterNetwork must not be empty", label)) @@ -359,14 +402,16 @@ func validateRemoteCluster( label := fmt.Sprintf("remoteClusters[%d]", i) var errs []error - normalizedNextHop := res.NextHop.String() - if res.NextHop.Equal(nodeIP) || (nodeIPv6 != nil && res.NextHop.Equal(nodeIPv6)) { - errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalizedNextHop)) - } - if prev, ok := seenNextHops[normalizedNextHop]; ok { - errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalizedNextHop, prev)) - } else { - seenNextHops[normalizedNextHop] = i + for _, hop := range res.NextHops { + normalized := hop.String() + if hop.Equal(nodeIP) || (nodeIPv6 != nil && hop.Equal(nodeIPv6)) { + errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalized)) + } + if prev, ok := seenNextHops[normalized]; ok { + errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalized, prev)) + } else { + seenNextHops[normalized] = i + } } for j, cidrNet := range res.ClusterNetwork { @@ -406,6 +451,7 @@ func validateRemoteCluster( errs = append(errs, validateNetworkShapeNets(res.ClusterNetwork, res.ServiceNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ClusterNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ServiceNetwork, label)...) + errs = append(errs, validateNextHopCoverage(res.NextHops, res.ClusterNetwork, res.ServiceNetwork, label)...) return errs } @@ -462,6 +508,23 @@ func validateRemoteIPFamilyCompatibility(localV4, localV6 bool, remoteCIDRs []*n return errs } +func validateNextHopCoverage(nextHops map[int]net.IP, clusterNetwork, serviceNetwork []*net.IPNet, label string) []error { + needed := make(map[int]bool) + for _, cidr := range clusterNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + for _, cidr := range serviceNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + var errs []error + for family := range needed { + if _, ok := nextHops[family]; !ok { + errs = append(errs, fmt.Errorf("%s has %s CIDRs but no %s nextHop", label, familyName(family), familyName(family))) + } + } + return errs +} + func checkCIDRConflicts(cidr *net.IPNet, cidrStr, label string, seenCIDRs []labeledCIDR, hostIPs []net.IP) []error { var errs []error for _, existing := range seenCIDRs { diff --git a/packaging/microshift/config.yaml b/packaging/microshift/config.yaml index bd42e27fe4..653bbc8e3b 100644 --- a/packaging/microshift/config.yaml +++ b/packaging/microshift/config.yaml @@ -62,8 +62,9 @@ clusterToCluster: # Services are reachable as ..svc.. # Optional — if empty, no DNS forwarding is configured for this remote. domain: "" - # IP address of the remote cluster's node, used as next-hop for routing. - nextHop: "" + # IP addresses of the remote cluster's node, used as next-hop for routing. + # At most one IPv4 and one IPv6 address. Dual-stack clusters need both. + nextHop: [] # Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. serviceNetwork: [] # Linux policy routing table settings for C2CC routes. diff --git a/pkg/config/c2cc.go b/pkg/config/c2cc.go index 6d7ee786fb..2ceca76e72 100644 --- a/pkg/config/c2cc.go +++ b/pkg/config/c2cc.go @@ -68,8 +68,9 @@ type C2CC struct { } type RemoteCluster struct { - // IP address of the remote cluster's node, used as next-hop for routing. - NextHop string `json:"nextHop"` + // IP addresses of the remote cluster's node, used as next-hop for routing. + // At most one IPv4 and one IPv6 address. Dual-stack clusters need both. + NextHop []string `json:"nextHop"` // Pod CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. ClusterNetwork []string `json:"clusterNetwork"` // Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. @@ -80,8 +81,9 @@ type RemoteCluster struct { Domain string `json:"domain,omitempty"` } + type ResolvedRemoteCluster struct { - NextHop net.IP + NextHops map[int]net.IP // key: netlink.FAMILY_V4 or netlink.FAMILY_V6 ClusterNetwork []*net.IPNet ServiceNetwork []*net.IPNet Domain string @@ -89,6 +91,32 @@ type ResolvedRemoteCluster struct { ProbeIP string // 11th IP of ServiceNetwork[0], deterministic probe service ClusterIP } +func (rc *ResolvedRemoteCluster) NextHopForFamily(family int) (net.IP, bool) { + ip, ok := rc.NextHops[family] + return ip, ok +} + +func (rc *ResolvedRemoteCluster) PrimaryNextHop() net.IP { + if len(rc.ClusterNetwork) == 0 { + return nil + } + return rc.NextHops[ipFamilyOfIPNet(rc.ClusterNetwork[0])] +} + +func ipFamilyOfIPNet(ipNet *net.IPNet) int { + if ipNet.IP.To4() != nil { + return netlink.FAMILY_V4 + } + return netlink.FAMILY_V6 +} + +func familyName(family int) string { + if family == netlink.FAMILY_V4 { + return "IPv4" + } + return "IPv6" +} + func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet { all := make([]*net.IPNet, 0, len(rc.ClusterNetwork)+len(rc.ServiceNetwork)) all = append(all, rc.ClusterNetwork...) @@ -114,7 +142,7 @@ func (c *C2CC) AllRemoteCIDRStrings() []string { } func (rc *RemoteCluster) isEmpty() bool { - return rc.NextHop == "" && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" + return len(rc.NextHop) == 0 && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" } func (c *C2CC) stripEmptyRemoteClusters() { @@ -155,11 +183,26 @@ func (c *C2CC) parseRemoteClusters() ([]ResolvedRemoteCluster, []error) { rc := &c.RemoteClusters[i] label := fmt.Sprintf("remoteClusters[%d]", i) - ip := net.ParseIP(rc.NextHop) - if ip == nil { - errs = append(errs, fmt.Errorf("%s.nextHop %q is not a valid IP address", label, rc.NextHop)) + if len(rc.NextHop) == 0 { + errs = append(errs, fmt.Errorf("%s.nextHop must not be empty", label)) + } + resolved[i].NextHops = make(map[int]net.IP, len(rc.NextHop)) + for j, hopStr := range rc.NextHop { + ip := net.ParseIP(hopStr) + if ip == nil { + errs = append(errs, fmt.Errorf("%s.nextHop[%d] %q is not a valid IP address", label, j, hopStr)) + continue + } + family := netlink.FAMILY_V4 + if ip.To4() == nil { + family = netlink.FAMILY_V6 + } + if _, dup := resolved[i].NextHops[family]; dup { + errs = append(errs, fmt.Errorf("%s.nextHop has multiple %s addresses (max 1 per family)", label, familyName(family))) + continue + } + resolved[i].NextHops[family] = ip } - resolved[i].NextHop = ip if len(rc.ClusterNetwork) == 0 { errs = append(errs, fmt.Errorf("%s.clusterNetwork must not be empty", label)) @@ -359,14 +402,16 @@ func validateRemoteCluster( label := fmt.Sprintf("remoteClusters[%d]", i) var errs []error - normalizedNextHop := res.NextHop.String() - if res.NextHop.Equal(nodeIP) || (nodeIPv6 != nil && res.NextHop.Equal(nodeIPv6)) { - errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalizedNextHop)) - } - if prev, ok := seenNextHops[normalizedNextHop]; ok { - errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalizedNextHop, prev)) - } else { - seenNextHops[normalizedNextHop] = i + for _, hop := range res.NextHops { + normalized := hop.String() + if hop.Equal(nodeIP) || (nodeIPv6 != nil && hop.Equal(nodeIPv6)) { + errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalized)) + } + if prev, ok := seenNextHops[normalized]; ok { + errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalized, prev)) + } else { + seenNextHops[normalized] = i + } } for j, cidrNet := range res.ClusterNetwork { @@ -406,6 +451,7 @@ func validateRemoteCluster( errs = append(errs, validateNetworkShapeNets(res.ClusterNetwork, res.ServiceNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ClusterNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ServiceNetwork, label)...) + errs = append(errs, validateNextHopCoverage(res.NextHops, res.ClusterNetwork, res.ServiceNetwork, label)...) return errs } @@ -462,6 +508,23 @@ func validateRemoteIPFamilyCompatibility(localV4, localV6 bool, remoteCIDRs []*n return errs } +func validateNextHopCoverage(nextHops map[int]net.IP, clusterNetwork, serviceNetwork []*net.IPNet, label string) []error { + needed := make(map[int]bool) + for _, cidr := range clusterNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + for _, cidr := range serviceNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + var errs []error + for family := range needed { + if _, ok := nextHops[family]; !ok { + errs = append(errs, fmt.Errorf("%s has %s CIDRs but no %s nextHop", label, familyName(family), familyName(family))) + } + } + return errs +} + func checkCIDRConflicts(cidr *net.IPNet, cidrStr, label string, seenCIDRs []labeledCIDR, hostIPs []net.IP) []error { var errs []error for _, existing := range seenCIDRs { diff --git a/pkg/config/c2cc_test.go b/pkg/config/c2cc_test.go index f0ea1c7852..a5db9d0fc6 100644 --- a/pkg/config/c2cc_test.go +++ b/pkg/config/c2cc_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" "k8s.io/utils/ptr" ) @@ -20,7 +21,7 @@ func TestC2CC_IsEnabled(t *testing.T) { t.Run("with remote clusters", func(t *testing.T) { c := C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -43,13 +44,13 @@ func TestC2CC_StripEmptyRemoteClusters(t *testing.T) { c := C2CC{ RemoteClusters: []RemoteCluster{ {}, - {NextHop: "10.0.0.1", ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}}, + {NextHop: []string{"10.0.0.1"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}}, {}, }, } c.stripEmptyRemoteClusters() assert.Len(t, c.RemoteClusters, 1) - assert.Equal(t, "10.0.0.1", c.RemoteClusters[0].NextHop) + assert.Equal(t, []string{"10.0.0.1"}, c.RemoteClusters[0].NextHop) }) t.Run("no-op on empty list", func(t *testing.T) { @@ -149,7 +150,7 @@ func TestC2CC_Validate(t *testing.T) { name: "valid single remote IPv4", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -160,12 +161,12 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }, { - NextHop: "10.100.0.3", + NextHop: []string{"10.100.0.3"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.56.0.0/16"}, }, @@ -176,17 +177,29 @@ func TestC2CC_Validate(t *testing.T) { name: "valid dual-stack remote with dual-stack local", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, }), }, + { + name: "dual-stack CIDRs with only IPv4 nextHop", + cfg: mkDualStackC2CCConfig(C2CC{ + RemoteClusters: []RemoteCluster{{ + NextHop: []string{"10.100.0.2"}, + ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, + ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, + }}, + }), + expectErr: true, + errMsg: "has IPv6 CIDRs but no IPv6 nextHop", + }, { name: "invalid NextHop", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "not-an-ip", + NextHop: []string{"not-an-ip"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -198,7 +211,7 @@ func TestC2CC_Validate(t *testing.T) { name: "invalid CIDR format", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"not-a-cidr"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -210,7 +223,7 @@ func TestC2CC_Validate(t *testing.T) { name: "local cluster network overlap", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.42.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -222,7 +235,7 @@ func TestC2CC_Validate(t *testing.T) { name: "local service network overlap", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.43.0.0/16"}, }}, @@ -234,7 +247,7 @@ func TestC2CC_Validate(t *testing.T) { name: "same remote clusterNetwork overlaps own serviceNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.45.0.0/16"}, }}, @@ -247,12 +260,12 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }, { - NextHop: "10.100.0.3", + NextHop: []string{"10.100.0.3"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.45.0.0/16"}, }, @@ -266,12 +279,12 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }, { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.56.0.0/16"}, }, @@ -284,7 +297,7 @@ func TestC2CC_Validate(t *testing.T) { name: "NextHop equals local node IP", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.1", + NextHop: []string{"10.100.0.1"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -296,7 +309,7 @@ func TestC2CC_Validate(t *testing.T) { name: "CIDR mask too short IPv4", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.0.0.0/4"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -308,7 +321,7 @@ func TestC2CC_Validate(t *testing.T) { name: "CIDR mask too short IPv6", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd00::/16"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, @@ -321,7 +334,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: func() *Config { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -336,7 +349,7 @@ func TestC2CC_Validate(t *testing.T) { name: "remote CIDR contains host interface IP", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.100.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -348,7 +361,7 @@ func TestC2CC_Validate(t *testing.T) { name: "multiple IPv4 entries in clusterNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16", "10.47.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -360,7 +373,7 @@ func TestC2CC_Validate(t *testing.T) { name: "multiple IPv6 entries in clusterNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"fd03::/48", "fd05::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -372,7 +385,7 @@ func TestC2CC_Validate(t *testing.T) { name: "IPv6 remote with IPv4-only local", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -384,7 +397,7 @@ func TestC2CC_Validate(t *testing.T) { name: "IPv4 remote with IPv6-only local", cfg: mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -396,7 +409,7 @@ func TestC2CC_Validate(t *testing.T) { name: "NextHop equals local node IP non-canonical form", cfg: mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00:0:0:0:0:0:0:1", + NextHop: []string{"fd00:0:0:0:0:0:0:1"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -408,7 +421,7 @@ func TestC2CC_Validate(t *testing.T) { name: "empty clusterNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -420,7 +433,7 @@ func TestC2CC_Validate(t *testing.T) { name: "empty serviceNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{}, }}, @@ -432,7 +445,7 @@ func TestC2CC_Validate(t *testing.T) { name: "invalid domain", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "not a valid domain!", @@ -445,7 +458,7 @@ func TestC2CC_Validate(t *testing.T) { name: "valid domain", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "cluster-b.remote", @@ -457,13 +470,13 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "cluster-b.remote", }, { - NextHop: "10.100.0.3", + NextHop: []string{"10.100.0.3"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.56.0.0/16"}, Domain: "cluster-b.remote", @@ -477,7 +490,7 @@ func TestC2CC_Validate(t *testing.T) { name: "NextHop equals local NodeIPV6 in dual-stack", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::1", + NextHop: []string{"fd00::1"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -489,7 +502,7 @@ func TestC2CC_Validate(t *testing.T) { name: "single-stack IPv4 remote with dual-stack local", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -499,7 +512,7 @@ func TestC2CC_Validate(t *testing.T) { name: "clusterNetwork and serviceNetwork cardinality mismatch", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -511,7 +524,7 @@ func TestC2CC_Validate(t *testing.T) { name: "clusterNetwork and serviceNetwork IP family mismatch at same index", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"fd04::/112", "10.46.0.0/16"}, }}, @@ -524,7 +537,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ DNS: C2CCDNS{CacheTTL: ptr.To(-1), CacheNegativeTTL: ptr.To(10)}, RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -537,7 +550,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ DNS: C2CCDNS{CacheTTL: ptr.To(10), CacheNegativeTTL: ptr.To(-5)}, RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -550,7 +563,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ DNS: C2CCDNS{CacheTTL: ptr.To(0), CacheNegativeTTL: ptr.To(0)}, RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -581,7 +594,7 @@ func TestC2CC_ValidateDualStack(t *testing.T) { t.Run("valid IPv6-only remote with IPv6-only local", func(t *testing.T) { cfg := mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -592,7 +605,7 @@ func TestC2CC_ValidateDualStack(t *testing.T) { t.Run("dual-stack remote with dual-stack local", func(t *testing.T) { cfg := mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, @@ -603,7 +616,7 @@ func TestC2CC_ValidateDualStack(t *testing.T) { t.Run("single-stack IPv6 remote with dual-stack local", func(t *testing.T) { cfg := mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -619,7 +632,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "500ms", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -633,7 +646,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "6m", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -647,7 +660,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "not-a-duration", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -661,7 +674,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "1s", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -673,7 +686,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "5m", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -685,7 +698,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "30s", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -701,7 +714,7 @@ func TestC2CC_ProbeIP(t *testing.T) { t.Run("IPv4 service network", func(t *testing.T) { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -714,7 +727,7 @@ func TestC2CC_ProbeIP(t *testing.T) { t.Run("IPv6 service network", func(t *testing.T) { cfg := mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -727,7 +740,7 @@ func TestC2CC_ProbeIP(t *testing.T) { t.Run("dual-stack uses first service network", func(t *testing.T) { cfg := mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, @@ -744,7 +757,7 @@ func TestC2CC_DNSIP(t *testing.T) { t.Run("DNSIP populated when domain is set", func(t *testing.T) { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "cluster-b.remote", @@ -757,7 +770,7 @@ func TestC2CC_DNSIP(t *testing.T) { t.Run("DNSIP empty when domain is not set", func(t *testing.T) { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -769,7 +782,7 @@ func TestC2CC_DNSIP(t *testing.T) { t.Run("DNSIP for IPv6 service network", func(t *testing.T) { cfg := mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, Domain: "cluster-b.remote", @@ -790,7 +803,7 @@ func parseCIDR(t *testing.T, s string) *net.IPNet { func TestRenderC2CCDNSBlocks(t *testing.T) { t.Run("no domains configured", func(t *testing.T) { resolved := []ResolvedRemoteCluster{{ - NextHop: net.ParseIP("10.100.0.2"), + NextHops: map[int]net.IP{netlink.FAMILY_V4: net.ParseIP("10.100.0.2")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, }} @@ -800,7 +813,7 @@ func TestRenderC2CCDNSBlocks(t *testing.T) { t.Run("single domain with default TTLs", func(t *testing.T) { resolved := []ResolvedRemoteCluster{{ - NextHop: net.ParseIP("10.100.0.2"), + NextHops: map[int]net.IP{netlink.FAMILY_V4: net.ParseIP("10.100.0.2")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, Domain: "cluster-b.remote", @@ -873,7 +886,7 @@ func TestC2CC_RoutingTableValidation(t *testing.T) { stubHostIPs(t, nil) validRemote := []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }} @@ -1005,7 +1018,7 @@ func TestC2CC_IncorporateUserSettings(t *testing.T) { user := &Config{ C2CC: C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, diff --git a/pkg/controllers/c2cc/healthcheck.go b/pkg/controllers/c2cc/healthcheck.go index b27c3b3a35..b6fa1a8cf6 100644 --- a/pkg/controllers/c2cc/healthcheck.go +++ b/pkg/controllers/c2cc/healthcheck.go @@ -88,7 +88,7 @@ func (h *healthcheckCRManager) reconcile(ctx context.Context) error { func (h *healthcheckCRManager) buildDesiredCRs() map[string]*microshiftv1alpha1.RemoteCluster { desired := make(map[string]*microshiftv1alpha1.RemoteCluster, len(h.cfg.C2CC.Resolved)) for _, rc := range h.cfg.C2CC.Resolved { - name := crNameForRemote(rc.NextHop) + name := crNameForRemote(rc.PrimaryNextHop()) desired[name] = µshiftv1alpha1.RemoteCluster{ ObjectMeta: metav1.ObjectMeta{ Name: name, diff --git a/pkg/controllers/c2cc/healthcheck_test.go b/pkg/controllers/c2cc/healthcheck_test.go index febf2389ba..a1dd30b82a 100644 --- a/pkg/controllers/c2cc/healthcheck_test.go +++ b/pkg/controllers/c2cc/healthcheck_test.go @@ -41,12 +41,14 @@ func TestBuildDesiredCRs(t *testing.T) { ResolvedProbeInterval: 15 * time.Second, Resolved: []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("10.100.0.2"), - ProbeIP: "10.46.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.2")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, + ProbeIP: "10.46.0.11", }, { - NextHop: net.ParseIP("10.100.0.3"), - ProbeIP: "10.47.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.3")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.55.0.0/16")}, + ProbeIP: "10.47.0.11", }, }, }, @@ -80,8 +82,9 @@ func TestReconcileCreatesNewCRs(t *testing.T) { ResolvedProbeInterval: 10 * time.Second, Resolved: []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("10.100.0.2"), - ProbeIP: "10.46.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.2")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, + ProbeIP: "10.46.0.11", }, }, }, @@ -154,8 +157,9 @@ func TestReconcileUpdatesCR(t *testing.T) { ResolvedProbeInterval: 15 * time.Second, Resolved: []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("10.100.0.2"), - ProbeIP: "10.46.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.2")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, + ProbeIP: "10.46.0.11", }, }, }, diff --git a/pkg/controllers/c2cc/helpers_test.go b/pkg/controllers/c2cc/helpers_test.go index b00f12bfcc..bf7e576048 100644 --- a/pkg/controllers/c2cc/helpers_test.go +++ b/pkg/controllers/c2cc/helpers_test.go @@ -9,7 +9,7 @@ import ( ) type testRemoteConfig struct { - nextHop string + nextHops []string clusterNetwork []string serviceNetwork []string domain string @@ -17,7 +17,7 @@ type testRemoteConfig struct { func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRemoteConfig { return testRemoteConfig{ - nextHop: nextHop, + nextHops: []string{nextHop}, clusterNetwork: clusterNetwork, serviceNetwork: serviceNetwork, } @@ -25,13 +25,36 @@ func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRem func testRemoteWithDomain(nextHop string, clusterNetwork, serviceNetwork []string, domain string) testRemoteConfig { return testRemoteConfig{ - nextHop: nextHop, + nextHops: []string{nextHop}, clusterNetwork: clusterNetwork, serviceNetwork: serviceNetwork, domain: domain, } } +func testDualStackRemote(nextHops []string, clusterNetwork, serviceNetwork []string) testRemoteConfig { + return testRemoteConfig{ + nextHops: nextHops, + clusterNetwork: clusterNetwork, + serviceNetwork: serviceNetwork, + } +} + +func parseNextHops(t *testing.T, hops []string) map[int]net.IP { + t.Helper() + m := make(map[int]net.IP, len(hops)) + for _, h := range hops { + ip := net.ParseIP(h) + require.NotNil(t, ip, "invalid nextHop: %s", h) + family := 2 // FAMILY_V4 + if ip.To4() == nil { + family = 10 // FAMILY_V6 + } + m[family] = ip + } + return m +} + func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Config { t.Helper() @@ -41,10 +64,9 @@ func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Co for _, r := range remotes { resolved := config.ResolvedRemoteCluster{ - NextHop: net.ParseIP(r.nextHop), - Domain: r.domain, + NextHops: parseNextHops(t, r.nextHops), + Domain: r.domain, } - require.NotNil(t, resolved.NextHop, "invalid nextHop: %s", r.nextHop) for _, cn := range r.clusterNetwork { _, ipNet, err := net.ParseCIDR(cn) @@ -56,6 +78,7 @@ func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Co require.NoError(t, err) resolved.ServiceNetwork = append(resolved.ServiceNetwork, ipNet) } + require.NotNil(t, resolved.PrimaryNextHop(), "no valid nextHops in: %v", r.nextHops) cfg.C2CC.Resolved = append(cfg.C2CC.Resolved, resolved) cfg.C2CC.ResolvedAllCIDRs = append(cfg.C2CC.ResolvedAllCIDRs, resolved.AllCIDRs()...) } diff --git a/pkg/controllers/c2cc/ovn.go b/pkg/controllers/c2cc/ovn.go index 2d27ab693b..063b891f89 100644 --- a/pkg/controllers/c2cc/ovn.go +++ b/pkg/controllers/c2cc/ovn.go @@ -42,11 +42,15 @@ func newOVNRouteManager(nbClient client.Client, nodeName string, resolved []conf var desired []LogicalRouterStaticRoute for i := range resolved { - nexthop := resolved[i].NextHop.String() - for _, cidr := range resolved[i].AllCIDRs() { + rc := &resolved[i] + for _, cidr := range rc.AllCIDRs() { + gw, ok := rc.NextHopForFamily(ipFamilyOf(cidr)) + if !ok { + continue + } desired = append(desired, LogicalRouterStaticRoute{ IPPrefix: cidr.String(), - Nexthop: nexthop, + Nexthop: gw.String(), ExternalIDs: map[string]string{ownerControllerKey: c2ccOwnerController}, }) } diff --git a/pkg/controllers/c2cc/ovn_test.go b/pkg/controllers/c2cc/ovn_test.go index 9ba1011e26..4e27df8d44 100644 --- a/pkg/controllers/c2cc/ovn_test.go +++ b/pkg/controllers/c2cc/ovn_test.go @@ -12,7 +12,7 @@ import ( func TestNewOVNRouteManager_DesiredRoutes(t *testing.T) { resolved := []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("192.168.1.10"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.10")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, }, @@ -34,12 +34,12 @@ func TestNewOVNRouteManager_DesiredRoutes(t *testing.T) { func TestNewOVNRouteManager_MultipleRemotes(t *testing.T) { resolved := []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("192.168.1.10"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.10")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, }, { - NextHop: net.ParseIP("192.168.1.20"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.20")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.55.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.56.0.0/16")}, }, @@ -69,7 +69,7 @@ func TestNewOVNRouteManager_EmptyResolved(t *testing.T) { func TestOVNRouteManager_OwnerTag(t *testing.T) { resolved := []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("192.168.1.10"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.10")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, }, } diff --git a/pkg/controllers/c2cc/routes.go b/pkg/controllers/c2cc/routes.go index bd1b542792..355d6032fd 100644 --- a/pkg/controllers/c2cc/routes.go +++ b/pkg/controllers/c2cc/routes.go @@ -19,8 +19,9 @@ const ( type linuxRouteManager struct { policyRouteTable - desiredDsts []*net.IPNet - desiredGWs map[string]net.IP + desiredDsts []*net.IPNet + desiredDstKeys []string + desiredGWs map[string]net.IP } func newLinuxRouteManager(cfg *config.Config) *linuxRouteManager { @@ -34,9 +35,16 @@ func newLinuxRouteManager(cfg *config.Config) *linuxRouteManager { } for i := range cfg.C2CC.Resolved { - for _, cidr := range cfg.C2CC.Resolved[i].AllCIDRs() { + rc := &cfg.C2CC.Resolved[i] + for _, cidr := range rc.AllCIDRs() { + gw, ok := rc.NextHopForFamily(ipFamilyOf(cidr)) + if !ok { + continue + } + key := cidr.String() m.desiredDsts = append(m.desiredDsts, cidr) - m.desiredGWs[cidr.String()] = cfg.C2CC.Resolved[i].NextHop + m.desiredDstKeys = append(m.desiredDstKeys, key) + m.desiredGWs[key] = gw } } @@ -45,10 +53,10 @@ func newLinuxRouteManager(cfg *config.Config) *linuxRouteManager { func (m *linuxRouteManager) reconcile(ctx context.Context) error { desired := make([]netlink.Route, 0, len(m.desiredDsts)) - for _, cidr := range m.desiredDsts { + for i, cidr := range m.desiredDsts { desired = append(desired, netlink.Route{ Dst: cidr, - Gw: m.desiredGWs[cidr.String()], + Gw: m.desiredGWs[m.desiredDstKeys[i]], Table: m.table, Protocol: netlink.RouteProtocol(m.proto), }) @@ -77,8 +85,8 @@ func (m *linuxRouteManager) reconcileRules() error { } var errs []error - for _, cidr := range m.desiredDsts { - dst := cidr.String() + for i, cidr := range m.desiredDsts { + dst := m.desiredDstKeys[i] if _, exists := actualByDst[dst]; exists { delete(actualByDst, dst) continue diff --git a/test/assets/c2cc/hello-microshift.yaml b/test/assets/c2cc/hello-microshift.yaml index 34ffcb717f..90f9064364 100644 --- a/test/assets/c2cc/hello-microshift.yaml +++ b/test/assets/c2cc/hello-microshift.yaml @@ -49,6 +49,7 @@ metadata: labels: app: hello-microshift spec: + ipFamilyPolicy: PreferDualStack selector: app: hello-microshift ports: diff --git a/test/bin/c2cc_common.sh b/test/bin/c2cc_common.sh index c04024bb73..b8f07a6ed1 100644 --- a/test/bin/c2cc_common.sh +++ b/test/bin/c2cc_common.sh @@ -17,6 +17,14 @@ CLUSTER_C_POD_CIDR="10.48.0.0/16" CLUSTER_C_SVC_CIDR="10.49.0.0/16" CLUSTER_C_DOMAIN="cluster-c.remote" +# Dual-stack secondary CIDRs (empty unless overridden by scenario) +CLUSTER_A_POD_CIDR_DUAL="" +CLUSTER_A_SVC_CIDR_DUAL="" +CLUSTER_B_POD_CIDR_DUAL="" +CLUSTER_B_SVC_CIDR_DUAL="" +CLUSTER_C_POD_CIDR_DUAL="" +CLUSTER_C_SVC_CIDR_DUAL="" + export TEST_RANDOMIZATION=suites get_host_ip() { @@ -24,6 +32,16 @@ get_host_ip() { get_vm_property "${host}" ip || { echo "failed to get ${host} ip" >&2; return 1; } } +get_host_ipv6() { + local host=$1 + local host_ip + host_ip=$(get_host_ip "${host}") || return 1 + # Get the global-scope IPv6 address from the VM (excluding link-local fe80::) + run_command_on_vm "${host}" \ + "ip -6 addr show scope global | grep -oP '(?<=inet6 )([0-9a-f:]+)' | head -1" \ + | tail -1 | tr -d '\r' +} + wait_for_greenboot_on_hosts() { local junit_label=$1 local host @@ -42,7 +60,9 @@ wait_for_greenboot_on_hosts() { configure_c2cc_host() { local host=$1 shift - # Remaining args are sets of 4: remote_ip remote_pod_cidr remote_svc_cidr remote_domain (repeat) + # Remaining args are sets of 7: + # remote_ip remote_ipv6 remote_pod_cidr remote_svc_cidr remote_domain remote_pod_cidr_dual remote_svc_cidr_dual + # remote_ipv6 and the last two may be empty for single-stack scenarios. run_command_on_vm "${host}" "sudo mkdir -p /etc/microshift/config.d" @@ -53,19 +73,38 @@ configure_c2cc_host() { while [ $# -gt 0 ]; do local remote_ip=$1 - local remote_pod_cidr=$2 - local remote_svc_cidr=$3 - local remote_domain=$4 - shift 4 - - yaml_content+=$'\n'" - nextHop: ${remote_ip}" + local remote_ipv6=${2:-} + local remote_pod_cidr=$3 + local remote_svc_cidr=$4 + local remote_domain=$5 + local remote_pod_cidr_dual=${6:-} + local remote_svc_cidr_dual=${7:-} + shift 7 + + yaml_content+=$'\n'" - nextHop:" + yaml_content+=$'\n'" - ${remote_ip}" + if [ -n "${remote_ipv6}" ]; then + yaml_content+=$'\n'" - ${remote_ipv6}" + fi yaml_content+=$'\n'" clusterNetwork:" yaml_content+=$'\n'" - ${remote_pod_cidr}" + if [ -n "${remote_pod_cidr_dual}" ]; then + yaml_content+=$'\n'" - ${remote_pod_cidr_dual}" + fi yaml_content+=$'\n'" serviceNetwork:" yaml_content+=$'\n'" - ${remote_svc_cidr}" + if [ -n "${remote_svc_cidr_dual}" ]; then + yaml_content+=$'\n'" - ${remote_svc_cidr_dual}" + fi yaml_content+=$'\n'" domain: ${remote_domain}" firewall_cidrs+=("${remote_pod_cidr}" "${remote_svc_cidr}") + if [ -n "${remote_pod_cidr_dual}" ]; then + firewall_cidrs+=("${remote_pod_cidr_dual}") + fi + if [ -n "${remote_svc_cidr_dual}" ]; then + firewall_cidrs+=("${remote_svc_cidr_dual}") + fi done run_command_on_vm "${host}" "sudo tee /etc/microshift/config.d/50-c2cc.yaml > /dev/null <> "${ks_dir}/post-microshift.cfg" <>/etc/microshift/config.yaml <> "${host2_ks_dir}/post-microshift.cfg" <>/etc/microshift/config.yaml <> "${host3_ks_dir}/post-microshift.cfg" <>/etc/microshift/config.yaml <