From fc15ef8cf4ee5cace5cf1b81034aca0145c6e976 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 26 Jun 2026 14:09:26 +0200 Subject: [PATCH 1/7] Extend NextHop to be a slice --- .../config/config-openapi-spec.json | 7 +- docs/user/howto_config.md | 4 +- .../openshift/microshift/pkg/config/c2cc.go | 95 ++++++++++--- packaging/microshift/config.yaml | 5 +- pkg/config/c2cc.go | 95 ++++++++++--- pkg/config/c2cc_test.go | 129 ++++++++++-------- 6 files changed, 239 insertions(+), 96 deletions(-) diff --git a/cmd/generate-config/config/config-openapi-spec.json b/cmd/generate-config/config/config-openapi-spec.json index d303b11e96..803f737ccf 100755 --- a/cmd/generate-config/config/config-openapi-spec.json +++ b/cmd/generate-config/config/config-openapi-spec.json @@ -232,8 +232,11 @@ "type": "string" }, "nextHop": { - "description": "IP address of the remote cluster's node, used as next-hop for routing.", - "type": "string" + "description": "IP addresses of the remote cluster's node, used as next-hop for routing.\nAt most one IPv4 and one IPv6 address. Dual-stack clusters need both.", + "type": "array", + "items": { + "type": "string" + } }, "serviceNetwork": { "description": "Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes.", diff --git a/docs/user/howto_config.md b/docs/user/howto_config.md index 1e7fd62732..2ae85bb83e 100644 --- a/docs/user/howto_config.md +++ b/docs/user/howto_config.md @@ -38,7 +38,7 @@ clusterToCluster: remoteClusters: - clusterNetwork: [] domain: "" - nextHop: "" + nextHop: [] serviceNetwork: [] routing: routeTableID: 0 @@ -207,7 +207,7 @@ clusterToCluster: remoteClusters: - clusterNetwork: [] domain: "" - nextHop: "" + nextHop: [] serviceNetwork: [] routing: routeTableID: 200 diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go index 6d7ee786fb..2ceca76e72 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/c2cc.go @@ -68,8 +68,9 @@ type C2CC struct { } type RemoteCluster struct { - // IP address of the remote cluster's node, used as next-hop for routing. - NextHop string `json:"nextHop"` + // IP addresses of the remote cluster's node, used as next-hop for routing. + // At most one IPv4 and one IPv6 address. Dual-stack clusters need both. + NextHop []string `json:"nextHop"` // Pod CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. ClusterNetwork []string `json:"clusterNetwork"` // Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. @@ -80,8 +81,9 @@ type RemoteCluster struct { Domain string `json:"domain,omitempty"` } + type ResolvedRemoteCluster struct { - NextHop net.IP + NextHops map[int]net.IP // key: netlink.FAMILY_V4 or netlink.FAMILY_V6 ClusterNetwork []*net.IPNet ServiceNetwork []*net.IPNet Domain string @@ -89,6 +91,32 @@ type ResolvedRemoteCluster struct { ProbeIP string // 11th IP of ServiceNetwork[0], deterministic probe service ClusterIP } +func (rc *ResolvedRemoteCluster) NextHopForFamily(family int) (net.IP, bool) { + ip, ok := rc.NextHops[family] + return ip, ok +} + +func (rc *ResolvedRemoteCluster) PrimaryNextHop() net.IP { + if len(rc.ClusterNetwork) == 0 { + return nil + } + return rc.NextHops[ipFamilyOfIPNet(rc.ClusterNetwork[0])] +} + +func ipFamilyOfIPNet(ipNet *net.IPNet) int { + if ipNet.IP.To4() != nil { + return netlink.FAMILY_V4 + } + return netlink.FAMILY_V6 +} + +func familyName(family int) string { + if family == netlink.FAMILY_V4 { + return "IPv4" + } + return "IPv6" +} + func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet { all := make([]*net.IPNet, 0, len(rc.ClusterNetwork)+len(rc.ServiceNetwork)) all = append(all, rc.ClusterNetwork...) @@ -114,7 +142,7 @@ func (c *C2CC) AllRemoteCIDRStrings() []string { } func (rc *RemoteCluster) isEmpty() bool { - return rc.NextHop == "" && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" + return len(rc.NextHop) == 0 && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" } func (c *C2CC) stripEmptyRemoteClusters() { @@ -155,11 +183,26 @@ func (c *C2CC) parseRemoteClusters() ([]ResolvedRemoteCluster, []error) { rc := &c.RemoteClusters[i] label := fmt.Sprintf("remoteClusters[%d]", i) - ip := net.ParseIP(rc.NextHop) - if ip == nil { - errs = append(errs, fmt.Errorf("%s.nextHop %q is not a valid IP address", label, rc.NextHop)) + if len(rc.NextHop) == 0 { + errs = append(errs, fmt.Errorf("%s.nextHop must not be empty", label)) + } + resolved[i].NextHops = make(map[int]net.IP, len(rc.NextHop)) + for j, hopStr := range rc.NextHop { + ip := net.ParseIP(hopStr) + if ip == nil { + errs = append(errs, fmt.Errorf("%s.nextHop[%d] %q is not a valid IP address", label, j, hopStr)) + continue + } + family := netlink.FAMILY_V4 + if ip.To4() == nil { + family = netlink.FAMILY_V6 + } + if _, dup := resolved[i].NextHops[family]; dup { + errs = append(errs, fmt.Errorf("%s.nextHop has multiple %s addresses (max 1 per family)", label, familyName(family))) + continue + } + resolved[i].NextHops[family] = ip } - resolved[i].NextHop = ip if len(rc.ClusterNetwork) == 0 { errs = append(errs, fmt.Errorf("%s.clusterNetwork must not be empty", label)) @@ -359,14 +402,16 @@ func validateRemoteCluster( label := fmt.Sprintf("remoteClusters[%d]", i) var errs []error - normalizedNextHop := res.NextHop.String() - if res.NextHop.Equal(nodeIP) || (nodeIPv6 != nil && res.NextHop.Equal(nodeIPv6)) { - errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalizedNextHop)) - } - if prev, ok := seenNextHops[normalizedNextHop]; ok { - errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalizedNextHop, prev)) - } else { - seenNextHops[normalizedNextHop] = i + for _, hop := range res.NextHops { + normalized := hop.String() + if hop.Equal(nodeIP) || (nodeIPv6 != nil && hop.Equal(nodeIPv6)) { + errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalized)) + } + if prev, ok := seenNextHops[normalized]; ok { + errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalized, prev)) + } else { + seenNextHops[normalized] = i + } } for j, cidrNet := range res.ClusterNetwork { @@ -406,6 +451,7 @@ func validateRemoteCluster( errs = append(errs, validateNetworkShapeNets(res.ClusterNetwork, res.ServiceNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ClusterNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ServiceNetwork, label)...) + errs = append(errs, validateNextHopCoverage(res.NextHops, res.ClusterNetwork, res.ServiceNetwork, label)...) return errs } @@ -462,6 +508,23 @@ func validateRemoteIPFamilyCompatibility(localV4, localV6 bool, remoteCIDRs []*n return errs } +func validateNextHopCoverage(nextHops map[int]net.IP, clusterNetwork, serviceNetwork []*net.IPNet, label string) []error { + needed := make(map[int]bool) + for _, cidr := range clusterNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + for _, cidr := range serviceNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + var errs []error + for family := range needed { + if _, ok := nextHops[family]; !ok { + errs = append(errs, fmt.Errorf("%s has %s CIDRs but no %s nextHop", label, familyName(family), familyName(family))) + } + } + return errs +} + func checkCIDRConflicts(cidr *net.IPNet, cidrStr, label string, seenCIDRs []labeledCIDR, hostIPs []net.IP) []error { var errs []error for _, existing := range seenCIDRs { diff --git a/packaging/microshift/config.yaml b/packaging/microshift/config.yaml index bd42e27fe4..653bbc8e3b 100644 --- a/packaging/microshift/config.yaml +++ b/packaging/microshift/config.yaml @@ -62,8 +62,9 @@ clusterToCluster: # Services are reachable as ..svc.. # Optional — if empty, no DNS forwarding is configured for this remote. domain: "" - # IP address of the remote cluster's node, used as next-hop for routing. - nextHop: "" + # IP addresses of the remote cluster's node, used as next-hop for routing. + # At most one IPv4 and one IPv6 address. Dual-stack clusters need both. + nextHop: [] # Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. serviceNetwork: [] # Linux policy routing table settings for C2CC routes. diff --git a/pkg/config/c2cc.go b/pkg/config/c2cc.go index 6d7ee786fb..2ceca76e72 100644 --- a/pkg/config/c2cc.go +++ b/pkg/config/c2cc.go @@ -68,8 +68,9 @@ type C2CC struct { } type RemoteCluster struct { - // IP address of the remote cluster's node, used as next-hop for routing. - NextHop string `json:"nextHop"` + // IP addresses of the remote cluster's node, used as next-hop for routing. + // At most one IPv4 and one IPv6 address. Dual-stack clusters need both. + NextHop []string `json:"nextHop"` // Pod CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. ClusterNetwork []string `json:"clusterNetwork"` // Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. @@ -80,8 +81,9 @@ type RemoteCluster struct { Domain string `json:"domain,omitempty"` } + type ResolvedRemoteCluster struct { - NextHop net.IP + NextHops map[int]net.IP // key: netlink.FAMILY_V4 or netlink.FAMILY_V6 ClusterNetwork []*net.IPNet ServiceNetwork []*net.IPNet Domain string @@ -89,6 +91,32 @@ type ResolvedRemoteCluster struct { ProbeIP string // 11th IP of ServiceNetwork[0], deterministic probe service ClusterIP } +func (rc *ResolvedRemoteCluster) NextHopForFamily(family int) (net.IP, bool) { + ip, ok := rc.NextHops[family] + return ip, ok +} + +func (rc *ResolvedRemoteCluster) PrimaryNextHop() net.IP { + if len(rc.ClusterNetwork) == 0 { + return nil + } + return rc.NextHops[ipFamilyOfIPNet(rc.ClusterNetwork[0])] +} + +func ipFamilyOfIPNet(ipNet *net.IPNet) int { + if ipNet.IP.To4() != nil { + return netlink.FAMILY_V4 + } + return netlink.FAMILY_V6 +} + +func familyName(family int) string { + if family == netlink.FAMILY_V4 { + return "IPv4" + } + return "IPv6" +} + func (rc *ResolvedRemoteCluster) AllCIDRs() []*net.IPNet { all := make([]*net.IPNet, 0, len(rc.ClusterNetwork)+len(rc.ServiceNetwork)) all = append(all, rc.ClusterNetwork...) @@ -114,7 +142,7 @@ func (c *C2CC) AllRemoteCIDRStrings() []string { } func (rc *RemoteCluster) isEmpty() bool { - return rc.NextHop == "" && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" + return len(rc.NextHop) == 0 && len(rc.ClusterNetwork) == 0 && len(rc.ServiceNetwork) == 0 && rc.Domain == "" } func (c *C2CC) stripEmptyRemoteClusters() { @@ -155,11 +183,26 @@ func (c *C2CC) parseRemoteClusters() ([]ResolvedRemoteCluster, []error) { rc := &c.RemoteClusters[i] label := fmt.Sprintf("remoteClusters[%d]", i) - ip := net.ParseIP(rc.NextHop) - if ip == nil { - errs = append(errs, fmt.Errorf("%s.nextHop %q is not a valid IP address", label, rc.NextHop)) + if len(rc.NextHop) == 0 { + errs = append(errs, fmt.Errorf("%s.nextHop must not be empty", label)) + } + resolved[i].NextHops = make(map[int]net.IP, len(rc.NextHop)) + for j, hopStr := range rc.NextHop { + ip := net.ParseIP(hopStr) + if ip == nil { + errs = append(errs, fmt.Errorf("%s.nextHop[%d] %q is not a valid IP address", label, j, hopStr)) + continue + } + family := netlink.FAMILY_V4 + if ip.To4() == nil { + family = netlink.FAMILY_V6 + } + if _, dup := resolved[i].NextHops[family]; dup { + errs = append(errs, fmt.Errorf("%s.nextHop has multiple %s addresses (max 1 per family)", label, familyName(family))) + continue + } + resolved[i].NextHops[family] = ip } - resolved[i].NextHop = ip if len(rc.ClusterNetwork) == 0 { errs = append(errs, fmt.Errorf("%s.clusterNetwork must not be empty", label)) @@ -359,14 +402,16 @@ func validateRemoteCluster( label := fmt.Sprintf("remoteClusters[%d]", i) var errs []error - normalizedNextHop := res.NextHop.String() - if res.NextHop.Equal(nodeIP) || (nodeIPv6 != nil && res.NextHop.Equal(nodeIPv6)) { - errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalizedNextHop)) - } - if prev, ok := seenNextHops[normalizedNextHop]; ok { - errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalizedNextHop, prev)) - } else { - seenNextHops[normalizedNextHop] = i + for _, hop := range res.NextHops { + normalized := hop.String() + if hop.Equal(nodeIP) || (nodeIPv6 != nil && hop.Equal(nodeIPv6)) { + errs = append(errs, fmt.Errorf("%s.nextHop %q must not equal the local node IP (routing loop)", label, normalized)) + } + if prev, ok := seenNextHops[normalized]; ok { + errs = append(errs, fmt.Errorf("%s.nextHop %q duplicates remoteClusters[%d]", label, normalized, prev)) + } else { + seenNextHops[normalized] = i + } } for j, cidrNet := range res.ClusterNetwork { @@ -406,6 +451,7 @@ func validateRemoteCluster( errs = append(errs, validateNetworkShapeNets(res.ClusterNetwork, res.ServiceNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ClusterNetwork, label)...) errs = append(errs, validateRemoteIPFamilyCompatibility(localV4, localV6, res.ServiceNetwork, label)...) + errs = append(errs, validateNextHopCoverage(res.NextHops, res.ClusterNetwork, res.ServiceNetwork, label)...) return errs } @@ -462,6 +508,23 @@ func validateRemoteIPFamilyCompatibility(localV4, localV6 bool, remoteCIDRs []*n return errs } +func validateNextHopCoverage(nextHops map[int]net.IP, clusterNetwork, serviceNetwork []*net.IPNet, label string) []error { + needed := make(map[int]bool) + for _, cidr := range clusterNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + for _, cidr := range serviceNetwork { + needed[ipFamilyOfIPNet(cidr)] = true + } + var errs []error + for family := range needed { + if _, ok := nextHops[family]; !ok { + errs = append(errs, fmt.Errorf("%s has %s CIDRs but no %s nextHop", label, familyName(family), familyName(family))) + } + } + return errs +} + func checkCIDRConflicts(cidr *net.IPNet, cidrStr, label string, seenCIDRs []labeledCIDR, hostIPs []net.IP) []error { var errs []error for _, existing := range seenCIDRs { diff --git a/pkg/config/c2cc_test.go b/pkg/config/c2cc_test.go index f0ea1c7852..a5db9d0fc6 100644 --- a/pkg/config/c2cc_test.go +++ b/pkg/config/c2cc_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" "k8s.io/utils/ptr" ) @@ -20,7 +21,7 @@ func TestC2CC_IsEnabled(t *testing.T) { t.Run("with remote clusters", func(t *testing.T) { c := C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -43,13 +44,13 @@ func TestC2CC_StripEmptyRemoteClusters(t *testing.T) { c := C2CC{ RemoteClusters: []RemoteCluster{ {}, - {NextHop: "10.0.0.1", ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}}, + {NextHop: []string{"10.0.0.1"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}}, {}, }, } c.stripEmptyRemoteClusters() assert.Len(t, c.RemoteClusters, 1) - assert.Equal(t, "10.0.0.1", c.RemoteClusters[0].NextHop) + assert.Equal(t, []string{"10.0.0.1"}, c.RemoteClusters[0].NextHop) }) t.Run("no-op on empty list", func(t *testing.T) { @@ -149,7 +150,7 @@ func TestC2CC_Validate(t *testing.T) { name: "valid single remote IPv4", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -160,12 +161,12 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }, { - NextHop: "10.100.0.3", + NextHop: []string{"10.100.0.3"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.56.0.0/16"}, }, @@ -176,17 +177,29 @@ func TestC2CC_Validate(t *testing.T) { name: "valid dual-stack remote with dual-stack local", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, }), }, + { + name: "dual-stack CIDRs with only IPv4 nextHop", + cfg: mkDualStackC2CCConfig(C2CC{ + RemoteClusters: []RemoteCluster{{ + NextHop: []string{"10.100.0.2"}, + ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, + ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, + }}, + }), + expectErr: true, + errMsg: "has IPv6 CIDRs but no IPv6 nextHop", + }, { name: "invalid NextHop", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "not-an-ip", + NextHop: []string{"not-an-ip"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -198,7 +211,7 @@ func TestC2CC_Validate(t *testing.T) { name: "invalid CIDR format", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"not-a-cidr"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -210,7 +223,7 @@ func TestC2CC_Validate(t *testing.T) { name: "local cluster network overlap", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.42.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -222,7 +235,7 @@ func TestC2CC_Validate(t *testing.T) { name: "local service network overlap", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.43.0.0/16"}, }}, @@ -234,7 +247,7 @@ func TestC2CC_Validate(t *testing.T) { name: "same remote clusterNetwork overlaps own serviceNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.45.0.0/16"}, }}, @@ -247,12 +260,12 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }, { - NextHop: "10.100.0.3", + NextHop: []string{"10.100.0.3"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.45.0.0/16"}, }, @@ -266,12 +279,12 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }, { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.56.0.0/16"}, }, @@ -284,7 +297,7 @@ func TestC2CC_Validate(t *testing.T) { name: "NextHop equals local node IP", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.1", + NextHop: []string{"10.100.0.1"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -296,7 +309,7 @@ func TestC2CC_Validate(t *testing.T) { name: "CIDR mask too short IPv4", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.0.0.0/4"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -308,7 +321,7 @@ func TestC2CC_Validate(t *testing.T) { name: "CIDR mask too short IPv6", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd00::/16"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, @@ -321,7 +334,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: func() *Config { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -336,7 +349,7 @@ func TestC2CC_Validate(t *testing.T) { name: "remote CIDR contains host interface IP", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.100.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -348,7 +361,7 @@ func TestC2CC_Validate(t *testing.T) { name: "multiple IPv4 entries in clusterNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16", "10.47.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -360,7 +373,7 @@ func TestC2CC_Validate(t *testing.T) { name: "multiple IPv6 entries in clusterNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"fd03::/48", "fd05::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -372,7 +385,7 @@ func TestC2CC_Validate(t *testing.T) { name: "IPv6 remote with IPv4-only local", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -384,7 +397,7 @@ func TestC2CC_Validate(t *testing.T) { name: "IPv4 remote with IPv6-only local", cfg: mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -396,7 +409,7 @@ func TestC2CC_Validate(t *testing.T) { name: "NextHop equals local node IP non-canonical form", cfg: mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00:0:0:0:0:0:0:1", + NextHop: []string{"fd00:0:0:0:0:0:0:1"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -408,7 +421,7 @@ func TestC2CC_Validate(t *testing.T) { name: "empty clusterNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -420,7 +433,7 @@ func TestC2CC_Validate(t *testing.T) { name: "empty serviceNetwork", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{}, }}, @@ -432,7 +445,7 @@ func TestC2CC_Validate(t *testing.T) { name: "invalid domain", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "not a valid domain!", @@ -445,7 +458,7 @@ func TestC2CC_Validate(t *testing.T) { name: "valid domain", cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "cluster-b.remote", @@ -457,13 +470,13 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{ { - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "cluster-b.remote", }, { - NextHop: "10.100.0.3", + NextHop: []string{"10.100.0.3"}, ClusterNetwork: []string{"10.55.0.0/16"}, ServiceNetwork: []string{"10.56.0.0/16"}, Domain: "cluster-b.remote", @@ -477,7 +490,7 @@ func TestC2CC_Validate(t *testing.T) { name: "NextHop equals local NodeIPV6 in dual-stack", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::1", + NextHop: []string{"fd00::1"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -489,7 +502,7 @@ func TestC2CC_Validate(t *testing.T) { name: "single-stack IPv4 remote with dual-stack local", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -499,7 +512,7 @@ func TestC2CC_Validate(t *testing.T) { name: "clusterNetwork and serviceNetwork cardinality mismatch", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -511,7 +524,7 @@ func TestC2CC_Validate(t *testing.T) { name: "clusterNetwork and serviceNetwork IP family mismatch at same index", cfg: mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"fd04::/112", "10.46.0.0/16"}, }}, @@ -524,7 +537,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ DNS: C2CCDNS{CacheTTL: ptr.To(-1), CacheNegativeTTL: ptr.To(10)}, RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -537,7 +550,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ DNS: C2CCDNS{CacheTTL: ptr.To(10), CacheNegativeTTL: ptr.To(-5)}, RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -550,7 +563,7 @@ func TestC2CC_Validate(t *testing.T) { cfg: mkC2CCConfig(C2CC{ DNS: C2CCDNS{CacheTTL: ptr.To(0), CacheNegativeTTL: ptr.To(0)}, RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -581,7 +594,7 @@ func TestC2CC_ValidateDualStack(t *testing.T) { t.Run("valid IPv6-only remote with IPv6-only local", func(t *testing.T) { cfg := mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -592,7 +605,7 @@ func TestC2CC_ValidateDualStack(t *testing.T) { t.Run("dual-stack remote with dual-stack local", func(t *testing.T) { cfg := mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, @@ -603,7 +616,7 @@ func TestC2CC_ValidateDualStack(t *testing.T) { t.Run("single-stack IPv6 remote with dual-stack local", func(t *testing.T) { cfg := mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -619,7 +632,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "500ms", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -633,7 +646,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "6m", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -647,7 +660,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "not-a-duration", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -661,7 +674,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "1s", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -673,7 +686,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "5m", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -685,7 +698,7 @@ func TestC2CC_ProbeIntervalValidation(t *testing.T) { cfg := mkC2CCConfig(C2CC{ ProbeInterval: "30s", RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -701,7 +714,7 @@ func TestC2CC_ProbeIP(t *testing.T) { t.Run("IPv4 service network", func(t *testing.T) { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -714,7 +727,7 @@ func TestC2CC_ProbeIP(t *testing.T) { t.Run("IPv6 service network", func(t *testing.T) { cfg := mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, }}, @@ -727,7 +740,7 @@ func TestC2CC_ProbeIP(t *testing.T) { t.Run("dual-stack uses first service network", func(t *testing.T) { cfg := mkDualStackC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2", "fd00::2"}, ClusterNetwork: []string{"10.45.0.0/16", "fd03::/48"}, ServiceNetwork: []string{"10.46.0.0/16", "fd04::/112"}, }}, @@ -744,7 +757,7 @@ func TestC2CC_DNSIP(t *testing.T) { t.Run("DNSIP populated when domain is set", func(t *testing.T) { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, Domain: "cluster-b.remote", @@ -757,7 +770,7 @@ func TestC2CC_DNSIP(t *testing.T) { t.Run("DNSIP empty when domain is not set", func(t *testing.T) { cfg := mkC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, @@ -769,7 +782,7 @@ func TestC2CC_DNSIP(t *testing.T) { t.Run("DNSIP for IPv6 service network", func(t *testing.T) { cfg := mkIPv6OnlyC2CCConfig(C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "fd00::2", + NextHop: []string{"fd00::2"}, ClusterNetwork: []string{"fd03::/48"}, ServiceNetwork: []string{"fd04::/112"}, Domain: "cluster-b.remote", @@ -790,7 +803,7 @@ func parseCIDR(t *testing.T, s string) *net.IPNet { func TestRenderC2CCDNSBlocks(t *testing.T) { t.Run("no domains configured", func(t *testing.T) { resolved := []ResolvedRemoteCluster{{ - NextHop: net.ParseIP("10.100.0.2"), + NextHops: map[int]net.IP{netlink.FAMILY_V4: net.ParseIP("10.100.0.2")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, }} @@ -800,7 +813,7 @@ func TestRenderC2CCDNSBlocks(t *testing.T) { t.Run("single domain with default TTLs", func(t *testing.T) { resolved := []ResolvedRemoteCluster{{ - NextHop: net.ParseIP("10.100.0.2"), + NextHops: map[int]net.IP{netlink.FAMILY_V4: net.ParseIP("10.100.0.2")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, Domain: "cluster-b.remote", @@ -873,7 +886,7 @@ func TestC2CC_RoutingTableValidation(t *testing.T) { stubHostIPs(t, nil) validRemote := []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }} @@ -1005,7 +1018,7 @@ func TestC2CC_IncorporateUserSettings(t *testing.T) { user := &Config{ C2CC: C2CC{ RemoteClusters: []RemoteCluster{{ - NextHop: "10.100.0.2", + NextHop: []string{"10.100.0.2"}, ClusterNetwork: []string{"10.45.0.0/16"}, ServiceNetwork: []string{"10.46.0.0/16"}, }}, From 12239bb72f4e6231497c7e31156f850c6aadee3f Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 26 Jun 2026 14:10:02 +0200 Subject: [PATCH 2/7] Setup multiple next hops --- pkg/controllers/c2cc/healthcheck.go | 2 +- pkg/controllers/c2cc/healthcheck_test.go | 20 ++++++++------ pkg/controllers/c2cc/helpers_test.go | 35 ++++++++++++++++++++---- pkg/controllers/c2cc/ovn.go | 10 +++++-- pkg/controllers/c2cc/ovn_test.go | 8 +++--- pkg/controllers/c2cc/routes.go | 24 ++++++++++------ 6 files changed, 69 insertions(+), 30 deletions(-) diff --git a/pkg/controllers/c2cc/healthcheck.go b/pkg/controllers/c2cc/healthcheck.go index b27c3b3a35..b6fa1a8cf6 100644 --- a/pkg/controllers/c2cc/healthcheck.go +++ b/pkg/controllers/c2cc/healthcheck.go @@ -88,7 +88,7 @@ func (h *healthcheckCRManager) reconcile(ctx context.Context) error { func (h *healthcheckCRManager) buildDesiredCRs() map[string]*microshiftv1alpha1.RemoteCluster { desired := make(map[string]*microshiftv1alpha1.RemoteCluster, len(h.cfg.C2CC.Resolved)) for _, rc := range h.cfg.C2CC.Resolved { - name := crNameForRemote(rc.NextHop) + name := crNameForRemote(rc.PrimaryNextHop()) desired[name] = µshiftv1alpha1.RemoteCluster{ ObjectMeta: metav1.ObjectMeta{ Name: name, diff --git a/pkg/controllers/c2cc/healthcheck_test.go b/pkg/controllers/c2cc/healthcheck_test.go index febf2389ba..a1dd30b82a 100644 --- a/pkg/controllers/c2cc/healthcheck_test.go +++ b/pkg/controllers/c2cc/healthcheck_test.go @@ -41,12 +41,14 @@ func TestBuildDesiredCRs(t *testing.T) { ResolvedProbeInterval: 15 * time.Second, Resolved: []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("10.100.0.2"), - ProbeIP: "10.46.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.2")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, + ProbeIP: "10.46.0.11", }, { - NextHop: net.ParseIP("10.100.0.3"), - ProbeIP: "10.47.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.3")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.55.0.0/16")}, + ProbeIP: "10.47.0.11", }, }, }, @@ -80,8 +82,9 @@ func TestReconcileCreatesNewCRs(t *testing.T) { ResolvedProbeInterval: 10 * time.Second, Resolved: []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("10.100.0.2"), - ProbeIP: "10.46.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.2")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, + ProbeIP: "10.46.0.11", }, }, }, @@ -154,8 +157,9 @@ func TestReconcileUpdatesCR(t *testing.T) { ResolvedProbeInterval: 15 * time.Second, Resolved: []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("10.100.0.2"), - ProbeIP: "10.46.0.11", + NextHops: map[int]net.IP{2: net.ParseIP("10.100.0.2")}, + ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, + ProbeIP: "10.46.0.11", }, }, }, diff --git a/pkg/controllers/c2cc/helpers_test.go b/pkg/controllers/c2cc/helpers_test.go index b00f12bfcc..bf7e576048 100644 --- a/pkg/controllers/c2cc/helpers_test.go +++ b/pkg/controllers/c2cc/helpers_test.go @@ -9,7 +9,7 @@ import ( ) type testRemoteConfig struct { - nextHop string + nextHops []string clusterNetwork []string serviceNetwork []string domain string @@ -17,7 +17,7 @@ type testRemoteConfig struct { func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRemoteConfig { return testRemoteConfig{ - nextHop: nextHop, + nextHops: []string{nextHop}, clusterNetwork: clusterNetwork, serviceNetwork: serviceNetwork, } @@ -25,13 +25,36 @@ func testRemote(nextHop string, clusterNetwork, serviceNetwork []string) testRem func testRemoteWithDomain(nextHop string, clusterNetwork, serviceNetwork []string, domain string) testRemoteConfig { return testRemoteConfig{ - nextHop: nextHop, + nextHops: []string{nextHop}, clusterNetwork: clusterNetwork, serviceNetwork: serviceNetwork, domain: domain, } } +func testDualStackRemote(nextHops []string, clusterNetwork, serviceNetwork []string) testRemoteConfig { + return testRemoteConfig{ + nextHops: nextHops, + clusterNetwork: clusterNetwork, + serviceNetwork: serviceNetwork, + } +} + +func parseNextHops(t *testing.T, hops []string) map[int]net.IP { + t.Helper() + m := make(map[int]net.IP, len(hops)) + for _, h := range hops { + ip := net.ParseIP(h) + require.NotNil(t, ip, "invalid nextHop: %s", h) + family := 2 // FAMILY_V4 + if ip.To4() == nil { + family = 10 // FAMILY_V6 + } + m[family] = ip + } + return m +} + func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Config { t.Helper() @@ -41,10 +64,9 @@ func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Co for _, r := range remotes { resolved := config.ResolvedRemoteCluster{ - NextHop: net.ParseIP(r.nextHop), - Domain: r.domain, + NextHops: parseNextHops(t, r.nextHops), + Domain: r.domain, } - require.NotNil(t, resolved.NextHop, "invalid nextHop: %s", r.nextHop) for _, cn := range r.clusterNetwork { _, ipNet, err := net.ParseCIDR(cn) @@ -56,6 +78,7 @@ func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Co require.NoError(t, err) resolved.ServiceNetwork = append(resolved.ServiceNetwork, ipNet) } + require.NotNil(t, resolved.PrimaryNextHop(), "no valid nextHops in: %v", r.nextHops) cfg.C2CC.Resolved = append(cfg.C2CC.Resolved, resolved) cfg.C2CC.ResolvedAllCIDRs = append(cfg.C2CC.ResolvedAllCIDRs, resolved.AllCIDRs()...) } diff --git a/pkg/controllers/c2cc/ovn.go b/pkg/controllers/c2cc/ovn.go index 2d27ab693b..063b891f89 100644 --- a/pkg/controllers/c2cc/ovn.go +++ b/pkg/controllers/c2cc/ovn.go @@ -42,11 +42,15 @@ func newOVNRouteManager(nbClient client.Client, nodeName string, resolved []conf var desired []LogicalRouterStaticRoute for i := range resolved { - nexthop := resolved[i].NextHop.String() - for _, cidr := range resolved[i].AllCIDRs() { + rc := &resolved[i] + for _, cidr := range rc.AllCIDRs() { + gw, ok := rc.NextHopForFamily(ipFamilyOf(cidr)) + if !ok { + continue + } desired = append(desired, LogicalRouterStaticRoute{ IPPrefix: cidr.String(), - Nexthop: nexthop, + Nexthop: gw.String(), ExternalIDs: map[string]string{ownerControllerKey: c2ccOwnerController}, }) } diff --git a/pkg/controllers/c2cc/ovn_test.go b/pkg/controllers/c2cc/ovn_test.go index 9ba1011e26..4e27df8d44 100644 --- a/pkg/controllers/c2cc/ovn_test.go +++ b/pkg/controllers/c2cc/ovn_test.go @@ -12,7 +12,7 @@ import ( func TestNewOVNRouteManager_DesiredRoutes(t *testing.T) { resolved := []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("192.168.1.10"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.10")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, }, @@ -34,12 +34,12 @@ func TestNewOVNRouteManager_DesiredRoutes(t *testing.T) { func TestNewOVNRouteManager_MultipleRemotes(t *testing.T) { resolved := []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("192.168.1.10"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.10")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.46.0.0/16")}, }, { - NextHop: net.ParseIP("192.168.1.20"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.20")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.55.0.0/16")}, ServiceNetwork: []*net.IPNet{parseCIDR(t, "10.56.0.0/16")}, }, @@ -69,7 +69,7 @@ func TestNewOVNRouteManager_EmptyResolved(t *testing.T) { func TestOVNRouteManager_OwnerTag(t *testing.T) { resolved := []config.ResolvedRemoteCluster{ { - NextHop: net.ParseIP("192.168.1.10"), + NextHops: map[int]net.IP{2: net.ParseIP("192.168.1.10")}, ClusterNetwork: []*net.IPNet{parseCIDR(t, "10.45.0.0/16")}, }, } diff --git a/pkg/controllers/c2cc/routes.go b/pkg/controllers/c2cc/routes.go index bd1b542792..355d6032fd 100644 --- a/pkg/controllers/c2cc/routes.go +++ b/pkg/controllers/c2cc/routes.go @@ -19,8 +19,9 @@ const ( type linuxRouteManager struct { policyRouteTable - desiredDsts []*net.IPNet - desiredGWs map[string]net.IP + desiredDsts []*net.IPNet + desiredDstKeys []string + desiredGWs map[string]net.IP } func newLinuxRouteManager(cfg *config.Config) *linuxRouteManager { @@ -34,9 +35,16 @@ func newLinuxRouteManager(cfg *config.Config) *linuxRouteManager { } for i := range cfg.C2CC.Resolved { - for _, cidr := range cfg.C2CC.Resolved[i].AllCIDRs() { + rc := &cfg.C2CC.Resolved[i] + for _, cidr := range rc.AllCIDRs() { + gw, ok := rc.NextHopForFamily(ipFamilyOf(cidr)) + if !ok { + continue + } + key := cidr.String() m.desiredDsts = append(m.desiredDsts, cidr) - m.desiredGWs[cidr.String()] = cfg.C2CC.Resolved[i].NextHop + m.desiredDstKeys = append(m.desiredDstKeys, key) + m.desiredGWs[key] = gw } } @@ -45,10 +53,10 @@ func newLinuxRouteManager(cfg *config.Config) *linuxRouteManager { func (m *linuxRouteManager) reconcile(ctx context.Context) error { desired := make([]netlink.Route, 0, len(m.desiredDsts)) - for _, cidr := range m.desiredDsts { + for i, cidr := range m.desiredDsts { desired = append(desired, netlink.Route{ Dst: cidr, - Gw: m.desiredGWs[cidr.String()], + Gw: m.desiredGWs[m.desiredDstKeys[i]], Table: m.table, Protocol: netlink.RouteProtocol(m.proto), }) @@ -77,8 +85,8 @@ func (m *linuxRouteManager) reconcileRules() error { } var errs []error - for _, cidr := range m.desiredDsts { - dst := cidr.String() + for i, cidr := range m.desiredDsts { + dst := m.desiredDstKeys[i] if _, exists := actualByDst[dst]; exists { delete(actualByDst, dst) continue From 3b7ac86656d08200b38ae2085aaa685bf2df129c Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 25 Jun 2026 11:58:58 +0200 Subject: [PATCH 3/7] Prepare c2cc_common.sh for dualstack scenario --- test/bin/c2cc_common.sh | 159 +++++++++++++++++++++++++++++++--------- 1 file changed, 124 insertions(+), 35 deletions(-) diff --git a/test/bin/c2cc_common.sh b/test/bin/c2cc_common.sh index c04024bb73..b8f07a6ed1 100644 --- a/test/bin/c2cc_common.sh +++ b/test/bin/c2cc_common.sh @@ -17,6 +17,14 @@ CLUSTER_C_POD_CIDR="10.48.0.0/16" CLUSTER_C_SVC_CIDR="10.49.0.0/16" CLUSTER_C_DOMAIN="cluster-c.remote" +# Dual-stack secondary CIDRs (empty unless overridden by scenario) +CLUSTER_A_POD_CIDR_DUAL="" +CLUSTER_A_SVC_CIDR_DUAL="" +CLUSTER_B_POD_CIDR_DUAL="" +CLUSTER_B_SVC_CIDR_DUAL="" +CLUSTER_C_POD_CIDR_DUAL="" +CLUSTER_C_SVC_CIDR_DUAL="" + export TEST_RANDOMIZATION=suites get_host_ip() { @@ -24,6 +32,16 @@ get_host_ip() { get_vm_property "${host}" ip || { echo "failed to get ${host} ip" >&2; return 1; } } +get_host_ipv6() { + local host=$1 + local host_ip + host_ip=$(get_host_ip "${host}") || return 1 + # Get the global-scope IPv6 address from the VM (excluding link-local fe80::) + run_command_on_vm "${host}" \ + "ip -6 addr show scope global | grep -oP '(?<=inet6 )([0-9a-f:]+)' | head -1" \ + | tail -1 | tr -d '\r' +} + wait_for_greenboot_on_hosts() { local junit_label=$1 local host @@ -42,7 +60,9 @@ wait_for_greenboot_on_hosts() { configure_c2cc_host() { local host=$1 shift - # Remaining args are sets of 4: remote_ip remote_pod_cidr remote_svc_cidr remote_domain (repeat) + # Remaining args are sets of 7: + # remote_ip remote_ipv6 remote_pod_cidr remote_svc_cidr remote_domain remote_pod_cidr_dual remote_svc_cidr_dual + # remote_ipv6 and the last two may be empty for single-stack scenarios. run_command_on_vm "${host}" "sudo mkdir -p /etc/microshift/config.d" @@ -53,19 +73,38 @@ configure_c2cc_host() { while [ $# -gt 0 ]; do local remote_ip=$1 - local remote_pod_cidr=$2 - local remote_svc_cidr=$3 - local remote_domain=$4 - shift 4 - - yaml_content+=$'\n'" - nextHop: ${remote_ip}" + local remote_ipv6=${2:-} + local remote_pod_cidr=$3 + local remote_svc_cidr=$4 + local remote_domain=$5 + local remote_pod_cidr_dual=${6:-} + local remote_svc_cidr_dual=${7:-} + shift 7 + + yaml_content+=$'\n'" - nextHop:" + yaml_content+=$'\n'" - ${remote_ip}" + if [ -n "${remote_ipv6}" ]; then + yaml_content+=$'\n'" - ${remote_ipv6}" + fi yaml_content+=$'\n'" clusterNetwork:" yaml_content+=$'\n'" - ${remote_pod_cidr}" + if [ -n "${remote_pod_cidr_dual}" ]; then + yaml_content+=$'\n'" - ${remote_pod_cidr_dual}" + fi yaml_content+=$'\n'" serviceNetwork:" yaml_content+=$'\n'" - ${remote_svc_cidr}" + if [ -n "${remote_svc_cidr_dual}" ]; then + yaml_content+=$'\n'" - ${remote_svc_cidr_dual}" + fi yaml_content+=$'\n'" domain: ${remote_domain}" firewall_cidrs+=("${remote_pod_cidr}" "${remote_svc_cidr}") + if [ -n "${remote_pod_cidr_dual}" ]; then + firewall_cidrs+=("${remote_pod_cidr_dual}") + fi + if [ -n "${remote_svc_cidr_dual}" ]; then + firewall_cidrs+=("${remote_svc_cidr_dual}") + fi done run_command_on_vm "${host}" "sudo tee /etc/microshift/config.d/50-c2cc.yaml > /dev/null <> "${ks_dir}/post-microshift.cfg" <>/etc/microshift/config.yaml <> "${host2_ks_dir}/post-microshift.cfg" <>/etc/microshift/config.yaml <> "${host3_ks_dir}/post-microshift.cfg" <>/etc/microshift/config.yaml < Date: Thu, 25 Jun 2026 13:13:36 +0200 Subject: [PATCH 4/7] Add dual-stack keywords and variables --- test/resources/c2cc.resource | 223 ++++++++++++++++++++++++++--------- 1 file changed, 168 insertions(+), 55 deletions(-) diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index f51aa73da7..794b148d65 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -13,42 +13,58 @@ Resource microshift-host.resource *** Variables *** -${CLUSTER_A_POD_CIDR} ${EMPTY} -${CLUSTER_A_SVC_CIDR} ${EMPTY} -${CLUSTER_A_DOMAIN} ${EMPTY} -${CLUSTER_B_POD_CIDR} ${EMPTY} -${CLUSTER_B_SVC_CIDR} ${EMPTY} -${CLUSTER_B_DOMAIN} ${EMPTY} -${HOST2_IP} ${EMPTY} -${HOST2_SSH_PORT} ${EMPTY} -${HOST2_API_PORT} ${EMPTY} -${KUBECONFIG_B} ${EMPTY} -${CLUSTER_C_POD_CIDR} ${EMPTY} -${CLUSTER_C_SVC_CIDR} ${EMPTY} -${CLUSTER_C_DOMAIN} ${EMPTY} -${HOST3_IP} ${EMPTY} -${HOST3_SSH_PORT} ${EMPTY} -${HOST3_API_PORT} ${EMPTY} -${KUBECONFIG_C} ${EMPTY} -&{C2CC_KUBECONFIGS} &{EMPTY} -&{C2CC_SSH_IDS} &{EMPTY} -@{C2CC_REMOTE_ALIASES} @{EMPTY} -${IP_FAMILY} ${EMPTY} -${IP_CMD} ${{'ip -6' if '${IP_FAMILY}' == 'ipv6' else 'ip -4'}} -&{NAMESPACES} cluster-a=${EMPTY} cluster-b=${EMPTY} cluster-c=${EMPTY} +${CLUSTER_A_POD_CIDR} ${EMPTY} +${CLUSTER_A_SVC_CIDR} ${EMPTY} +${CLUSTER_A_DOMAIN} ${EMPTY} +${CLUSTER_B_POD_CIDR} ${EMPTY} +${CLUSTER_B_SVC_CIDR} ${EMPTY} +${CLUSTER_B_DOMAIN} ${EMPTY} +${HOST2_IP} ${EMPTY} +${HOST2_IPV6} ${EMPTY} +${HOST2_SSH_PORT} ${EMPTY} +${HOST2_API_PORT} ${EMPTY} +${KUBECONFIG_B} ${EMPTY} +${CLUSTER_C_POD_CIDR} ${EMPTY} +${CLUSTER_C_SVC_CIDR} ${EMPTY} +${CLUSTER_C_DOMAIN} ${EMPTY} +${HOST3_IP} ${EMPTY} +${HOST3_IPV6} ${EMPTY} +${HOST3_SSH_PORT} ${EMPTY} +${HOST3_API_PORT} ${EMPTY} +${KUBECONFIG_C} ${EMPTY} +&{C2CC_KUBECONFIGS} &{EMPTY} +&{C2CC_SSH_IDS} &{EMPTY} +@{C2CC_REMOTE_ALIASES} @{EMPTY} +${CLUSTER_A_POD_CIDR_DUAL} ${EMPTY} +${CLUSTER_A_SVC_CIDR_DUAL} ${EMPTY} +${CLUSTER_B_POD_CIDR_DUAL} ${EMPTY} +${CLUSTER_B_SVC_CIDR_DUAL} ${EMPTY} +${CLUSTER_C_POD_CIDR_DUAL} ${EMPTY} +${CLUSTER_C_SVC_CIDR_DUAL} ${EMPTY} +${IP_FAMILY} ${EMPTY} +${IP_CMD} ${{'ip -6' if '${IP_FAMILY}' == 'ipv6' else 'ip -4'}} +&{NAMESPACES} cluster-a=${EMPTY} cluster-b=${EMPTY} cluster-c=${EMPTY} &{DOMAIN_MAP} -... cluster-a=${CLUSTER_A_DOMAIN} -... cluster-b=${CLUSTER_B_DOMAIN} -... cluster-c=${CLUSTER_C_DOMAIN} +... cluster-a=${CLUSTER_A_DOMAIN} +... cluster-b=${CLUSTER_B_DOMAIN} +... cluster-c=${CLUSTER_C_DOMAIN} &{POD_CIDR_MAP} -... cluster-a=${CLUSTER_A_POD_CIDR} -... cluster-b=${CLUSTER_B_POD_CIDR} -... cluster-c=${CLUSTER_C_POD_CIDR} +... cluster-a=${CLUSTER_A_POD_CIDR} +... cluster-b=${CLUSTER_B_POD_CIDR} +... cluster-c=${CLUSTER_C_POD_CIDR} &{SVC_CIDR_MAP} -... cluster-a=${CLUSTER_A_SVC_CIDR} -... cluster-b=${CLUSTER_B_SVC_CIDR} -... cluster-c=${CLUSTER_C_SVC_CIDR} -@{ALL_CLUSTERS} cluster-a cluster-b cluster-c +... cluster-a=${CLUSTER_A_SVC_CIDR} +... cluster-b=${CLUSTER_B_SVC_CIDR} +... cluster-c=${CLUSTER_C_SVC_CIDR} +&{POD_CIDR_MAP_DUAL} +... cluster-a=${CLUSTER_A_POD_CIDR_DUAL} +... cluster-b=${CLUSTER_B_POD_CIDR_DUAL} +... cluster-c=${CLUSTER_C_POD_CIDR_DUAL} +&{SVC_CIDR_MAP_DUAL} +... cluster-a=${CLUSTER_A_SVC_CIDR_DUAL} +... cluster-b=${CLUSTER_B_SVC_CIDR_DUAL} +... cluster-c=${CLUSTER_C_SVC_CIDR_DUAL} +@{ALL_CLUSTERS} cluster-a cluster-b cluster-c *** Keywords *** @@ -160,29 +176,29 @@ Verify Cluster Is Healthy Verify Routes In Table 200 [Documentation] Check that routes for the given CIDRs exist in table 200. - [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - ${stdout}= Command On Cluster ${alias} ${IP_CMD} route show table 200 + [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${ip_cmd}=${IP_CMD} + ${stdout}= Command On Cluster ${alias} ${ip_cmd} route show table 200 Should Contain ${stdout} ${remote_pod_cidr} Should Contain ${stdout} ${remote_svc_cidr} Verify IP Rules For Table 200 [Documentation] Check that IP rules at priority 100 exist for the given CIDRs. - [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - ${stdout}= Command On Cluster ${alias} ${IP_CMD} rule show + [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${ip_cmd}=${IP_CMD} + ${stdout}= Command On Cluster ${alias} ${ip_cmd} rule show Should Contain ${stdout} to ${remote_pod_cidr} lookup 200 Should Contain ${stdout} to ${remote_svc_cidr} lookup 200 Verify Service IP Rules [Documentation] Check that IP rules at priority 99 exist for cross-cluster service routing. - [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} - ${stdout}= Command On Cluster ${alias} ${IP_CMD} rule show + [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} ${ip_cmd}=${IP_CMD} + ${stdout}= Command On Cluster ${alias} ${ip_cmd} rule show Should Contain ${stdout} from ${remote_pod_cidr} to ${local_svc_cidr} lookup 201 Should Contain ${stdout} from ${remote_svc_cidr} to ${local_svc_cidr} lookup 201 Verify All IP Rules [Documentation] Check all IP rules (table 200 and 201) in a single SSH call. - [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} - ${stdout}= Command On Cluster ${alias} ${IP_CMD} rule show + [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} ${ip_cmd}=${IP_CMD} + ${stdout}= Command On Cluster ${alias} ${ip_cmd} rule show Should Contain ${stdout} to ${remote_pod_cidr} lookup 200 Should Contain ${stdout} to ${remote_svc_cidr} lookup 200 Should Contain ${stdout} from ${remote_pod_cidr} to ${local_svc_cidr} lookup 201 @@ -190,8 +206,8 @@ Verify All IP Rules Verify Routes In Table 201 [Documentation] Check that service routes exist in table 201 for the local service CIDR. - [Arguments] ${alias} ${local_svc_cidr} - ${stdout}= Command On Cluster ${alias} ${IP_CMD} route show table 201 + [Arguments] ${alias} ${local_svc_cidr} ${ip_cmd}=${IP_CMD} + ${stdout}= Command On Cluster ${alias} ${ip_cmd} route show table 201 Should Contain ${stdout} ${local_svc_cidr} Verify NFTables Bypass Rules @@ -361,11 +377,48 @@ Get Curl Pod IP ... oc get pod curl-pod -n ${NAMESPACES}[${alias}] -o jsonpath='{.status.podIP}' RETURN ${ip} +Get Hello Pod Dual IP + [Documentation] Get the secondary (dual-stack) pod IP of hello-microshift. + [Arguments] ${alias} + ${ip}= Oc On Cluster ${alias} + ... oc get pod hello-microshift -n ${NAMESPACES}[${alias}] -o jsonpath='{.status.podIPs[1].ip}' + RETURN ${ip} + +Get Hello Service Dual IP + [Documentation] Get the secondary (dual-stack) ClusterIP of the hello-microshift service. + [Arguments] ${alias} + ${ip}= Oc On Cluster ${alias} + ... oc get svc hello-microshift -n ${NAMESPACES}[${alias}] -o jsonpath='{.spec.clusterIPs[1]}' + RETURN ${ip} + +Get Curl Pod Dual IP + [Documentation] Get the secondary (dual-stack) pod IP of curl-pod. + [Arguments] ${alias} + ${ip}= Oc On Cluster ${alias} + ... oc get pod curl-pod -n ${NAMESPACES}[${alias}] -o jsonpath='{.status.podIPs[1].ip}' + RETURN ${ip} + +Get Curl Pod IP Matching Family + [Documentation] Get the curl-pod IP that matches the address family of the given destination IP. + ... OVN-K always assigns IPv4 first in podIPs regardless of clusterNetwork order, + ... so status.podIP is always IPv4 and podIPs[1] is IPv6. Services, however, + ... follow the config order — spec.clusterIP is IPv6 in IPv6-primary clusters. + ... This keyword returns whichever curl-pod IP matches the destination family. + [Arguments] ${alias} ${dest_ip} + ${dest_is_ipv6}= Evaluate ':' in '''${dest_ip}''' + IF ${dest_is_ipv6} + ${ip}= Get Curl Pod Dual IP ${alias} + ELSE + ${ip}= Get Curl Pod IP ${alias} + END + RETURN ${ip} + Curl From Cluster [Documentation] Exec curl from curl-pod on the given cluster to the target IP and port. [Arguments] ${alias} ${ip} ${port} + ${is_ipv6}= Evaluate ':' in '''${ip}''' ${url}= Set Variable If - ... '${IP_FAMILY}' == 'ipv6' + ... ${is_ipv6} ... http://[${ip}]:${port}/cgi-bin/hello ... http://${ip}:${port}/cgi-bin/hello ${stdout}= Oc On Cluster ${alias} @@ -388,7 +441,6 @@ Test Connectivity Between Clusters Test Source IP Preserved Between Clusters [Documentation] Verify ${source} to ${destination} pod-to-${endpoint_type} traffic preserves the source pod IP (no SNAT). [Arguments] ${source} ${destination} ${endpoint_type} - ${curl_pod_ip}= Get Curl Pod IP ${source} IF '${endpoint_type}' == 'pod' ${ip_dest}= Get Hello Pod IP ${destination} ELSE IF '${endpoint_type}' == 'service' @@ -396,6 +448,34 @@ Test Source IP Preserved Between Clusters ELSE Fail Invalid endpoint_type: ${endpoint_type}. Must be 'pod' or 'service'. END + ${curl_pod_ip}= Get Curl Pod IP Matching Family ${source} ${ip_dest} + ${stdout}= Curl From Cluster ${source} ${ip_dest} 8080 + Should Contain ${stdout} source: ${curl_pod_ip} + +Test Dual Stack Connectivity Between Clusters + [Documentation] Verify pod on ${source} can reach ${endpoint_type} dual-stack IP on ${destination}. + [Arguments] ${source} ${destination} ${endpoint_type} + IF '${endpoint_type}' == 'pod' + ${ip_dest}= Get Hello Pod Dual IP ${destination} + ELSE IF '${endpoint_type}' == 'service' + ${ip_dest}= Get Hello Service Dual IP ${destination} + ELSE + Fail Invalid endpoint_type: ${endpoint_type}. Must be 'pod' or 'service'. + END + ${stdout}= Curl From Cluster ${source} ${ip_dest} 8080 + Should Contain ${stdout} Hello from + +Test Dual Stack Source IP Preserved Between Clusters + [Documentation] Verify dual-stack traffic preserves the source pod IP (no SNAT). + [Arguments] ${source} ${destination} ${endpoint_type} + IF '${endpoint_type}' == 'pod' + ${ip_dest}= Get Hello Pod Dual IP ${destination} + ELSE IF '${endpoint_type}' == 'service' + ${ip_dest}= Get Hello Service Dual IP ${destination} + ELSE + Fail Invalid endpoint_type: ${endpoint_type}. Must be 'pod' or 'service'. + END + ${curl_pod_ip}= Get Curl Pod IP Matching Family ${source} ${ip_dest} ${stdout}= Curl From Cluster ${source} ${ip_dest} 8080 Should Contain ${stdout} source: ${curl_pod_ip} @@ -471,20 +551,43 @@ RemoteCluster CR Name From IP ${dashed}= Replace String ${dashed} : - RETURN c2cc-${dashed} +Primary NextHop IP For Host + [Documentation] Return the IP used as primary next-hop for a host. In IPv6-primary + ... scenarios (IP_FAMILY=ipv6), the IPv6 address is primary; otherwise IPv4. + [Arguments] ${ipv4} ${ipv6}=${EMPTY} + ${is_ipv6_primary}= Evaluate '${IP_FAMILY}' == 'ipv6' and '${ipv6}' != '' + ${result}= Set Variable If ${is_ipv6_primary} ${ipv6} ${ipv4} + RETURN ${result} + Verify RemoteCluster Unhealthy On Observers [Documentation] Wait for observer clusters to report a disrupted cluster's CR as Unhealthy. - [Arguments] ${disrupted_ip} @{observers} - ${cr_name}= RemoteCluster CR Name From IP ${disrupted_ip} + [Arguments] ${disrupted_ip} @{observers} ${disrupted_ipv6}=${EMPTY} + ${primary}= Primary NextHop IP For Host ${disrupted_ip} ${disrupted_ipv6} + ${cr_name}= RemoteCluster CR Name From IP ${primary} FOR ${observer} IN @{observers} Wait Until Keyword Succeeds 3m 10s ... Verify RemoteCluster State By Name ${observer} ${cr_name} Unhealthy END +Ensure All Clusters Healthy + [Documentation] Pre-condition: all clusters must be Healthy before fault injection. + FOR ${alias} IN cluster-a cluster-b cluster-c + Wait Until Keyword Succeeds 3m 10s + ... Verify RemoteCluster State ${alias} Healthy + END + +IP Command For CIDR + [Documentation] Return 'ip -6' if the given CIDR is IPv6, otherwise 'ip -4'. + [Arguments] ${cidr} + ${is_ipv6}= Evaluate ':' in '''${cidr}''' + ${cmd}= Set Variable If ${is_ipv6} ip -6 ip -4 + RETURN ${cmd} + Verify C2CC Infrastructure For Remote [Documentation] Verify C2CC infrastructure for one remote cluster's CIDRs. - [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} - Verify Routes In Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - Verify All IP Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} + [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} ${ip_cmd}=${IP_CMD} + Verify Routes In Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${ip_cmd} + Verify All IP Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} ${ip_cmd} Verify NFTables Bypass Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} Verify OVN Static Routes ${alias} ${remote_pod_cidr} ${remote_svc_cidr} Verify Node SNAT Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} @@ -492,16 +595,26 @@ Verify C2CC Infrastructure For Remote Verify C2CC Infrastructure On Cluster [Documentation] Run all C2CC infrastructure checks for all remote CIDRs on the given cluster. [Arguments] ${alias} - ${local_svc_cidr}= Get From Dictionary ${SVC_CIDR_MAP} ${alias} + Verify C2CC Infrastructure For CIDR Family ${alias} ${POD_CIDR_MAP} ${SVC_CIDR_MAP} + ${local_svc_cidr_dual}= Get From Dictionary ${SVC_CIDR_MAP_DUAL} ${alias} + IF '${local_svc_cidr_dual}' != '' + Verify C2CC Infrastructure For CIDR Family ${alias} ${POD_CIDR_MAP_DUAL} ${SVC_CIDR_MAP_DUAL} + END + +Verify C2CC Infrastructure For CIDR Family + [Documentation] Verify C2CC infrastructure for one CIDR family (primary or dual) on a cluster. + [Arguments] ${alias} ${pod_map} ${svc_map} + ${local_svc_cidr}= Get From Dictionary ${svc_map} ${alias} + ${ip_cmd}= IP Command For CIDR ${local_svc_cidr} FOR ${remote} IN @{ALL_CLUSTERS} IF '${remote}' != '${alias}' - ${remote_pod_cidr}= Get From Dictionary ${POD_CIDR_MAP} ${remote} - ${remote_svc_cidr}= Get From Dictionary ${SVC_CIDR_MAP} ${remote} + ${remote_pod_cidr}= Get From Dictionary ${pod_map} ${remote} + ${remote_svc_cidr}= Get From Dictionary ${svc_map} ${remote} Verify C2CC Infrastructure For Remote - ... ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} + ... ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} ${ip_cmd} END END - Verify Routes In Table 201 ${alias} ${local_svc_cidr} + Verify Routes In Table 201 ${alias} ${local_svc_cidr} ${ip_cmd} Verify Cross Cluster Connectivity [Documentation] Verify a representative subset of cross-cluster connectivity. From cde7a5ed29d226e66031c2ed5bb7c1fa6ddad8a7 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 25 Jun 2026 13:28:35 +0200 Subject: [PATCH 5/7] Add dual-stack RF test cases --- test/suites/c2cc/cleanup.robot | 46 ++++++++++++ test/suites/c2cc/connectivity.robot | 32 ++++++++ test/suites/c2cc/disruptive.robot | 2 + test/suites/c2cc/infrastructure.robot | 101 ++++++++++++++++++++++++++ test/suites/c2cc/probe.robot | 3 +- test/suites/c2cc/reconciliation.robot | 61 ++++++++++++++-- 6 files changed, 236 insertions(+), 9 deletions(-) diff --git a/test/suites/c2cc/cleanup.robot b/test/suites/c2cc/cleanup.robot index 59af6e8ea6..8422bfd9f2 100644 --- a/test/suites/c2cc/cleanup.robot +++ b/test/suites/c2cc/cleanup.robot @@ -86,6 +86,52 @@ No C2CC Tracking Annotation After Disable ... oc get node -o jsonpath='{.items[0].metadata.annotations.microshift\\.io/c2cc-snat-subnets}' Should Be Empty ${stdout} +No Dual Stack Linux Routes In Table 200 After Disable + [Documentation] Dual-stack routes to remote CIDRs in table 200 should be gone. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_B_POD_CIDR_DUAL} + ${stdout}= Command On Cluster cluster-a ${ip_cmd} route show table 200 + FOR ${cidr} IN + ... ${CLUSTER_B_POD_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${CLUSTER_C_POD_CIDR_DUAL} + ... ${CLUSTER_C_SVC_CIDR_DUAL} + Should Not Contain ${stdout} ${cidr} + END + +No Dual Stack IP Rules For Table 200 After Disable + [Documentation] Dual-stack IP rules directing to table 200 should be gone. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_B_POD_CIDR_DUAL} + ${stdout}= Command On Cluster cluster-a ${ip_cmd} rule show + FOR ${cidr} IN + ... ${CLUSTER_B_POD_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${CLUSTER_C_POD_CIDR_DUAL} + ... ${CLUSTER_C_SVC_CIDR_DUAL} + Should Not Contain ${stdout} to ${cidr} lookup 200 + END + +No Dual Stack Service Routes In Table 201 After Disable + [Documentation] Dual-stack service routes in table 201 should be gone. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_A_SVC_CIDR_DUAL} + ${stdout}= Command On Cluster cluster-a ${ip_cmd} route show table 201 + Should Not Contain ${stdout} ${CLUSTER_A_SVC_CIDR_DUAL} + +No Dual Stack Service IP Rules After Disable + [Documentation] Dual-stack service IP rules for table 201 should be gone. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_B_POD_CIDR_DUAL} + ${stdout}= Command On Cluster cluster-a ${ip_cmd} rule show + FOR ${cidr} IN + ... ${CLUSTER_B_POD_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${CLUSTER_C_POD_CIDR_DUAL} + ... ${CLUSTER_C_SVC_CIDR_DUAL} + Should Not Contain ${stdout} from ${cidr} to ${CLUSTER_A_SVC_CIDR_DUAL} lookup 201 + END + C2CC Controller Logged Cleanup [Documentation] The controller should have logged that it is disabled and cleaning up. ${stdout}= Command On Cluster cluster-a diff --git a/test/suites/c2cc/connectivity.robot b/test/suites/c2cc/connectivity.robot index ad37aa8cce..2c7dffa80a 100644 --- a/test/suites/c2cc/connectivity.robot +++ b/test/suites/c2cc/connectivity.robot @@ -47,6 +47,38 @@ Test Cross Cluster Source IP Preservation cluster-c cluster-b pod cluster-c cluster-b service +Test Dual Stack Cross Cluster Connectivity + [Documentation] Verify dual-stack pods on all clusters can reach pods/services on all other clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + Test Dual Stack Connectivity Between Clusters cluster-a cluster-b pod + Test Dual Stack Connectivity Between Clusters cluster-a cluster-b service + Test Dual Stack Connectivity Between Clusters cluster-a cluster-c pod + Test Dual Stack Connectivity Between Clusters cluster-a cluster-c service + Test Dual Stack Connectivity Between Clusters cluster-b cluster-a pod + Test Dual Stack Connectivity Between Clusters cluster-b cluster-a service + Test Dual Stack Connectivity Between Clusters cluster-b cluster-c pod + Test Dual Stack Connectivity Between Clusters cluster-b cluster-c service + Test Dual Stack Connectivity Between Clusters cluster-c cluster-a pod + Test Dual Stack Connectivity Between Clusters cluster-c cluster-a service + Test Dual Stack Connectivity Between Clusters cluster-c cluster-b pod + Test Dual Stack Connectivity Between Clusters cluster-c cluster-b service + +Test Dual Stack Cross Cluster Source IP Preservation + [Documentation] Verify dual-stack cross cluster traffic preserves source pod IP (no SNAT). + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-b pod + Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-b service + Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-c pod + Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-c service + Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-a pod + Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-a service + Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-c pod + Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-c service + Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-a pod + Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-a service + Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-b pod + Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-b service + *** Keywords *** Setup diff --git a/test/suites/c2cc/disruptive.robot b/test/suites/c2cc/disruptive.robot index ce3d79fbf4..569a6b5702 100644 --- a/test/suites/c2cc/disruptive.robot +++ b/test/suites/c2cc/disruptive.robot @@ -59,6 +59,7 @@ Recovery After NIC Outage On One Cluster VAR ${DISABLED_VM} ${HOST2_VM_NAME} scope=TEST VAR @{DISABLED_IFACES} @{vnet_ifaces} scope=TEST Verify RemoteCluster Unhealthy On Observers ${HOST2_IP} cluster-a cluster-c + ... disrupted_ipv6=${HOST2_IPV6} Enable All NICs For VM ${HOST2_VM_NAME} ${vnet_ifaces} Reconnect To Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} ... timeout=${RECOVERY_TIMEOUT} @@ -97,6 +98,7 @@ Recovery After OVN-K Restart On One Cluster And NIC Outage On Another Cluster VAR ${DISABLED_VM} ${HOST3_VM_NAME} scope=TEST VAR @{DISABLED_IFACES} @{vnet_ifaces} scope=TEST Verify RemoteCluster Unhealthy On Observers ${HOST3_IP} cluster-a + ... disrupted_ipv6=${HOST3_IPV6} Enable All NICs For VM ${HOST3_VM_NAME} ${vnet_ifaces} Wait For OVN-K Pods Ready On Cluster cluster-b Reconnect To Cluster cluster-c ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} diff --git a/test/suites/c2cc/infrastructure.robot b/test/suites/c2cc/infrastructure.robot index 265510a89f..a6e567a452 100644 --- a/test/suites/c2cc/infrastructure.robot +++ b/test/suites/c2cc/infrastructure.robot @@ -82,6 +82,107 @@ Node Annotation Set cluster-c ${CLUSTER_A_POD_CIDR} ${CLUSTER_A_SVC_CIDR} cluster-c ${CLUSTER_B_POD_CIDR} ${CLUSTER_B_SVC_CIDR} +Dual Stack Linux Routes Table 200 Exist + [Documentation] Verify dual-stack routes to remote CIDRs exist in table 200 between all clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_A_POD_CIDR_DUAL} + Verify Routes In Table 200 cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} ${ip_cmd} + Verify Routes In Table 200 cluster-a ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} ${ip_cmd} + Verify Routes In Table 200 cluster-b ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} ${ip_cmd} + Verify Routes In Table 200 cluster-b ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} ${ip_cmd} + Verify Routes In Table 200 cluster-c ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} ${ip_cmd} + Verify Routes In Table 200 cluster-c ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} ${ip_cmd} + +Dual Stack IP Rules For Remote CIDRs Exist + [Documentation] Verify dual-stack IP rules direct remote CIDRs to table 200 between all clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_A_POD_CIDR_DUAL} + Verify IP Rules For Table 200 cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} ${ip_cmd} + Verify IP Rules For Table 200 cluster-a ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} ${ip_cmd} + Verify IP Rules For Table 200 cluster-b ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} ${ip_cmd} + Verify IP Rules For Table 200 cluster-b ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} ${ip_cmd} + Verify IP Rules For Table 200 cluster-c ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} ${ip_cmd} + Verify IP Rules For Table 200 cluster-c ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} ${ip_cmd} + +Dual Stack Service Routes Table 201 Exist + [Documentation] Verify dual-stack service routes exist in table 201 on all clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_A_SVC_CIDR_DUAL} + Verify Routes In Table 201 cluster-a ${CLUSTER_A_SVC_CIDR_DUAL} ${ip_cmd} + Verify Routes In Table 201 cluster-b ${CLUSTER_B_SVC_CIDR_DUAL} ${ip_cmd} + Verify Routes In Table 201 cluster-c ${CLUSTER_C_SVC_CIDR_DUAL} ${ip_cmd} + +Dual Stack Service IP Rules Exist + [Documentation] Verify dual-stack service IP rules on all clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_A_POD_CIDR_DUAL} + Verify Service IP Rules + ... cluster-a + ... ${CLUSTER_B_POD_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${CLUSTER_A_SVC_CIDR_DUAL} + ... ${ip_cmd} + Verify Service IP Rules + ... cluster-a + ... ${CLUSTER_C_POD_CIDR_DUAL} + ... ${CLUSTER_C_SVC_CIDR_DUAL} + ... ${CLUSTER_A_SVC_CIDR_DUAL} + ... ${ip_cmd} + Verify Service IP Rules + ... cluster-b + ... ${CLUSTER_A_POD_CIDR_DUAL} + ... ${CLUSTER_A_SVC_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${ip_cmd} + Verify Service IP Rules + ... cluster-b + ... ${CLUSTER_C_POD_CIDR_DUAL} + ... ${CLUSTER_C_SVC_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${ip_cmd} + Verify Service IP Rules + ... cluster-c + ... ${CLUSTER_A_POD_CIDR_DUAL} + ... ${CLUSTER_A_SVC_CIDR_DUAL} + ... ${CLUSTER_C_SVC_CIDR_DUAL} + ... ${ip_cmd} + Verify Service IP Rules + ... cluster-c + ... ${CLUSTER_B_POD_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${CLUSTER_C_SVC_CIDR_DUAL} + ... ${ip_cmd} + +Dual Stack NFTables Bypass Rules Exist + [Documentation] Verify dual-stack nftables masquerade bypass rules on all clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + Verify NFTables Bypass Rules cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} + Verify NFTables Bypass Rules cluster-a ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} + Verify NFTables Bypass Rules cluster-b ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} + Verify NFTables Bypass Rules cluster-b ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} + Verify NFTables Bypass Rules cluster-c ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} + Verify NFTables Bypass Rules cluster-c ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} + +Dual Stack OVN Static Routes Exist + [Documentation] Verify dual-stack OVN NB static routes tagged with microshift-c2cc on all clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + Verify OVN Static Routes cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} + Verify OVN Static Routes cluster-a ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} + Verify OVN Static Routes cluster-b ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} + Verify OVN Static Routes cluster-b ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} + Verify OVN Static Routes cluster-c ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} + Verify OVN Static Routes cluster-c ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} + +Dual Stack Node Annotation Set + [Documentation] Verify SNAT-exclude annotation contains dual-stack remote CIDRs on all clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + Verify Node SNAT Annotation cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} + Verify Node SNAT Annotation cluster-a ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} + Verify Node SNAT Annotation cluster-b ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} + Verify Node SNAT Annotation cluster-b ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} + Verify Node SNAT Annotation cluster-c ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} + Verify Node SNAT Annotation cluster-c ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} + *** Keywords *** Setup diff --git a/test/suites/c2cc/probe.robot b/test/suites/c2cc/probe.robot index a140c969b2..bc5dfcc0ed 100644 --- a/test/suites/c2cc/probe.robot +++ b/test/suites/c2cc/probe.robot @@ -104,7 +104,8 @@ RemoteCluster Status Becomes Unhealthy When Probe Fails [Documentation] Block probe traffic on cluster-b and verify cluster-a ... reports Unhealthy for the corresponding RemoteCluster CR. [Setup] Verify All RemoteClusters Healthy - ${cr_name}= RemoteCluster CR Name From IP ${HOST2_IP} + ${primary_ip}= Primary NextHop IP For Host ${HOST2_IP} ${HOST2_IPV6} + ${cr_name}= RemoteCluster CR Name From IP ${primary_ip} # Apply a NetworkPolicy on cluster-b that denies all ingress to the probe pod, # causing cluster-a's probes to cluster-b to time out. Apply Probe Deny Policy cluster-b diff --git a/test/suites/c2cc/reconciliation.robot b/test/suites/c2cc/reconciliation.robot index 44eef57677..ee9d3eeebb 100644 --- a/test/suites/c2cc/reconciliation.robot +++ b/test/suites/c2cc/reconciliation.robot @@ -85,6 +85,51 @@ Reconcile SNAT Annotation Preserves Foreign Subnets Should Contain ${annotation} ${FOREIGN_CIDR} [Teardown] Remove Foreign Subnet From SNAT Annotation cluster-a +Reconcile Dual Stack Linux Route In Table 200 After Deletion + [Documentation] Delete a dual-stack route from table 200, verify the controller restores it. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_B_POD_CIDR_DUAL} + Delete Route From Table 200 On Cluster cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${ip_cmd} + Wait Until Keyword Succeeds ${RECONCILE_TIMEOUT} ${RECONCILE_RETRY} + ... Verify Routes In Table 200 + ... cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} ${ip_cmd} + +Reconcile Dual Stack IP Rule For Table 200 After Deletion + [Documentation] Delete a dual-stack IP rule for table 200, verify the controller restores it. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_B_POD_CIDR_DUAL} + Delete IP Rule For Table 200 On Cluster cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${ip_cmd} + Wait Until Keyword Succeeds ${RECONCILE_TIMEOUT} ${RECONCILE_RETRY} + ... Verify IP Rules For Table 200 + ... cluster-a ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} ${ip_cmd} + +Reconcile Dual Stack Service Route In Table 201 After Deletion + [Documentation] Delete a dual-stack service route from table 201, verify the controller restores it. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_A_SVC_CIDR_DUAL} + Delete Service Route From Table 201 On Cluster cluster-a ${CLUSTER_A_SVC_CIDR_DUAL} ${ip_cmd} + Wait Until Keyword Succeeds ${RECONCILE_TIMEOUT} ${RECONCILE_RETRY} + ... Verify Routes In Table 201 cluster-a ${CLUSTER_A_SVC_CIDR_DUAL} ${ip_cmd} + +Reconcile Dual Stack Service IP Rule After Deletion + [Documentation] Delete a dual-stack service IP rule, verify the controller restores it. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + ${ip_cmd}= IP Command For CIDR ${CLUSTER_B_POD_CIDR_DUAL} + Delete Service IP Rule On Cluster + ... cluster-a + ... ${CLUSTER_B_POD_CIDR_DUAL} + ... ${CLUSTER_A_SVC_CIDR_DUAL} + ... ${ip_cmd} + Wait Until Keyword Succeeds + ... ${RECONCILE_TIMEOUT} + ... ${RECONCILE_RETRY} + ... Verify Service IP Rules + ... cluster-a + ... ${CLUSTER_B_POD_CIDR_DUAL} + ... ${CLUSTER_B_SVC_CIDR_DUAL} + ... ${CLUSTER_A_SVC_CIDR_DUAL} + ... ${ip_cmd} + *** Keywords *** Setup @@ -110,24 +155,24 @@ Get Node Name On Cluster Delete Route From Table 200 On Cluster [Documentation] Delete a specific route from policy routing table 200. - [Arguments] ${alias} ${cidr} - Disruptive Command On Cluster ${alias} ${IP_CMD} route del ${cidr} table 200 + [Arguments] ${alias} ${cidr} ${ip_cmd}=${IP_CMD} + Disruptive Command On Cluster ${alias} ${ip_cmd} route del ${cidr} table 200 Delete IP Rule For Table 200 On Cluster [Documentation] Delete an IP rule directing traffic to table 200. - [Arguments] ${alias} ${cidr} - Disruptive Command On Cluster ${alias} ${IP_CMD} rule del to ${cidr} lookup 200 + [Arguments] ${alias} ${cidr} ${ip_cmd}=${IP_CMD} + Disruptive Command On Cluster ${alias} ${ip_cmd} rule del to ${cidr} lookup 200 Delete Service Route From Table 201 On Cluster [Documentation] Delete a service route from table 201. - [Arguments] ${alias} ${cidr} - Disruptive Command On Cluster ${alias} ${IP_CMD} route del ${cidr} table 201 + [Arguments] ${alias} ${cidr} ${ip_cmd}=${IP_CMD} + Disruptive Command On Cluster ${alias} ${ip_cmd} route del ${cidr} table 201 Delete Service IP Rule On Cluster [Documentation] Delete a service IP rule from table 201. - [Arguments] ${alias} ${from_cidr} ${to_cidr} + [Arguments] ${alias} ${from_cidr} ${to_cidr} ${ip_cmd}=${IP_CMD} Disruptive Command On Cluster ${alias} - ... ${IP_CMD} rule del from ${from_cidr} to ${to_cidr} lookup 201 + ... ${ip_cmd} rule del from ${from_cidr} to ${to_cidr} lookup 201 Delete NFTables C2CC Rule On Cluster [Documentation] Delete an nftables bypass rule by discovering its handle. From 1d8e84416efe2e0f7ed63a76191342d2d89645a7 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 26 Jun 2026 14:11:20 +0200 Subject: [PATCH 6/7] Update hello-microshift to PreferDualStack --- test/assets/c2cc/hello-microshift.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/assets/c2cc/hello-microshift.yaml b/test/assets/c2cc/hello-microshift.yaml index 34ffcb717f..90f9064364 100644 --- a/test/assets/c2cc/hello-microshift.yaml +++ b/test/assets/c2cc/hello-microshift.yaml @@ -49,6 +49,7 @@ metadata: labels: app: hello-microshift spec: + ipFamilyPolicy: PreferDualStack selector: app: hello-microshift ports: From 2f5de135279fda33663f9e63753f0503ef13ca03 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 25 Jun 2026 14:01:04 +0200 Subject: [PATCH 7/7] Dual-stack scenarios --- .../c2cc/el102-src@c2cc-dual-stack.sh | 41 ++++++++++++++++++ .../c2cc/el98-src@c2cc-dual-stack-v6.sh | 43 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100755 test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh create mode 100755 test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh new file mode 100755 index 0000000000..48cd060200 --- /dev/null +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +# Cluster A (host1): primary IPv4, secondary IPv6 +CLUSTER_A_POD_CIDR="10.42.0.0/16" +CLUSTER_A_SVC_CIDR="10.43.0.0/16" +CLUSTER_A_POD_CIDR_DUAL="fd01::/48" +CLUSTER_A_SVC_CIDR_DUAL="fd02::/112" +CLUSTER_A_DOMAIN="cluster-a.remote" + +# Cluster B (host2): primary IPv4, secondary IPv6 +CLUSTER_B_POD_CIDR="10.45.0.0/16" +CLUSTER_B_SVC_CIDR="10.46.0.0/16" +CLUSTER_B_POD_CIDR_DUAL="fd04::/48" +CLUSTER_B_SVC_CIDR_DUAL="fd05::/112" +CLUSTER_B_DOMAIN="cluster-b.remote" + +# Cluster C (host3): primary IPv4, secondary IPv6 +CLUSTER_C_POD_CIDR="10.48.0.0/16" +CLUSTER_C_SVC_CIDR="10.49.0.0/16" +CLUSTER_C_POD_CIDR_DUAL="fd07::/48" +CLUSTER_C_SVC_CIDR_DUAL="fd08::/112" +CLUSTER_C_DOMAIN="cluster-c.remote" + +scenario_create_vms() { + c2cc_create_vms rhel102-bootc-source rhel102-bootc "${VM_DUAL_STACK_NETWORK}" dual-stack +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + # shellcheck disable=SC2119 + configure_c2cc_hosts + c2cc_run_tests "suites/c2cc/" "192.0.2.0/24" dual-stack +} diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh new file mode 100755 index 0000000000..60c8d0ed8a --- /dev/null +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +# Cluster A (host1): primary IPv6, secondary IPv4 +CLUSTER_A_POD_CIDR="fd01::/48" +CLUSTER_A_SVC_CIDR="fd02::/112" +CLUSTER_A_POD_CIDR_DUAL="10.42.0.0/16" +CLUSTER_A_SVC_CIDR_DUAL="10.43.0.0/16" +CLUSTER_A_DOMAIN="cluster-a.remote" + +# Cluster B (host2): primary IPv6, secondary IPv4 +CLUSTER_B_POD_CIDR="fd04::/48" +CLUSTER_B_SVC_CIDR="fd05::/112" +CLUSTER_B_POD_CIDR_DUAL="10.45.0.0/16" +CLUSTER_B_SVC_CIDR_DUAL="10.46.0.0/16" +CLUSTER_B_DOMAIN="cluster-b.remote" + +# Cluster C (host3): primary IPv6, secondary IPv4 +CLUSTER_C_POD_CIDR="fd07::/48" +CLUSTER_C_SVC_CIDR="fd08::/112" +CLUSTER_C_POD_CIDR_DUAL="10.48.0.0/16" +CLUSTER_C_SVC_CIDR_DUAL="10.49.0.0/16" +CLUSTER_C_DOMAIN="cluster-c.remote" + +scenario_create_vms() { + c2cc_create_vms rhel98-bootc-source rhel98-bootc "${VM_DUAL_STACK_NETWORK}" dual-stack +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + # shellcheck disable=SC2119 + configure_c2cc_hosts + # IP_FAMILY=ipv6 so IP_CMD defaults to 'ip -6' for primary (IPv6) CIDRs. + # Dual-stack (IPv4) tests derive ip_cmd from CIDR content via IP Command For CIDR. + c2cc_run_tests "suites/c2cc/" "192.0.2.0/24" ipv6 +}