Fix overlay network IP assignment using Peers-based positioning

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Test
+
+permissions:
+  contents: read
+
+on: [push]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: "1.26"
+          cache: false
+
+      - name: Install dependencies
+        run: go get .
+      - name: Check go fix
+        run: go fix ./... && git diff --exit-code
+      - name: Test
+        run: go test -v ./...

main.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
-	"hash/fnv"
 	"io"
 	"log"
 	"net"
@@ -181,10 +181,11 @@ func loadConfig(l *logrus.Logger, configFile string) *Config {
 }
 
 type manager struct {
-	l          *logrus.Logger
-	ctx        context.Context
-	cli        *client.Client
-	nodeOffset int
+	l                 *logrus.Logger
+	ctx               context.Context
+	cli               *client.Client
+	nodeAddr          string
+	overlayContainers int
 }
 
 func NewManager(l *logrus.Logger, overlayContainers int) (*manager, error) {
@@ -194,19 +195,16 @@ func NewManager(l *logrus.Logger, overlayContainers int) (*manager, error) {
 		return nil, err
 	}
 
-	nodeOffset := 0
+	var nodeAddr string
 	if overlayContainers > 0 {
 		info, err := cli.Info(ctx)
-		if err == nil && info.Swarm.NodeID != "" {
-			slots := min(32, 250/overlayContainers)
-			h := fnv.New32a()
-			h.Write([]byte(info.Swarm.NodeID))
-			nodeOffset = int(h.Sum32()%uint32(slots)) * overlayContainers
-			l.Infof("Swarm node %s, IP offset %d (%d overlay containers, %d slots)", info.Swarm.NodeID, nodeOffset, overlayContainers, slots)
+		if err == nil && info.Swarm.NodeAddr != "" {
+			nodeAddr = info.Swarm.NodeAddr
+			l.Infof("Swarm node %s (addr %s), %d overlay containers", info.Swarm.NodeID, nodeAddr, overlayContainers)
 		}
 	}
 
-	return &manager{l, ctx, cli, nodeOffset}, nil
+	return &manager{l: l, ctx: ctx, cli: cli, nodeAddr: nodeAddr, overlayContainers: overlayContainers}, nil
 }
 
 func (m *manager) run(config *Config) {
@@ -222,6 +220,7 @@ func (m *manager) run(config *Config) {
 	}
 	m.l.WithField("networks", networkNames).Debugf("Networks: %s", networkNames)
 
+	overlayIndex := 0
 	for _, c := range config.Containers {
 
 		var networks []string = nil
@@ -231,14 +230,19 @@ func (m *manager) run(config *Config) {
 		}
 		networks = lo.Ternary(len(networks) > 0, networks, []string{"bridge"})
 
-		err = m.ensureContainer(c, networks)
+		currentIndex := overlayIndex
+		if c.AttachAllNetwork {
+			overlayIndex++
+		}
+
+		err = m.ensureContainer(c, networks, currentIndex)
 		if err != nil {
 			m.l.WithField("name", c.Name).WithError(err).Errorf("failed to enure container %s", c.Name)
 		}
 	}
 }
 
-func (m *manager) ensureContainer(config Container, networks []string) error {
+func (m *manager) ensureContainer(config Container, networks []string, overlayIndex int) error {
 	containers, err := m.cli.ContainerList(m.ctx, container.ListOptions{All: true, Filters: filters.NewArgs(filters.Arg("name", fmt.Sprintf("^/%s$", config.Name)))})
 	if err != nil {
 		return err
@@ -284,7 +288,7 @@ func (m *manager) ensureContainer(config Container, networks []string) error {
 					return e.Name == config.Name
 				})
 				if !alreadyAttached {
-					m.connectNetworkHighIP(n, config.Name, config.Name)
+					m.connectNetworkHighIP(n, config.Name, config.Name, overlayIndex)
 				}
 			}
 
@@ -411,7 +415,7 @@ func (m *manager) ensureContainer(config Container, networks []string) error {
 		// Connect networks after start so overlay handshakes happen immediately
 		// and NetworkConnect returns the real error on IP conflict
 		for _, n := range networks {
-			m.connectNetworkHighIP(n, c.ID, config.Name)
+			m.connectNetworkHighIP(n, c.ID, config.Name, overlayIndex)
 		}
 
 		return nil
@@ -509,10 +513,10 @@ func (m *manager) detachNetwork(network string, container string) {
 }
 
 // connectNetworkHighIP connects a container to a network using an IP from the
-// upper end of the subnet. If the chosen IP conflicts (e.g. used on another
-// swarm node), it retries with the next IP down, up to 100 attempts.
-func (m *manager) connectNetworkHighIP(networkName string, containerID string, containerName string) {
-	candidates := m.highIPCandidates(networkName, containerName)
+// upper end of the subnet. If the chosen IP conflicts, it falls back to
+// Docker-assigned.
+func (m *manager) connectNetworkHighIP(networkName string, containerID string, containerName string, overlayIndex int) {
+	candidates := m.highIPCandidates(networkName, overlayIndex)
 	if candidates == nil {
 		err := m.cli.NetworkConnect(m.ctx, networkName, containerID, nil)
 		if err != nil {
@@ -532,19 +536,18 @@ func (m *manager) connectNetworkHighIP(networkName string, containerID string, c
 			m.l.WithField("name", containerName).Infof("Assigned IP %s on network %s", ip, networkName)
 			return
 		}
-		m.l.WithField("name", containerName).Debugf("IP %s taken on network %s, trying next", ip, networkName)
+		m.l.WithField("name", containerName).Debugf("IP %s unavailable on network %s, trying next", ip, networkName)
 	}
 
-	m.l.WithField("name", containerName).Warnf("exhausted 100 high IPs on network %s, letting Docker assign", networkName)
-	err := m.cli.NetworkConnect(m.ctx, networkName, containerID, nil)
-	if err != nil {
-		m.l.WithField("name", containerName).WithError(err).Errorf("failed to connect network %s", networkName)
-	}
+	// Never fall back to Docker-assigned: a low IP would collide with Swarm's bottom-up allocation
+	m.l.WithField("name", containerName).Warnf("all %d candidate IPs exhausted on network %s, skipping", len(candidates), networkName)
 }
 
-// highIPCandidates returns up to 100 free IPs from the top of the network's
-// subnet, or nil if static IP assignment should be skipped.
-func (m *manager) highIPCandidates(networkName string, containerName string) []net.IP {
+// highIPCandidates returns IPs from a tight band at the top of the network's
+// subnet, using the overlay Peers list for deterministic per-node positioning.
+// Returns nil for non-overlay networks (caller should let Docker assign).
+// Returns a non-nil (possibly empty) slice for overlay networks.
+func (m *manager) highIPCandidates(networkName string, overlayIndex int) []net.IP {
 	networkInfo, err := m.cli.NetworkInspect(m.ctx, networkName, types.NetworkInspectOptions{})
 	if err != nil {
 		return nil
@@ -567,30 +570,72 @@ func (m *manager) highIPCandidates(networkName string, containerName string) []n
 		return nil
 	}
 
-	usedIPs := lo.FilterMap(lo.Values(networkInfo.Containers), func(e types.EndpointResource, _ int) (string, bool) {
-		ip, _, err := net.ParseCIDR(e.IPv4Address)
-		if err != nil {
-			return "", false
+	if len(networkInfo.Peers) == 0 {
+		return nil
+	}
+
+	peerIPs := make([]string, len(networkInfo.Peers))
+	for i, p := range networkInfo.Peers {
+		peerIPs[i] = p.IP
+	}
+
+	// Build set of IPs already used on this network (avoids 20s timeout for known conflicts)
+	usedIPs := make(map[string]bool)
+	for _, container := range networkInfo.Containers {
+		if container.IPv4Address != "" {
+			ip, _, parseErr := net.ParseCIDR(container.IPv4Address)
+			if parseErr == nil {
+				usedIPs[ip.String()] = true
+			}
 		}
-		return ip.String(), true
-	})
-	if gw := networkInfo.IPAM.Config[0].Gateway; gw != "" {
-		usedIPs = append(usedIPs, gw)
 	}
 
-	var candidates []net.IP
-	broadcast := broadcastAddr(ipNet)
-	candidate := prevIP(broadcast)
-	skipped := 0
-	for ; ipNet.Contains(candidate) && !candidate.Equal(ipNet.IP) && len(candidates) < 100; candidate = prevIP(candidate) {
-		if lo.Contains(usedIPs, candidate.String()) {
+	candidates := computeIPCandidatesMultiRound(ipNet, peerIPs, m.nodeAddr, m.overlayContainers, 3)
+
+	// Pick this container's lane (every overlayContainers-th IP), skipping locally-visible conflicts
+	selected := []net.IP{}
+	for i := overlayIndex; i < len(candidates); i += m.overlayContainers {
+		if usedIPs[candidates[i].String()] {
+			m.l.WithField("network", networkName).Debugf("skipping %s (already used on this node)", candidates[i])
 			continue
 		}
-		if skipped < m.nodeOffset {
-			skipped++
-			continue
+		selected = append(selected, candidates[i])
+	}
+	return selected
+}
+
+func computeIPCandidates(ipNet *net.IPNet, peerIPs []string, nodeAddr string, overlayContainers int) []net.IP {
+	return computeIPCandidatesMultiRound(ipNet, peerIPs, nodeAddr, overlayContainers, 1)
+}
+
+// computeIPCandidatesMultiRound generates candidate IPs across multiple rounds.
+// Each round provides overlayContainers IPs for this node, spaced below all peers' bands.
+func computeIPCandidatesMultiRound(ipNet *net.IPNet, peerIPs []string, nodeAddr string, overlayContainers int, maxRounds int) []net.IP {
+	peerIPs = slices.Clone(peerIPs)
+	slices.SortFunc(peerIPs, func(a, b string) int {
+		return bytes.Compare(net.ParseIP(a).To4(), net.ParseIP(b).To4())
+	})
+
+	peerIndex := len(peerIPs)
+	for i, ip := range peerIPs {
+		if ip == nodeAddr {
+			peerIndex = i
+			break
+		}
+	}
+
+	numPeers := len(peerIPs)
+	broadcast := broadcastAddr(ipNet)
+	var candidates []net.IP
+	for round := range maxRounds {
+		startOffset := 1 + (round*numPeers+peerIndex)*overlayContainers
+		for i := range overlayContainers {
+			ip := addToIP(broadcast, -(startOffset + i))
+			if !ipNet.Contains(ip) || ip.Equal(ipNet.IP) {
+				return candidates
+			}
+			candidates = append(candidates, ip)
 		}
-		candidates = append(candidates, candidate)
 	}
 
 	return candidates
@@ -608,14 +653,14 @@ func broadcastAddr(n *net.IPNet) net.IP {
 	return broadcast
 }
 
-func prevIP(ip net.IP) net.IP {
-	prev := make(net.IP, len(ip))
-	copy(prev, ip)
-	for i := len(prev) - 1; i >= 0; i-- {
-		prev[i]--
-		if prev[i] != 255 {
-			break
-		}
+func addToIP(ip net.IP, delta int) net.IP {
+	result := make(net.IP, len(ip))
+	copy(result, ip)
+	carry := delta
+	for i := len(result) - 1; i >= 0 && carry != 0; i-- {
+		sum := int(result[i]) + carry
+		result[i] = byte(sum & 0xFF)
+		carry = sum >> 8
 	}
-	return prev
+	return result
 }