Use Sonic VPP image

.github/workflows/base-image.yaml

@@ -16,6 +16,7 @@ jobs:
           - name: 202311
           - name: 202411
           - name: 202505
+          - name: 202511-vpp
 
     steps:
     - name: Log in to the container registry

Makefile

@@ -10,8 +10,8 @@ KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml)
 KUBECONFIG := $(shell pwd)/.kubeconfig
 
 METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin)
-METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal)
-METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.17.0.1.nip.io:8080)
+METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.42.0.42.nip.io:8080/metal)
+METAL_APIV2_URL := $(or $(METAL_APIV2_URL),http://v2.172.42.0.42.nip.io:8080)
 
 MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf
 # Default values
@@ -24,9 +24,14 @@ ANSIBLE_DISPLAY_SKIPPED_HOSTS=false
 
 MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic)
 MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
-MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
+MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:202511-vpp)
 MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)
 
+MINI_LAB_INTERNAL_NETWORK=mini_lab_internal
+# define this here as well so that kind picks up the network on a clean checkout,
+# where .env does not exist yet at make parse time (-include .env above)
+KIND_EXPERIMENTAL_DOCKER_NETWORK := $(or $(KIND_EXPERIMENTAL_DOCKER_NETWORK),$(MINI_LAB_INTERNAL_NETWORK))
+
 MACHINE_OS=debian-13.0
 MAX_RETRIES := 30
 
@@ -79,20 +84,12 @@ up: env gen-certs verify-deployment-image control-plane-bake partition-bake
 	@chmod 600 files/ssh/id_ed25519
 	docker compose $(COMPOSE_ARGS) up --pull=always --abort-on-container-failure --remove-orphans --force-recreate control-plane partition
 	@$(MAKE)	--no-print-directory	start-machines
-# for some reason an allocated machine will not be able to phone home
-# without restarting the metal-core
-# TODO: should be investigated and fixed if possible
-# check that underlay gets working
+
+# on old dell_sonic flavor metal-core needs a restart for nodes to register
+ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),)
 	sleep 10
 	ssh -F files/ssh/config leaf01 'systemctl restart metal-core'
 	ssh -F files/ssh/config leaf02 'systemctl restart metal-core'
-
-# for community SONiC versions > 202311 a bgp restart is needed in the virtual environment
-# TODO: should be investigated and fixed if possible
-ifeq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),)
-	sleep 15
-	ssh -F files/ssh/config leaf01 'systemctl restart bgp'
-	ssh -F files/ssh/config leaf02 'systemctl restart bgp'
 endif
 
 .PHONY: restart
@@ -126,13 +123,16 @@ create-proxy-registries:
 
 .PHONY: control-plane-bake
 control-plane-bake:
+
+	@if ! docker network ls | grep -q mini_lab_internal; then docker network create mini_lab_internal --gateway 172.42.0.1 --ip-range=172.42.0.0/16 --subnet=172.42.0.0/16 --ipv6=false ; fi
 	@if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi
 	@if ! kind get clusters | grep metal-control-plane > /dev/null; then \
 		kind create cluster $(KIND_ARGS) \
 			--name metal-control-plane \
 			--config $(KINDCONFIG) \
 			--kubeconfig $(KUBECONFIG); fi
 	$(MAKE) create-proxy-registries
+	docker compose up -d --force-recreate cloud-provider-kind
 
 .PHONY: partition
 partition: partition-bake
@@ -184,6 +184,7 @@ env:
 
 .PHONY: cleanup
 cleanup: cleanup-control-plane cleanup-partition
+	docker network rm --force mini_lab_internal
 
 .PHONY: cleanup-control-plane
 cleanup-control-plane:
@@ -430,6 +431,7 @@ build-sonic-base:
 	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311
 	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411
 	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505
+	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202511-vpp images/sonic/base-202511-vpp
 
 ## DEV TARGETS ##
 
@@ -457,7 +459,7 @@ build-dell-sonic:
 fetch-virtual-kubeconfig:
 	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
 	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
-	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443
 	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden

compose.yaml

@@ -126,6 +126,20 @@ services:
       - REGISTRY_PROXY_TTL=168h
       - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
+  cloud-provider-kind:
+    image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0
+    restart: always
+    networks:
+      - kind
+    environment:
+      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind}
+    command:
+    # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel
+    # using envoy-gateway deployed via roles/gateway instead
+      - --gateway-channel
+      - disabled
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
 volumes:
   proxy-docker:
   proxy-gcr:

control-plane/kind.yaml

@@ -2,7 +2,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   apiServerPort: 6443
-  apiServerAddress: 0.0.0.0
+  apiServerAddress: 172.42.0.1
 nodes:
   - role: control-plane
     extraMounts:
@@ -13,10 +13,6 @@ nodes:
         hostPort: 4443
       - containerPort: 8080
         hostPort: 8080
-      - containerPort: 4150
-        hostPort: 4150
-      - containerPort: 50051
-        hostPort: 50051
     # if you want to run gardener operator + metal-stack, you need more pods
     kubeadmConfigPatches:
       - |

deploy_control_plane.yaml

@@ -6,10 +6,12 @@
   roles:
     - name: ansible-common
       tags: always
-    - name: ingress-controller
-      tags: ingress-controller
     - name: metal-roles/control-plane/roles/prepare
       tags: prepare
+    - name: gateway
+      tags: gateway
+    - name: ingress-controller
+      tags: ingress-controller
     - name: metal-roles/control-plane/roles/nsq
       tags: nsq
     - name: metal-roles/control-plane/roles/metal-db

deploy_gardener.yaml

@@ -13,7 +13,7 @@
             name: shoot-info
             namespace: kube-system
           data:
-            nodeNetwork: 172.18.0.0/16
+            nodeNetwork: 172.42.0.0/16
             podNetwork: 10.244.0.0/24
             serviceNetwork: 10.96.0.0/16
       tags: gardener
@@ -81,7 +81,7 @@
             status:
               loadBalancer:
                 ingress:
-                  - ip: "172.17.0.1"
+                  - ip: "172.42.0.1"
         tags: gardener
 
       - name: Expose istio gateway through ingress-nginx (for local environments)

deploy_partition.yaml

@@ -7,7 +7,7 @@
     - name: Wait for system to become reachable
       ansible.builtin.wait_for_connection:
         delay: 10
-        timeout: 50
+        timeout: 120
   roles:
     - name: ansible-common
       tags: always
@@ -132,14 +132,35 @@
       when: monitoring_enabled
       tags: sonic-exporter
 
-- name: Deploy metal-core
-  hosts: leaves
+# So, route propagation was broken in community sonic for a while now. Turns out
+# during a change in upstream sonic some time in the end of 2024 sonic switched
+# the module used for FRR->fpmsyncd communication, which does not provide a
+# default connection endpoint. We just needed to add `fpm address 127.0.0.1`
+# to the frr.conf template and could remove all route-propagation related 
+# workarounds
+- name: Deploy metal-core (SONiC with dplane_fpm_sonic)
+  hosts: leaves:!dell_sonic
   any_errors_fatal: true
   become: true
   pre_tasks:
-    - name: Wait some time
-      pause:
-        seconds: 120
+    - name: copy custom sonic_frr.tpl with fpm configured
+      ansible.builtin.copy:
+        dest: /sonic_frr.tpl
+        src: /mini-lab/files/sonic_frr_with_fpm.tpl
+  roles:
+    - name: ansible-common
+      tags: always
+    - name: metal-roles/partition/roles/metal-core
+      tags: metal-core
+      vars:
+        metal_core_frr_tpl_file: /sonic_frr.tpl
+        metal_core_additional_volume_mounts:
+        - /sonic_frr.tpl:/sonic_frr.tpl
+
+- name: Deploy metal-core (Dell SONiC without dplane_fpm_sonic)
+  hosts: dell_sonic
+  any_errors_fatal: true
+  become: true
   roles:
     - name: ansible-common
       tags: always

env.sh

@@ -24,4 +24,5 @@ DEPLOYMENT_BASE_IMAGE_TAG=${DEPLOYMENT_BASE_IMAGE_TAG}
 CI=${CI:=false}
 DOCKER_HUB_USER=${DOCKER_HUB_USER:=}
 DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN:=}
+KIND_EXPERIMENTAL_DOCKER_NETWORK=${MINI_LAB_INTERNAL_NETWORK:=mini_lab_internal}
 EOF

files/certs/grpc/server.json

@@ -1,7 +1,7 @@
 {
   "CN": "metal-api",
   "hosts": [
-    "172.17.0.1",
+    "172.42.0.42",
     "203.0.113.1"
   ],
   "key": {

files/certs/metal-api/server.json

@@ -0,0 +1,20 @@
+{
+  "CN": "metal-api",
+  "hosts": [
+    "api.172.42.0.42.nip.io",
+    "v2.172.42.0.42.nip.io"
+  ],
+  "key": {
+    "algo": "rsa",
+    "size": 4096
+  },
+  "names": [
+    {
+      "C":  "DE",
+      "L":  "Munich",
+      "O":  "metal-stack",
+      "OU": "DevOps",
+      "ST": "Bavaria"
+    }
+  ]
+}

files/dev_images.yaml

@@ -1,6 +1,6 @@
 ---
 # Do not change these values
 metal_api_image_tag: dev
-metal_core_image_name: 172.17.0.1:5000/metalstack/metal-core
+metal_core_image_name: 172.42.0.42:5000/metalstack/metal-core
 metal_core_image_tag: dev
-metal_hammer_image_url: http://172.17.0.1:20015/metal-hammer-initrd.img.lz4
+metal_hammer_image_url: http://172.42.0.42:20015/metal-hammer-initrd.img.lz4