[WIP] Test Cilium 4.20-dev with KubeVirt

ci-operator/step-registry/cilium/conf/cilium-conf-commands.sh

@@ -5,8 +5,12 @@ set -o errexit
 set -o pipefail
 set -x
 
-cilium_olm_rev="main"
-cv="$CILIUM_VERSION"
+CILIUM_VERSION="${CILIUM_VERSION:-1.19.4}"
+CILIUM_REPOSITORY="${CILIUM_REPOSITORY:-oci://quay.io/cilium/charts/cilium}"
+CILIUM_CLI_VERSION="${CILIUM_CLI_VERSION:-0.19.2}"
+ENDPOINT_ROUTES="${ENDPOINT_ROUTES:-true}"
+HUBBLE="${HUBBLE:-true}"
+SHARED_DIR="${SHARED_DIR:-/tmp/shared_dir}"
 
 if [[ -f "${SHARED_DIR}/install-config.yaml" ]]; then
   sed -i "s/networkType: .*/networkType: Cilium/" "${SHARED_DIR}/install-config.yaml"
@@ -26,60 +30,49 @@ spec:
   - 172.30.0.0/16
 EOF
 
-# OLD -- Include all Cilium OLM manifest from https://github.com/cilium/cilium-olm/tree/${cilium_olm_rev}/manifests/cilium.v${cv}
-# New -- Migrating to new OLM ( https://github.com/isovalent/olm-for-cilium )
+mkdir -p /tmp/bin
+curl --fail --retry 3 -sS -L \
+  "https://github.com/cilium/cilium-cli/releases/download/v${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz" \
+  | tar -xzC /tmp/bin/
+chmod +x /tmp/bin/cilium
+export PATH=/tmp/bin:$PATH
 
-OLM_URL="https://github.com/isovalent/olm-for-cilium"
-
-curl --silent --location --fail --show-error "${OLM_URL}/archive/${cilium_olm_rev}.tar.gz" --output /tmp/cilium-olm.tgz
-tar -C /tmp -xf /tmp/cilium-olm.tgz
-
-cd "/tmp/olm-for-cilium-${cilium_olm_rev}/manifests/cilium.v${cv}"
-# Overwrite the CiliumConfig
-cat > cluster-network-07-cilium-ciliumconfig.yaml << EOF
-apiVersion: cilium.io/v1alpha1
-kind: CiliumConfig
+cat > "${SHARED_DIR}/manifest_cilium-00-namespace.yaml" <<EOF
+apiVersion: v1
+kind: Namespace
 metadata:
   name: cilium
+EOF
+
+# Workaround for OCPBUGS-86033: override the default 0.3.1 cniVersion
+cat > "${SHARED_DIR}/manifest_cilium-00-cni-override-configmap.yaml" <<EOF
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium-cni-override
   namespace: cilium
-spec:
-  cni:
-    binPath: /var/lib/cni/bin
-    confPath: /var/run/multus/cni/net.d
-  endpointRoutes:
-    enabled: ${ENDPOINT_ROUTES}
-  hubble:
-    enabled: ${HUBBLE}
-  ipam:
-    mode: cluster-pool
-    operator:
-      clusterPoolIPv4MaskSize: "23"
-      clusterPoolIPv4PodCIDRList:
-      - 10.128.0.0/14
-  kubeProxyReplacement: disabled
-  nativeRoutingCIDR: 10.128.0.0/14
-  operator:
-    prometheus:
-      enabled: true
-      serviceMonitor:
-        enabled: true
-  prometheus:
-    enabled: true
-    serviceMonitor:
-      enabled: true
-  securityContext:
-    privileged: true
-  sessionAffinity: true
-  clusterHealthPort: 9940
-  tunnelPort: 4789
+data:
+  cilium-override.conf: |
+    {
+      "cniVersion": "0.4.0",
+      "name": "portmap",
+      "plugins": [
+        {
+            "type": "cilium-cni",
+            "enable-debug": true,
+            "log-file": "/var/run/cilium/cilium-cni.log"
+        },
+        {
+          "type": "portmap",
+          "capabilities": {"portMappings": true}
+        }
+      ]
+    }
 EOF
-for manifest in *.yaml ; do
-  cp "${manifest}" "${SHARED_DIR}/manifest_${manifest}"
-done
 
 # Workaround for OCPBUGS-85607: Apply Cilium NetworkPolicy to allow DNS pods to reach kube-apiserver
 # This needs to be applied on the management cluster for Hypershift Cilium jobs
-cat > "${SHARED_DIR}/manifest_cilium-network-policy-dns.yaml" <<EOF
+cat > "${SHARED_DIR}/manifest_cilium-00-network-policy-dns.yaml" <<EOF
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
@@ -94,3 +87,71 @@ spec:
     - host
     - kube-apiserver
 EOF
+
+cat > "${SHARED_DIR}/manifest_cilium-00-scc-privileged.yaml" <<EOF
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-scc-privileged
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:openshift:scc:privileged
+subjects:
+- kind: ServiceAccount
+  name: cilium
+  namespace: cilium
+- kind: ServiceAccount
+  name: cilium-operator
+  namespace: cilium
+- kind: ServiceAccount
+  name: cilium-envoy
+  namespace: cilium
+EOF
+
+WORKDIR=$(mktemp -d)
+
+# Note: In order to test with a development version, use:
+# --repository oci://quay.io/cilium-charts-dev/cilium --version <version>
+# where <version> is a tag from https://quay.io/repository/cilium-charts-dev/cilium
+cilium install \
+    --dry-run \
+    --namespace cilium \
+    --repository "${CILIUM_REPOSITORY}" \
+    --version "${CILIUM_VERSION}" \
+    --set debug.enabled=true \
+    --set k8s.requireIPv4PodCIDR=true \
+    --set logSystemLoad=true \
+    --set ipv6.enabled=false \
+    --set identityChangeGracePeriod=0s \
+    --set ipam.mode=cluster-pool \
+    --set "ipam.operator.clusterPoolIPv4PodCIDRList={10.128.0.0/14}" \
+    --set ipam.operator.clusterPoolIPv4MaskSize=23 \
+    --set ipv4NativeRoutingCIDR=10.128.0.0/14 \
+    --set cni.binPath=/var/lib/cni/bin \
+    --set cni.confPath=/var/run/multus/cni/net.d \
+    --set sessionAffinity=true \
+    --set endpointRoutes.enabled="${ENDPOINT_ROUTES}" \
+    --set hubble.enabled="${HUBBLE}" \
+    --set tunnelPort=4790 \
+    --set clusterHealthPort=9940 \
+    --set socketLB.enabled=true \
+    --set cni.readCniConf=/etc/cilium-cni/cilium-override.conf \
+    --set extraVolumes[0].name=cni-override \
+    --set extraVolumes[0].configMap.name=cilium-cni-override \
+    --set extraVolumeMounts[0].name=cni-override \
+    --set extraVolumeMounts[0].mountPath=/etc/cilium-cni \
+    > "${WORKDIR}/cilium-install-all.yaml"
+
+# Split the multi-document YAML into individual manifest files
+csplit -z -f "${WORKDIR}/cilium-part-" -b '%02d.yaml' "${WORKDIR}/cilium-install-all.yaml" '/^---$/' '{*}'
+INDEX=1
+for f in "${WORKDIR}"/cilium-part-*.yaml; do
+  sed -i '/^---$/d' "$f"
+  [[ ! -s "$f" ]] && rm -f "$f" && continue
+  PADDED=$(printf "%02d" "$INDEX")
+  KIND=$(grep '^kind:' "$f" | head -1 | awk '{print $2}' | tr '[:upper:]' '[:lower:]')
+  NAME=$(grep '^  name:' "$f" | head -1 | awk '{print $2}' | tr -d '"')
+  mv "$f" "${SHARED_DIR}/manifest_cilium-${PADDED}-${KIND}-${NAME}.yaml"
+  INDEX=$((INDEX + 1))
+done

ci-operator/step-registry/cilium/conf/cilium-conf-ref.yaml

@@ -8,9 +8,17 @@ ref:
       memory: 100Mi
   env:
   - name: CILIUM_VERSION
-    default: "1.13.9"
+    default: "1.19.4"
     documentation: |-
-      This var will default to 1.13.9, however users can override to a different Cilium version, ensure the manifests exists in the repo.
+      Cilium version to install.
+  - name: CILIUM_REPOSITORY
+    default: "oci://quay.io/cilium/charts/cilium"
+    documentation: |-
+      Cilium repository to install from.
+  - name: CILIUM_CLI_VERSION
+    default: "0.19.2"
+    documentation: |-
+      Cilium CLI version used to generate installation manifests.
   - name: HUBBLE
     default: "true"
     documentation: |-
@@ -20,4 +28,4 @@ ref:
     documentation: |-
       This var will default to true, however users can override to disable endpoint routes.
   documentation: |-
-    This steps installs Cilium OLM manifests (https://github.com/cilium/cilium-olm)
+    This step generates Cilium CNI manifests using cilium install --dry-run for day-0 cluster installation.

ci-operator/step-registry/cucushift/hypershift-extended/cilium/cucushift-hypershift-extended-cilium-commands.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 CILIUM_VERSION=${CILIUM_VERSION:-"1.19.4"}
 CILIUM_CLI_VERSION=${CILIUM_CLI_VERSION:-"0.19.2"}
+CILIUM_REPOSITORY=${CILIUM_REPOSITORY:-"oci://quay.io/cilium/charts/cilium"}
 
 function set_proxy () {
     if test -s "${SHARED_DIR}/proxy-conf.sh" ; then
@@ -76,7 +77,7 @@ EOF
 # where <version> is a tag from https://quay.io/repository/cilium-charts-dev/cilium
 cilium install \
     --namespace cilium \
-    --repository oci://quay.io/cilium/charts/cilium \
+    --repository "${CILIUM_REPOSITORY}" \
     --version "${CILIUM_VERSION}" \
     --set debug.enabled=true \
     --set k8s.requireIPv4PodCIDR=true \

ci-operator/step-registry/cucushift/hypershift-extended/cilium/cucushift-hypershift-extended-cilium-ref.yaml

@@ -10,6 +10,10 @@ ref:
   env:
   - name: CILIUM_VERSION
     default: "1.19.4"
+  - name: CILIUM_REPOSITORY
+    default: "oci://quay.io/cilium/charts/cilium"
+    documentation: |-
+      The Cilium repository to install from.
   - name: CILIUM_CLI_VERSION
     default: "0.19.2"
     documentation: |-

ci-operator/step-registry/hypershift/kubevirt/baremetalds/conformance-cilium/hypershift-kubevirt-baremetalds-conformance-cilium-workflow.yaml

@@ -10,13 +10,16 @@ workflow:
     Track HyperShift's development here: https://issues.redhat.com/projects/HOSTEDCP
   steps:
     post:
+    - ref: wait
     - chain: hypershift-dump
     - chain: gather-core-dump
+    - chain: cilium-dump
     - chain: hypershift-kubevirt-destroy
     - chain: baremetalds-ofcir-post
     test:
     - chain: hypershift-conformance
     pre:
+    - ref: cilium-conf
     - chain: baremetalds-ofcir-pre
     - ref: enable-qe-catalogsource
     - ref: deploy-konflux-operator
@@ -31,6 +34,8 @@ workflow:
     - ref: cucushift-hypershift-extended-cilium-network-policies
     - ref: cucushift-hypershift-extended-cilium-health-check
     env:
+      CILIUM_VERSION: "1.20.0-dev-dev.505-e54a19029f1d"
+      CILIUM_REPOSITORY: "oci://quay.io/cilium-charts-dev/cilium"
       HYPERSHIFT_NETWORK_TYPE: "Other" # Required for Cilium.
       KONFLUX_DEPLOY_OPERATORS: "false"
       KONFLUX_TARGET_OPERATORS: metallb,local-storage
@@ -46,8 +51,9 @@ workflow:
         should ensure an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed\|
         Services should serve endpoints on same port and different protocols\|
         Netpol NetworkPolicy between server and client should enforce
-        except clause while egress access to server in CIDR block\| Unidling
-        \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
+        except clause while egress access to server in CIDR block\|
+        Netpol NetworkPolicy between server and client should deny ingress access to updated pod\|
+        Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
         should work with UDP\| Unidling with Deployments \[apigroup:route.openshift.io\]
         should work with TCP (when fully idled)\| Unidling
         \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\] should work with
@@ -62,11 +68,12 @@ workflow:
         evicted pods\|\[ovn-kubernetes-ote\]
       DEVSCRIPTS_CONFIG: |
         IP_STACK=v4
-        NETWORK_TYPE=OVNKubernetes
+        NETWORK_TYPE=Cilium
         NUM_WORKERS=0
         NUM_MASTERS=3
         MASTER_VCPU=16
         MASTER_MEMORY=81920
         VM_EXTRADISKS=true
         VM_EXTRADISKS_LIST="vda vdb"
         VM_EXTRADISKS_SIZE=250G
+        ENABLE_LOCAL_REGISTRY=

ci-operator/step-registry/hypershift/mce/agent/metal3/conformance/cilium/hypershift-mce-agent-metal3-conformance-cilium-workflow.yaml

@@ -31,8 +31,9 @@ workflow:
         should ensure an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed\|
         Services should serve endpoints on same port and different protocols\|
         Netpol NetworkPolicy between server and client should enforce
-        except clause while egress access to server in CIDR block\| Unidling
-        \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
+        except clause while egress access to server in CIDR block\|
+        Netpol NetworkPolicy between server and client should deny ingress access to updated pod\|
+        Unidling \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\]
         should work with UDP\| Unidling with Deployments \[apigroup:route.openshift.io\]
         should work with TCP (when fully idled)\| Unidling
         \[apigroup:apps.openshift.io\]\[apigroup:route.openshift.io\] should work with