From b24302761717147247c8ec91a6a07f67b2a5d688 Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Tue, 12 May 2026 17:55:33 +0200 Subject: [PATCH 1/3] First draft --- .../tenant-network/.$detail.drawio.bkp | 190 +++++++++++++ .../tenant-network/.$overview.drawio.bkp | 79 ++++++ .../tenant-network/api-lb.conf | 29 ++ .../tenant-network/detail.drawio | 187 +++++++++++++ .../tenant-network/index.md | 221 +++++++++++++++ .../ingress-controller-shard.yaml | 17 ++ .../ingress-shared-haproxy.conf | 39 +++ .../tenant-network/overview.drawio | 264 ++++++++++++++++++ .../tenant-network/vyos-router-2003.txt | 27 ++ mkdocs.yml | 3 + 10 files changed, 1056 insertions(+) create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/api-lb.conf create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/index.md create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/ingress-shared-haproxy.conf create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp b/content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp new file mode 100644 index 00000000..076cc4a1 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp @@ -0,0 +1,190 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp b/content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp new file mode 100644 index 00000000..3d1d9e74 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/api-lb.conf b/content/cluster-installation/hosted-control-plane/tenant-network/api-lb.conf new file mode 100644 index 00000000..b5a7e9a1 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/api-lb.conf @@ -0,0 +1,29 @@ +global + log 127.0.0.1 local2 + pidfile /var/run/haproxy.pid + maxconn 4000 + daemon +defaults + mode http + log global + option dontlognull + option http-server-close + option redispatch + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s + maxconn 3000 + +listen api + bind *:6443 + mode tcp + balance source + server ucs-blade-server-5 10.32.96.105:30918 check inter 1s + server ucs-blade-server-6 10.32.96.106:30918 check inter 1s + server ucs-blade-server-7 10.32.96.107:30918 check inter 1s + server ucs-blade-server-8 10.32.96.108:30918 check inter 1s diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio b/content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio new file mode 100644 index 00000000..8057f4e0 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio @@ -0,0 +1,187 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md new file mode 100644 index 00000000..a46cb89e --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -0,0 +1,221 @@ +--- +title: Hosted Control Plane and tenant networking +linktitle: Hosted Control Plane and tenant networking +description: Hosted Control Plane and tenant networking +tags: ['hcp','v4.21'] +--- +# Hosted Control Plane and tenant networking + +Official documentation: Net yet available + +Tested with: + +|Component|Version| +|---|---| +|OpenShift|v4.21.9| +|OpenShift Virt|v4.21.0| + +## Overview + +Challenge: running an hosted cluster with in different tenant network segment/vlan without widely open access from tenant segment to managment segment. + +Addtional requirement, the hub cluster should not have any address or network connection into the tenant network segment. It's only allowed to place virtual machines into the network segment. + +![](overview.drawio){ page="Page-1" } + +The worker nodes of the hosted cluster are quite easy to solve, just connected them into the tenant network segment (import, DHCP is required). + +The hosted control plane compontents to expose into tenant network segment is more challenging. Following components have to concider: + +* API Server +* OAuth +* Konnectivity +* Ignition + +Here an list of possible exposing options for these components: + +|Component/Service|Exposing strategy (`servicePublishingStrategy`)|Kubernetes Service type `LoadBalancer`|Ingress/Route| +|---|---|---|---| +|API Server|
  • LoadBalancer (Recommended, K8s Service Type Load Balancer)
  • NodePort* (not for production)
    • |✅|❌| +|OAuth|
  • Route/Ingress (default)
  • NodePort* (not for production)
    • |❌|✅| +|Konnectivity|
  • Route/Ingress (default)
  • LoadBalancer (K8s Service Type Load Balancer)
  • NodePort* (not for production)
    • |✅|✅| +|Ignition|
  • Route/Ingress (default)
  • NodePort* (not for production)
    • |✅|❌| + +For our proof of concept we want to try following, exposing the components via: + +* API Server: LoadBalancer +* OAuth: Router/Ingress: via a dedicted router shard. +* Konnectivity: via a dedicted router shard. +* Ignition: via a dedicted router shard. + +## Exposing compontents via router/ingress shard + +The idea with the dedicated router/ingress shared is to expose the router/ingress shard into the tenant network segment and only for the hosted cluster components. + +In front of the router/ingress shared is an external load balancer (for example, f5 bigip, netscaler,..) with access into the managment network segment and expose the router shared into the tenant network segment. + + + + +## Proof of concept envrioment overview + +![](overview.drawio){ page="Page-2" } + +### Router between Mgmt and Tenant-A + +[VyOS Router](https://vyos.io/) router & firewall. Do not allow Traffic between Mgmt and Tenant-A network except DNS and gateway. + +??? example "VyOS config commands" + + ```shell + --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt" + ``` + +### Ingress Sharding + +* [2.3.4. Ingress sharding in OpenShift Container Platform](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/ingress_and_load_balancing/configuring-ingress-cluster-traffic#nw-ingress-sharding-concept_configuring-ingress-cluster-traffic-ingress-controller) +* [3.1.3.8.1. Example load balancer configuration for user-provisioned clusters](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/installing_on_vmware_vsphere/user-provisioned-infrastructure) + + +???+ example "Ingress Controller" + + ```yaml + --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml" + ``` + +```shell +% oc get svc -n openshift-ingress router-nodeport-tenant-a +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +router-nodeport-tenant-a NodePort 172.30.141.209 80:32460/TCP,443:32488/TCP,1936:32095/TCP 106s +``` + +Ingress sharding load balancer is an RHEL 9 system with haproxy. + +* Install HAProxy `dnf install haproxy` +* Configure selinux `setsebool -P haproxy_connect_any 1` +* Apply Example haproxy.conf (don't forget to update ports) +* Enabel and start haproxy `systemctl enable --now haproxy` + +??? example "HAProxy config" + + ```shell + --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/ingress-shared-haproxy.conf" + ``` + +Add DNS Records + +```bind +konnectivity.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 +oauth.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 +ignition.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 +``` + + + + + + + + + +apiVersion: project.openshift.io/v1 +kind: Project +metadata: + name: 'clusters' +--- +apiVersion: hypershift.openshift.io/v1beta1 +kind: HostedCluster +metadata: + name: 'tenant-a' + namespace: 'clusters' + labels: + "cluster.open-cluster-management.io/clusterset": 'default' +spec: + configuration: + ingress: + appsDomain: apps.tenant-a.coe.muc.redhat.com + domain: '' + loadBalancer: + platform: + type: '' + channel: fast-4.21 + etcd: + managed: + storage: + persistentVolume: + size: 8Gi + type: PersistentVolume + managementType: Managed + release: + image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi + pullSecret: + name: pullsecret-cluster-tenant-a + sshKey: + name: sshkey-cluster-tenant-a + networking: + clusterNetwork: + - cidr: 10.132.0.0/14 + serviceNetwork: + - cidr: 172.31.0.0/16 + networkType: OVNKubernetes + controllerAvailabilityPolicy: SingleReplica + infrastructureAvailabilityPolicy: SingleReplica + platform: + type: KubeVirt + kubevirt: + baseDomainPassthrough: false + infraID: 'tenant-a' + services: + - service: APIServer + servicePublishingStrategy: + type: LoadBalancer + loadBalancer: + hostname: api.tenant-a.coe.muc.redhat.com + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: + hostname: oauth.tenant-a.coe.muc.redhat.com + - service: OIDC + servicePublishingStrategy: + type: Route + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: + hostname: konnectivity.tenant-a.coe.muc.redhat.com + - service: Ignition + servicePublishingStrategy: + type: Route + route: + hostname: ignition.tenant-a.coe.muc.redhat.com + +--- +apiVersion: hypershift.openshift.io/v1beta1 +kind: NodePool +metadata: + name: 'tenant-a' + namespace: 'clusters' +spec: + arch: amd64 + clusterName: 'tenant-a' + replicas: 2 + management: + autoRepair: false + upgradeType: Replace + platform: + type: KubeVirt + kubevirt: + compute: + cores: 2 + memory: 8Gi + rootVolume: + type: Persistent + persistent: + size: 32Gi + additionalNetworks: + - name: default/cudn-localnet1-2003 + attachDefaultNetwork: false + release: + image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi + diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml new file mode 100644 index 00000000..fcd9b0a1 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml @@ -0,0 +1,17 @@ +apiVersion: operator.openshift.io/v1 +kind: IngressController +metadata: + name: tenant-a + namespace: openshift-ingress-operator +spec: + domain: tenant-a.coe.muc.redhat.com + + endpointPublishingStrategy: + type: NodePortService + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - ingress-test + - clusters-tenant-a \ No newline at end of file diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/ingress-shared-haproxy.conf b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-shared-haproxy.conf new file mode 100644 index 00000000..e111e741 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-shared-haproxy.conf @@ -0,0 +1,39 @@ +global + log 127.0.0.1 local2 + pidfile /var/run/haproxy.pid + maxconn 4000 + daemon +defaults + mode http + log global + option dontlognull + option http-server-close + option redispatch + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s + maxconn 3000 + +listen ingress-router-443 + bind *:443 + mode tcp + balance source + server ucs-blade-server-5 10.32.96.105:32488 check inter 1s + server ucs-blade-server-6 10.32.96.106:32488 check inter 1s + server ucs-blade-server-7 10.32.96.107:32488 check inter 1s + server ucs-blade-server-8 10.32.96.108:32488 check inter 1s + +listen ingress-router-80 + bind *:80 + mode tcp + balance source + server ucs-blade-server-5 10.32.96.105:32460 check inter 1s + server ucs-blade-server-6 10.32.96.106:32460 check inter 1s + server ucs-blade-server-7 10.32.96.107:32460 check inter 1s + server ucs-blade-server-8 10.32.96.108:32460 check inter 1s + diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio b/content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio new file mode 100644 index 00000000..af78cf9b --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio @@ -0,0 +1,264 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt b/content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt new file mode 100644 index 00000000..5e09bc44 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt @@ -0,0 +1,27 @@ +set firewall group address-group ALLOWED-IPS address '10.32.96.1' +set firewall group address-group ALLOWED-IPS address '10.32.96.31' +set firewall group address-group ALLOWED-IPS address '10.32.111.254' +set firewall ipv4 forward filter rule 49 action 'accept' +set firewall ipv4 forward filter rule 49 description 'Allow IPs' +set firewall ipv4 forward filter rule 49 destination group address-group 'ALLOWED-IPS' +set firewall ipv4 forward filter rule 50 action 'drop' +set firewall ipv4 forward filter rule 50 description 'Drop enire coe lab' +set firewall ipv4 forward filter rule 50 destination address '10.32.96.0/20' + +set interfaces ethernet eth0 address 'dhcp' +set interfaces ethernet eth1 address '192.168.203.1/24' + +set nat source rule 100 outbound-interface name 'eth0' +set nat source rule 100 source address '192.168.203.0/24' +set nat source rule 100 translation address 'masquerade' +set service dhcp-server listen-interface 'eth1' +set service dhcp-server shared-network-name coe-2003 authoritative +set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 option default-router '192.168.203.1' +set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 option name-server '10.32.96.1' +set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 range 1 start '192.168.203.100' +set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 range 1 stop '192.168.203.200' +set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 subnet-id '1' +set service ssh +set system host-name 'router-2003' +set system name-server '10.32.96.1' +set system name-server '10.32.96.31' diff --git a/mkdocs.yml b/mkdocs.yml index 688dc6c3..ed84e0bc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -57,6 +57,7 @@ extra: extra_javascript: - https://viewer.diagrams.net/js/viewer-static.min.js - javascripts/drawio-reload.js + # Extensions markdown_extensions: - pymdownx.emoji: @@ -99,6 +100,7 @@ plugins: verbose: false - glightbox - drawio: + viewer_js: "https://viewer.diagrams.net/js/viewer-static.min.js" toolbar: false # control if hovering on a diagram shows a toolbar for zooming or not (default: true) tooltips: false # control if tooltips will be shown (default: true) edit: false # control if edit button will be shown in the lightbox view (default: true) @@ -131,6 +133,7 @@ nav: - Hosted Control Plane: - cluster-installation/hosted-control-plane/index.md - KubeVirt Networking: cluster-installation/hosted-control-plane/kubevirt-networking.md + - Tenant Network: cluster-installation/hosted-control-plane/tenant-network/index.md - STACKIT: - cluster-installation/stackit/index.md - Nvidia GPU: From 3e8cd96e35fc5806baf49037e6b665f01884091f Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 1 May 2026 23:05:55 +0200 Subject: [PATCH 2/3] Update mkdocs-drawio from 1.8.2 to 1.15.0 --- requirements.txt | 2 +- run-local.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3502fd9b..56da76a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ git+https://github.com/fralau/mkdocs_macros_plugin.git@v1.3.7 # Only for pre-commit checks not for mkdocs it selfe pre-commit==4.0.1 mkdocs-git-authors-plugin==0.9.2 -mkdocs-drawio==1.8.2 +mkdocs-drawio==1.15.0 mike==2.1.3 diff --git a/run-local.sh b/run-local.sh index 25ab3bbc..83382995 100755 --- a/run-local.sh +++ b/run-local.sh @@ -1,4 +1,4 @@ podman run -ti --user 0 --rm \ -v $(pwd):/opt/app-root/src:z \ - -p 8080:8080 quay.io/openshift-examples/builder:202601121657 + -p 8080:8080 quay.io/openshift-examples/builder:202604300846 From 84e9eef4b77d086aabd8fe1a1615ed435fe3305b Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 1 May 2026 23:09:05 +0200 Subject: [PATCH 3/3] Update tendant network --- .../tenant-network/.$detail.drawio.bkp | 190 ------------------ .../tenant-network/.$overview.drawio.bkp | 79 -------- .../tenant-network/detail.drawio | 187 ----------------- .../tenant-network/index.md | 146 +++++++++----- .../ingress-controller-shard.yaml | 10 +- .../tenant-network/ingress-lb.conf | 34 ++++ .../tenant-network/overview.drawio | 28 +-- mkdocs.yml | 1 + 8 files changed, 155 insertions(+), 520 deletions(-) delete mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp delete mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp delete mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/ingress-lb.conf diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp b/content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp deleted file mode 100644 index 076cc4a1..00000000 --- a/content/cluster-installation/hosted-control-plane/tenant-network/.$detail.drawio.bkp +++ /dev/null @@ -1,190 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp b/content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp deleted file mode 100644 index 3d1d9e74..00000000 --- a/content/cluster-installation/hosted-control-plane/tenant-network/.$overview.drawio.bkp +++ /dev/null @@ -1,79 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio b/content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio deleted file mode 100644 index 8057f4e0..00000000 --- a/content/cluster-installation/hosted-control-plane/tenant-network/detail.drawio +++ /dev/null @@ -1,187 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index a46cb89e..aa039960 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -6,7 +6,7 @@ tags: ['hcp','v4.21'] --- # Hosted Control Plane and tenant networking -Official documentation: Net yet available +Official documentation: Not yet available Tested with: @@ -17,53 +17,48 @@ Tested with: ## Overview -Challenge: running an hosted cluster with in different tenant network segment/vlan without widely open access from tenant segment to managment segment. +Challenge: running a hosted cluster in a different tenant network segment or VLAN without wide-open access from the tenant segment to the management segment. -Addtional requirement, the hub cluster should not have any address or network connection into the tenant network segment. It's only allowed to place virtual machines into the network segment. +Additional requirement: the hub cluster must not have arbitrary addressing or routing into the tenant network segment. The hub may only attach hosted-cluster workloads (for example, KubeVirt VMs) to that segment. ![](overview.drawio){ page="Page-1" } -The worker nodes of the hosted cluster are quite easy to solve, just connected them into the tenant network segment (import, DHCP is required). +Worker nodes are straightforward: attach them to the tenant network segment (DHCP or equivalent addressing is required). -The hosted control plane compontents to expose into tenant network segment is more challenging. Following components have to concider: +Exposing hosted control plane endpoints into the tenant segment is harder. The following components must be reachable from workers and clients in that segment: * API Server * OAuth * Konnectivity -* Ignition +* Ignition -Here an list of possible exposing options for these components: +Here is a summary of common publishing options for these components: -|Component/Service|Exposing strategy (`servicePublishingStrategy`)|Kubernetes Service type `LoadBalancer`|Ingress/Route| +|Component/Service|Exposing strategy (`servicePublishingStrategy`)|Kubernetes Service type `LoadBalancer`|Route (OpenShift router)| |---|---|---|---| -|API Server|
  • LoadBalancer (Recommended, K8s Service Type Load Balancer)
  • NodePort* (not for production)
    • |✅|❌| -|OAuth|
  • Route/Ingress (default)
  • NodePort* (not for production)
    • |❌|✅| -|Konnectivity|
  • Route/Ingress (default)
  • LoadBalancer (K8s Service Type Load Balancer)
  • NodePort* (not for production)
    • |✅|✅| -|Ignition|
  • Route/Ingress (default)
  • NodePort* (not for production)
    • |✅|❌| +|API Server|
  • LoadBalancer (recommended; Kubernetes `LoadBalancer` service)
  • NodePort* (not for production)
    • |✅|❌| +|OAuth|
  • Route (default)
  • NodePort* (not for production)
    • |❌|✅| +|Konnectivity|
  • Route (default)
  • LoadBalancer (Kubernetes `LoadBalancer` service)
  • NodePort* (not for production)
    • |✅|✅| +|Ignition|
  • Route (default)
  • NodePort* (not for production)
    • |✅|❌| -For our proof of concept we want to try following, exposing the components via: +For this proof of concept, endpoints are exposed as follows: -* API Server: LoadBalancer -* OAuth: Router/Ingress: via a dedicted router shard. -* Konnectivity: via a dedicted router shard. -* Ignition: via a dedicted router shard. +* API Server: `LoadBalancer` (fronted by external `api-lb` in the tenant segment; see below) +* OAuth, Konnectivity, Ignition: `Route` via a **dedicated ingress controller shard** on the hub, fronted by external `ingress-shared-lb` with VIPs/DNS in the tenant segment -## Exposing compontents via router/ingress shard +## Exposing components via a dedicated router shard -The idea with the dedicated router/ingress shared is to expose the router/ingress shard into the tenant network segment and only for the hosted cluster components. +Use a dedicated OpenShift Ingress Controller shard on the **hub** so only the hosted-cluster control-plane Routes are served by that shard. Tenant clients resolve OAuth, Konnectivity, and Ignition hostnames to `ingress-shared-lb`, which forwards to the shard’s NodePorts on the management network. -In front of the router/ingress shared is an external load balancer (for example, f5 bigip, netscaler,..) with access into the managment network segment and expose the router shared into the tenant network segment. +Place an external load balancer in front of that shard (for example F5 BIG-IP or NetScaler) that can reach the hub’s management network and present stable tenant-facing VIPs or addresses. - - - -## Proof of concept envrioment overview +## Proof of concept environment overview ![](overview.drawio){ page="Page-2" } ### Router between Mgmt and Tenant-A -[VyOS Router](https://vyos.io/) router & firewall. Do not allow Traffic between Mgmt and Tenant-A network except DNS and gateway. +[VyOS](https://vyos.io/) acts as router and firewall between management and Tenant-A. Restrict **lateral** traffic between the two segments (no full mesh); allow only what you need (for example DNS to resolvers, default route or NAT for internet egress). Hosted-cluster control-plane traffic from tenant nodes should flow to the **external load balancer VIPs** in the tenant segment (not directly into arbitrary management subnets). ??? example "VyOS config commands" @@ -71,12 +66,11 @@ In front of the router/ingress shared is an external load balancer (for example, --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt" ``` -### Ingress Sharding +### Ingress Sharding at Hub Cluster * [2.3.4. Ingress sharding in OpenShift Container Platform](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/ingress_and_load_balancing/configuring-ingress-cluster-traffic#nw-ingress-sharding-concept_configuring-ingress-cluster-traffic-ingress-controller) * [3.1.3.8.1. Example load balancer configuration for user-provisioned clusters](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/installing_on_vmware_vsphere/user-provisioned-infrastructure) - ???+ example "Ingress Controller" ```yaml @@ -89,12 +83,12 @@ NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) router-nodeport-tenant-a NodePort 172.30.141.209 80:32460/TCP,443:32488/TCP,1936:32095/TCP 106s ``` -Ingress sharding load balancer is an RHEL 9 system with haproxy. +The ingress shard load balancer is an RHEL 9 host running HAProxy (external load balancer `ingress-shared-lb`). -* Install HAProxy `dnf install haproxy` -* Configure selinux `setsebool -P haproxy_connect_any 1` -* Apply Example haproxy.conf (don't forget to update ports) -* Enabel and start haproxy `systemctl enable --now haproxy` +* Install HAProxy: `dnf install haproxy` +* Configure SELinux: `setsebool -P haproxy_connect_any 1` +* Apply the example `haproxy` configuration (update ports to match your NodePort service) +* Enable and start HAProxy: `systemctl enable --now haproxy` ??? example "HAProxy config" @@ -102,7 +96,7 @@ Ingress sharding load balancer is an RHEL 9 system with haproxy. --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/ingress-shared-haproxy.conf" ``` -Add DNS Records +Add DNS records ```bind konnectivity.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 @@ -110,19 +104,65 @@ oauth.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 ignition.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 ``` +### Deployment sequence (reference) +Three external load balancers appear in this write-up; keep their roles distinct: +| Name | Role | +|------|------| +| `ingress-shared-lb` | Tenant-facing VIPs for OAuth, Konnectivity, Ignition Routes on the **hub** ingress shard | +| `api-lb` | Tenant-facing VIP for the hosted cluster **API** (`APIServer` publishing) | +| `ingress-lb` | Tenant-facing VIP for **hosted cluster** application Routes (`*.apps…`) | +Suggested order: (1) hub ingress shard + `ingress-shared-lb` + DNS for the three control-plane hostnames, (2) `api-lb` + API DNS, (3) `ingress-lb` + wildcard apps DNS, then (4) apply `HostedCluster` and `NodePool`. Adjust if your automation creates services first and you backfill DNS once NodePorts or service endpoints are known. +The following two subsections describe (2) and (3); the hub shard and DNS for OAuth, Konnectivity, and Ignition are covered above. +### Deploy External Load Balancer for API (`api-lb`) +Use an RHEL 9 virtual machine with HAProxy. +* Install HAProxy: `dnf install haproxy` +* Configure SELinux: `setsebool -P haproxy_connect_any 1` +* Apply the example `haproxy` configuration (update ports to match your environment) +* Enable and start HAProxy: `systemctl enable --now haproxy` -apiVersion: project.openshift.io/v1 -kind: Project -metadata: - name: 'clusters' ---- +??? example "HAProxy config" + + ```shell + --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/api-lb.conf" + ``` + +Add DNS record: + +```bind +api.tenant-a.coe.muc.redhat.com. IN A 192.168.203. +``` + +### Deploy External Load Balancer for Ingress (`ingress-lb`) of hosted cluster + +Use an RHEL 9 virtual machine with HAProxy. + +* Install HAProxy: `dnf install haproxy` +* Configure SELinux: `setsebool -P haproxy_connect_any 1` +* Apply the example `haproxy` configuration (update ports to match your environment) +* Enable and start HAProxy: `systemctl enable --now haproxy` + +??? example "HAProxy config" + + ```shell + --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/ingress-lb.conf" + ``` + +Add DNS record: + +```bind +*.apps.tenant-a.coe.muc.redhat.com. IN A 192.168.203. +``` + +### Start hosted control plane and nodepool + +```yaml hl_lines="11 43-66" title="HostedCluster" apiVersion: hypershift.openshift.io/v1beta1 kind: HostedCluster metadata: @@ -133,7 +173,7 @@ metadata: spec: configuration: ingress: - appsDomain: apps.tenant-a.coe.muc.redhat.com + appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) domain: '' loadBalancer: platform: @@ -170,12 +210,12 @@ spec: servicePublishingStrategy: type: LoadBalancer loadBalancer: - hostname: api.tenant-a.coe.muc.redhat.com + hostname: api.tenant-a.coe.muc.redhat.com # (2) - service: OAuthServer servicePublishingStrategy: type: Route route: - hostname: oauth.tenant-a.coe.muc.redhat.com + hostname: oauth.tenant-a.coe.muc.redhat.com # (3) - service: OIDC servicePublishingStrategy: type: Route @@ -183,14 +223,21 @@ spec: servicePublishingStrategy: type: Route route: - hostname: konnectivity.tenant-a.coe.muc.redhat.com + hostname: konnectivity.tenant-a.coe.muc.redhat.com # (4) - service: Ignition servicePublishingStrategy: type: Route route: - hostname: ignition.tenant-a.coe.muc.redhat.com + hostname: ignition.tenant-a.coe.muc.redhat.com # (5) +``` ---- +1. `appsDomain`: resolve names under `apps.tenant-a.coe.muc.redhat.com` to **`ingress-lb`** (hosted cluster ingress), not the hub shard. +2. API server `loadBalancer.hostname`: resolve to **`api-lb`**, which forwards to the `APIServer` publishing target on the hub. +3. OAuth `route.hostname`: resolve to **`ingress-shared-lb`** (hub dedicated shard). +4. Konnectivity `route.hostname`: resolve to **`ingress-shared-lb`**. +5. Ignition `route.hostname`: resolve to **`ingress-shared-lb`**. + +```yaml hl_lines="24-26" title="NodePool" apiVersion: hypershift.openshift.io/v1beta1 kind: NodePool metadata: @@ -214,8 +261,17 @@ spec: persistent: size: 32Gi additionalNetworks: - - name: default/cudn-localnet1-2003 + - name: default/cudn-localnet1-2003 # (1) attachDefaultNetwork: false release: image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi +``` + +1. Attach NodePool VMs to the tenant segment using a user-defined network (UDN) `localnet` attachment (`default/cudn-localnet1-2003` in this lab). + +## Open topics +* Disable or constrain cloud provider integration so that Kubernetes `LoadBalancer` Service requests for the hosted cluster are not satisfied by the hub cluster cloud integration unless that is intentional. +* WebUI bug: ACM shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` for the console, but the URL should be `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. +* Add custom endpoint publishing strategy +* Find a solution for the NodePort chicken-and-egg problem of the external API load balancer diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml index fcd9b0a1..e5c7b064 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml @@ -10,8 +10,8 @@ spec: type: NodePortService namespaceSelector: matchExpressions: - - key: kubernetes.io/metadata.name - operator: In - values: - - ingress-test - - clusters-tenant-a \ No newline at end of file + - key: kubernetes.io/metadata.name + operator: In + values: + - ingress-test + - clusters-tenant-a diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/ingress-lb.conf b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-lb.conf new file mode 100644 index 00000000..518833e1 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/ingress-lb.conf @@ -0,0 +1,34 @@ +global + log 127.0.0.1 local2 + pidfile /var/run/haproxy.pid + maxconn 4000 + daemon +defaults + mode http + log global + option dontlognull + option http-server-close + option redispatch + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s + maxconn 3000 + +listen ingress-router-443 + bind *:443 + mode tcp + balance source + server tenant-a-gngj5-mfwp6 192.168.203.101:30190 check inter 1s + server tenant-a-gngj5-rrbmv 192.168.203.102:30190 check inter 1s + +listen ingress-router-80 + bind *:80 + mode tcp + balance source + server tenant-a-gngj5-mfwp6 192.168.203.101:30282 check inter 1s + server tenant-a-gngj5-rrbmv 192.168.203.102:30282 check inter 1s diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio b/content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio index af78cf9b..4f97b08f 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio +++ b/content/cluster-installation/hosted-control-plane/tenant-network/overview.drawio @@ -77,7 +77,7 @@ - + @@ -93,8 +93,8 @@ - - + + @@ -144,7 +144,7 @@ - + @@ -174,11 +174,11 @@ - - + + - - + + @@ -198,8 +198,8 @@ - - + + @@ -240,8 +240,8 @@ - - + + @@ -255,8 +255,8 @@ - - + + diff --git a/mkdocs.yml b/mkdocs.yml index ed84e0bc..2ffcebde 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ theme: - navigation.indexes - navigation.tracking - content.code.copy + - content.code.annotate # Extras extra: