Skip to content

OPNET-748: Support running with external loadbalancer #1825

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cybertron wants to merge 1 commit into
base: master
Choose a base branch
from
Open

OPNET-748: Support running with external loadbalancer #1825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions 06_create_cluster.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ if [[ ! -z "$INSTALLER_PROXY" ]]; then
fi
fi

if [ -n "$EXTERNAL_LOADBALANCER" ]; then
./external_loadbalancer.sh &
fi
Comment on lines +32 to +34
Copy link
Member

@mkowalski mkowalski Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One detail I would like to see is a failure of the script if you set EXTERNAL_LOADBALANCER but do not set ENABLE_BOOTSTRAP_STATIC_IP=true. Otherwise we will proceed with openshift-install which will fail and you will end up in a debugging nightmare


# Call openshift-installer to deploy the bootstrap node and masters
create_cluster ${OCP_DIR}

Expand Down
10 changes: 10 additions & 0 deletions config_example.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,16 @@ set -x
# When set to any value this will cause dev-scripts to include duplicate nics
# on the primary network. This is intended for testing bonded network configs
# and may not work without a bond config.
# export BOND_PRIMARY_INTERFACE=1

# EXTERNAL_LOADBALANCER -
# When set to any value this will cause dev-scripts to configure an haproxy
# loadbalancer on the host and configure the cluster to use it instead of the
# internal loadbalancer.
# Because of the way the loadbalancer config is written, this only works when
# using single stack (either ipv4 or ipv6) and a static bootstrap IP (see the
# ENABLE_BOOTSTRAP_STATIC_IP option above).
# export EXTERNAL_LOADBALANCER=1

################################################################################
## VM Settings
Expand Down
43 changes: 43 additions & 0 deletions external_loadbalancer.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ source utils.sh

sudo firewall-cmd --zone=libvirt --add-port=6443/tcp
sudo firewall-cmd --zone=libvirt --add-port=8080/tcp
sudo firewall-cmd --zone=libvirt --add-port=22623/tcp

haproxy_config="${WORKING_DIR}/haproxy.cfg"
echo $haproxy_config
Expand All @@ -21,13 +22,15 @@ then
master2=$(nth_ip $EXTERNAL_SUBNET_V6 22)
worker0=$(nth_ip $EXTERNAL_SUBNET_V6 23)
worker1=$(nth_ip $EXTERNAL_SUBNET_V6 24)
bootstrap=$(nth_ip $EXTERNAL_SUBNET_V6 9)
else

master0=$(nth_ip $EXTERNAL_SUBNET_V4 20)
master1=$(nth_ip $EXTERNAL_SUBNET_V4 21)
master2=$(nth_ip $EXTERNAL_SUBNET_V4 22)
worker0=$(nth_ip $EXTERNAL_SUBNET_V4 23)
worker1=$(nth_ip $EXTERNAL_SUBNET_V4 24)
bootstrap=$(nth_ip $EXTERNAL_SUBNET_V4 9)
fi

cat << EOF > "$haproxy_config"
Expand All @@ -43,13 +46,23 @@ frontend main
frontend ingress
bind :::8080 v4v6
default_backend ingress
frontend https
bind :::443 v4v6
default_backend https
frontend mcs
bind :::22623 v4v6
default_backend mcs
frontend ironic
bind :::6385 v4v6
default_backend ironic
backend api
option httpchk GET /readyz HTTP/1.0
option log-health-checks
balance roundrobin
server master-0 ${master0}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
server master-1 ${master1}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
server master-2 ${master2}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
server bootstrap ${bootstrap}:6443 check check-ssl inter 1s fall 2 rise 3 verify none
backend ingress
option httpchk GET /healthz/ready HTTP/1.0
option log-health-checks
Expand All @@ -59,6 +72,36 @@ backend ingress
server master-2 ${master2}:80 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
server w-0 ${worker0}:80 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
server w-1 ${worker1}:80 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
backend https
option httpchk GET /healthz/ready HTTP/1.0
option log-health-checks
balance roundrobin
server master-0 ${master0}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
server master-1 ${master1}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
server master-2 ${master2}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
server w-0 ${worker0}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
server w-1 ${worker1}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
server bootstrap ${bootstrap}:443 check check-ssl port 1936 inter 1s fall 2 rise 3 verify none
backend mcs
option httpchk GET /config/master HTTP/1.0
option log-health-checks
balance roundrobin
server master-0 ${master0}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
server master-1 ${master1}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
server master-2 ${master2}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
server w-0 ${worker0}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
server w-1 ${worker1}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
server bootstrap ${bootstrap}:22623 check check-ssl inter 1s fall 2 rise 3 verify none
backend ironic
option httpchk GET /v1 HTTP/1.0
option log-health-checks
balance roundrobin
server master-0 ${master0}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
server master-1 ${master1}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
server master-2 ${master2}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
server w-0 ${worker0}:6385 check check-ssl inter 1s fall 2 rise 3 verify none
server w-1 ${worker1}:6385 check check-ssl inter 1s fall 2 rise 3 verify none
server bootstrap ${bootstrap}:6385 check check-ssl inter 30s fall 2 rise 3 verify none
EOF

sudo podman run -d --net host -v "${WORKING_DIR}":/etc/haproxy/:z --entrypoint bash --name extlb quay.io/openshift/origin-haproxy-router -c 'haproxy -f /etc/haproxy/haproxy.cfg'
Expand Down
12 changes: 10 additions & 2 deletions network.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,12 +253,20 @@ function get_vips() {
#
if [[ -n "${EXTERNAL_SUBNET_V4}" ]]; then
API_VIPS_V4=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip ${BAREMETAL_NETWORK_NAME}) | awk '{print $NF}')
INGRESS_VIPS_V4=$(nth_ip $EXTERNAL_SUBNET_V4 4)
if [ -z "$EXTERNAL_LOADBALANCER" ]; then
INGRESS_VIPS_V4=$(nth_ip $EXTERNAL_SUBNET_V4 4)
else
INGRESS_VIPS_V4=$(nth_ip $EXTERNAL_SUBNET_V4 1)
fi
fi

if [[ -n "${EXTERNAL_SUBNET_V6}" ]]; then
API_VIPS_V6=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip ${BAREMETAL_NETWORK_NAME}) | awk '{print $NF}')
INGRESS_VIPS_V6=$(nth_ip $EXTERNAL_SUBNET_V6 4)
if [ -z "$EXTERNAL_LOADBALANCER" ]; then
INGRESS_VIPS_V6=$(nth_ip $EXTERNAL_SUBNET_V6 4)
else
INGRESS_VIPS_V6=$(nth_ip $EXTERNAL_SUBNET_V6 1)
fi
fi

if [[ "$IP_STACK" == "v4" || "$IP_STACK" == "v4v6" ]]; then
Expand Down
10 changes: 10 additions & 0 deletions ocp_install_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,15 @@ function setVIPs() {
esac
}

function loadbalancer_type() {
if [ -n "$EXTERNAL_LOADBALANCER" ]; then
cat <<EOF
loadBalancer:
type: UserManaged
EOF
fi
}

function featureSet() {
if [[ -n "$FEATURE_SET" ]]; then
cat <<EOF
Expand Down Expand Up @@ -395,6 +404,7 @@ $(cluster_os_image)
$(setVIPs apivips)
$(setVIPs ingressvips)
$(dnsvip)
$(loadbalancer_type)
hosts:
EOF

Expand Down
9 changes: 8 additions & 1 deletion vm_setup_vars.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ dns_extrahosts:
hostnames:
- "virthost"

dns_externalhosts:
- ip: "{{ baremetal_network_cidr | nthhost(1) }}"
hostnames:
- "virthost"
- "api"
- "api-int"

network_config_folder: "{{ lookup('env', 'NETWORK_CONFIG_FOLDER') | default(false) }}"
hosts_config: "{{ lookup('template', network_config_folder + '/hosts.yaml', errors='ignore') | default('[]', true) | from_yaml }}"
dns_customhosts: "{{ [] if not network_config_folder else hosts_config }}"
Expand Down Expand Up @@ -88,7 +95,7 @@ external_network:
- 65535
domain: "{{ cluster_domain }}"
dns:
hosts: "{{ dns_extrahosts + dns_customhosts + dns_dualstackhost if lookup('env', 'EXTERNAL_SUBNET_V6') else dns_extrahosts + dns_customhosts }}"
hosts: "{{ dns_externalhosts + dns_customhosts if lookup('env', 'EXTERNAL_LOADBALANCER') else dns_extrahosts + dns_customhosts + dns_dualstackhost if lookup('env', 'EXTERNAL_SUBNET_V6') else dns_extrahosts + dns_customhosts }}"
forwarders:
- domain: "apps.{{ cluster_domain }}"
addr: "127.0.0.1"
Expand Down