From e93f7ed7f9ce2f3384ad386075a2fca10179b4a8 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Fri, 13 Feb 2026 14:23:28 -0300 Subject: [PATCH] parameterize haproxy version add two new envvars used to parameterize how to start or reload haproxy. * ROUTER_HAPROXY_VERSION: defines the haproxy version to use. I must be one of the installed versions. If missing, it defaults to not use any suffix, preserving the same behavior before the change. * ROUTER_HAPROXY_MASTER_UNIX_SOCKET: defines the full qualified path of master's unix socket. If provided, a reload command is issued to master instead of using the reload script, so configuring this option makes ROUTER_HAPROXY_VERSION to be ignored. If missing, the previous behavior is used, which is to call the reload script. --- HACKING.md | 21 +++++++++++- hack/Dockerfile.multi | 40 +++++++++++++++++++++++ images/router/haproxy/reload-haproxy | 9 +++-- pkg/router/template/router.go | 49 ++++++++++++++++++++++++++-- 4 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 hack/Dockerfile.multi diff --git a/HACKING.md b/HACKING.md index cfefdf367..aab29c110 100644 --- a/HACKING.md +++ b/HACKING.md @@ -19,7 +19,7 @@ $ make * An admin-scoped `KUBECONFIG` for the cluster. * Install [imagebuilder](https://github.com/openshift/imagebuilder) -#### Building a Modified Router Image Locally & Deploying to the Cluster +### Building a Modified Router Image Locally & Deploying to the Cluster To test Router changes on an available cluster, utilize `Dockerfile.debug` and `Makefile.debug` in `hack/`. @@ -45,6 +45,25 @@ In case OpenShift is deployed as single node, `push` can be changed to `scp` by When done testing, use `make -f hack/Makefile.debug reset` to re-enable the CVO and Ingress Operator. +### Building router image with two or more haproxy versions + +WIP, missing some makefile work. + +```bash +make build && \ + podman build -t localhost/router -f hack/Dockerfile.multi . && \ + podman save localhost/router | gzip | ssh core@ sudo podman load +``` + +Defaults to create router with 2.8.18 and 3.2.11, override default using `haproxy_versions` arg, e.g. `--build-arg haproxy_versions=2.8.10,2.8.18,...` + +Optionally add the following envvar on router deployment, defaults to use the first one from the `haproxy_versions` arg. + +```yaml + - name: ROUTER_HAPROXY_VERSION + value: 2.8.18 # or any other pre-installed version +``` + ## Tests Run unit tests: diff --git a/hack/Dockerfile.multi b/hack/Dockerfile.multi new file mode 100644 index 000000000..8138cd3dd --- /dev/null +++ b/hack/Dockerfile.multi @@ -0,0 +1,40 @@ +ARG haproxy_versions=2.8.18,3.2.11 ## choose one of them to populate ROUTER_HAPROXY_VERSION + +FROM registry.access.redhat.com/ubi9/ubi as builder +ARG haproxy_versions +RUN yum install -y make pcre2-devel gcc openssl-devel util-linux && yum clean all +RUN mkdir -p /tmp/haproxy && \ + cd /tmp/haproxy && \ + for v in ${haproxy_versions//,/ }; do \ + curl -LO https://www.haproxy.org/download/${v%.*}/src/haproxy-${v}.tar.gz; \ + tar xzf haproxy-${v}.tar.gz; \ + done +RUN cd /tmp/haproxy && \ + for v in ${haproxy_versions//,/ }; do \ + cd haproxy-${v}; \ + make TARGET=linux-glibc USE_OPENSSL=1 USE_PROMEX=1 USE_PCRE2=1 USE_PCRE2_JIT=1; \ + ./haproxy -v; \ + cp haproxy /usr/sbin/haproxy-${v}; \ + setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy-${v}; \ + cd ..; \ + done + +FROM registry.access.redhat.com/ubi9/ubi +ARG haproxy_versions +COPY --from=builder /usr/sbin/haproxy-* /usr/sbin/ +RUN yum install -y rsyslog procps-ng socat && yum clean all +RUN mkdir -p /var/lib/haproxy/router/{certs,cacerts,allowlists} && \ + mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log} && \ + touch /var/lib/haproxy/conf/{{os_http_be,os_edge_reencrypt_be,os_tcp_be,os_sni_passthrough,os_route_http_redirect,cert_config,os_wildcard_domain}.map,haproxy.config} && \ + chown -R :0 /var/lib/haproxy && \ + chmod -R g+w /var/lib/haproxy +COPY images/router/haproxy/ /var/lib/haproxy/ +COPY openshift-router /usr/bin/openshift-router +USER 1001 +EXPOSE 80 443 1936 +WORKDIR /var/lib/haproxy/conf +ENV XDG_CONFIG_HOME=/tmp \ + TEMPLATE_FILE=/var/lib/haproxy/conf/haproxy-config.template \ + RELOAD_SCRIPT=/var/lib/haproxy/reload-haproxy \ + ROUTER_HAPROXY_VERSION=${haproxy_versions%%,*} +ENTRYPOINT ["/usr/bin/openshift-router", "--v=2"] diff --git a/images/router/haproxy/reload-haproxy b/images/router/haproxy/reload-haproxy index e0733ec21..c3a2ad5a5 100755 --- a/images/router/haproxy/reload-haproxy +++ b/images/router/haproxy/reload-haproxy @@ -73,12 +73,17 @@ if [ -n "${ROUTER_SHUTDOWN-}" ]; then exit 1 fi +haproxy_bin=/usr/sbin/haproxy +if [ -n "${ROUTER_HAPROXY_VERSION-}" ]; then + haproxy_bin+="-$ROUTER_HAPROXY_VERSION" +fi + reload_status=0 if [ -n "$old_pids" ]; then - /usr/sbin/haproxy -f $config_file -p $pid_file -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids + $haproxy_bin -f $config_file -p $pid_file -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids reload_status=$? else - /usr/sbin/haproxy -f $config_file -p $pid_file + $haproxy_bin -f $config_file -p $pid_file reload_status=$? fi diff --git a/pkg/router/template/router.go b/pkg/router/template/router.go index 6db171c12..562ebd114 100644 --- a/pkg/router/template/router.go +++ b/pkg/router/template/router.go @@ -2,6 +2,7 @@ package templaterouter import ( "bytes" + "context" "crypto/md5" "encoding/pem" "fmt" @@ -16,10 +17,12 @@ import ( "text/template" "time" + "github.com/bcicen/go-haproxy" "github.com/fsnotify/fsnotify" "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" routev1 "github.com/openshift/api/route/v1" @@ -649,8 +652,23 @@ func (r *templateRouter) writeCertificates(cfg *ServiceAliasConfig) error { return nil } -// reloadRouter executes the router's reload script. +// reloadRouter reloads HAProxy. func (r *templateRouter) reloadRouter(shutdown bool) error { + masterSocket := os.Getenv("ROUTER_HAPROXY_MASTER_UNIX_SOCKET") + if masterSocket != "" { + // We are in master/worker mode, and this can be either embedded or remote. + // Currently we only implement master/worker as a sidecar, so there is no local process to handle. + if shutdown { + // no action for now, master/worker is always a sidecar and haproxy already received SIGUSR1. + return nil + } + return r.reloadRouterExternal(masterSocket) + } + return r.reloadRouterEmbedded(shutdown) +} + +// reloadRouterEmbedded executes the router's reload script. +func (r *templateRouter) reloadRouterEmbedded(shutdown bool) error { if r.reloadFn != nil { return r.reloadFn(shutdown) } @@ -662,7 +680,34 @@ func (r *templateRouter) reloadRouter(shutdown bool) error { if err != nil { return fmt.Errorf("error reloading router: %v\n%s", err, string(out)) } - log.V(0).Info("router reloaded", "output", string(out)) + log.V(0).Info("router reloaded", "mode", "embedded", "output", string(out)) + return nil +} + +// reloadRouterExternal sends a reload command to the external HAProxy. +func (r *templateRouter) reloadRouterExternal(masterSocket string) error { + // TODO missing application's context + _ = wait.PollUntilContextCancel(context.Background(), 2*time.Second, true, func(ctx context.Context) (done bool, err error) { + _, errstat := os.Lstat(masterSocket) + if errstat != nil { + log.Info("waiting for haproxy socket", "message", errstat.Error()) + return false, nil + } + return true, nil + }) + client := haproxy.HAProxyClient{Addr: "unix://" + masterSocket, Timeout: 10 /*seconds*/} + outputBuffer, err := client.RunCommand("reload") + if err != nil { + return fmt.Errorf("error connecting haproxy: %w", err) + } + output := outputBuffer.String() + + // `reload` command is synchronous since haproxy 2.7, so it is safe to continue as soon as it returns. + // It should return Success=1 in the first line in case everything went well, anything else is considered a failure. + if !strings.HasPrefix(output, "Success=1") { + return fmt.Errorf("error reloading router: %s", output) + } + log.Info("router reloaded", "mode", "sidecar") return nil }