NE-2664: parameterize haproxy version

HACKING.md

@@ -19,7 +19,7 @@ $ make
 * An admin-scoped `KUBECONFIG` for the cluster.
 * Install [imagebuilder](https://github.com/openshift/imagebuilder)
 
-#### Building a Modified Router Image Locally & Deploying to the Cluster
+### Building a Modified Router Image Locally & Deploying to the Cluster
 
 To test Router changes on an available cluster, utilize `Dockerfile.debug` and
 `Makefile.debug` in `hack/`.
@@ -45,6 +45,25 @@ In case OpenShift is deployed as single node, `push` can be changed to `scp` by
 
 When done testing, use `make -f hack/Makefile.debug reset` to re-enable the CVO and Ingress Operator.
 
+### Building router image with two or more haproxy versions
+
+WIP, missing some makefile work.
+
+```bash
+make build && \
+  podman build -t localhost/router -f hack/Dockerfile.multi . && \
+  podman save localhost/router | gzip | ssh core@<node> sudo podman load
+```
+
+Defaults to create router with 2.8.18 and 3.2.11, override default using `haproxy_versions` arg, e.g. `--build-arg haproxy_versions=2.8.10,2.8.18,...`
+
+Optionally add the following envvar on router deployment, defaults to use the first one from the `haproxy_versions` arg.
+
+```yaml
+        - name: ROUTER_HAPROXY_VERSION
+          value: 2.8.18 # or any other pre-installed version
+```
+
 ## Tests
 
 Run unit tests:

hack/Dockerfile.multi

@@ -0,0 +1,40 @@
+ARG haproxy_versions=2.8.18,3.2.11 ## choose one of them to populate ROUTER_HAPROXY_VERSION
+
+FROM registry.access.redhat.com/ubi9/ubi as builder
+ARG haproxy_versions
+RUN yum install -y make pcre2-devel gcc openssl-devel util-linux && yum clean all
+RUN mkdir -p /tmp/haproxy && \
+    cd /tmp/haproxy && \
+    for v in ${haproxy_versions//,/ }; do \
+        curl -LO https://www.haproxy.org/download/${v%.*}/src/haproxy-${v}.tar.gz; \
+        tar xzf haproxy-${v}.tar.gz; \
+    done
+RUN cd /tmp/haproxy && \
+    for v in ${haproxy_versions//,/ }; do \
+        cd haproxy-${v}; \
+        make TARGET=linux-glibc USE_OPENSSL=1 USE_PROMEX=1 USE_PCRE2=1 USE_PCRE2_JIT=1; \
+        ./haproxy -v; \
+        cp haproxy /usr/sbin/haproxy-${v}; \
+        setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy-${v}; \
+        cd ..; \
+    done
+
+FROM registry.access.redhat.com/ubi9/ubi
+ARG haproxy_versions
+COPY --from=builder /usr/sbin/haproxy-* /usr/sbin/
+RUN yum install -y rsyslog procps-ng socat && yum clean all
+RUN mkdir -p /var/lib/haproxy/router/{certs,cacerts,allowlists} && \
+    mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log} && \
+    touch /var/lib/haproxy/conf/{{os_http_be,os_edge_reencrypt_be,os_tcp_be,os_sni_passthrough,os_route_http_redirect,cert_config,os_wildcard_domain}.map,haproxy.config} && \
+    chown -R :0 /var/lib/haproxy && \
+    chmod -R g+w /var/lib/haproxy
+COPY images/router/haproxy/ /var/lib/haproxy/
+COPY openshift-router /usr/bin/openshift-router
+USER 1001
+EXPOSE 80 443 1936
+WORKDIR /var/lib/haproxy/conf
+ENV XDG_CONFIG_HOME=/tmp \
+    TEMPLATE_FILE=/var/lib/haproxy/conf/haproxy-config.template \
+    RELOAD_SCRIPT=/var/lib/haproxy/reload-haproxy \
+    ROUTER_HAPROXY_VERSION=${haproxy_versions%%,*}
+ENTRYPOINT ["/usr/bin/openshift-router", "--v=2"]

images/router/haproxy/reload-haproxy

@@ -73,12 +73,17 @@ if [ -n "${ROUTER_SHUTDOWN-}" ]; then
   exit 1
 fi
 
+haproxy_bin=/usr/sbin/haproxy
+if [ -n "${ROUTER_HAPROXY_VERSION-}" ]; then
+  haproxy_bin+="-$ROUTER_HAPROXY_VERSION"
+fi
+
 reload_status=0
 if [ -n "$old_pids" ]; then
-  /usr/sbin/haproxy -f $config_file -p $pid_file -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids
+  $haproxy_bin -f $config_file -p $pid_file -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids
   reload_status=$?
 else
-  /usr/sbin/haproxy -f $config_file -p $pid_file
+  $haproxy_bin -f $config_file -p $pid_file
   reload_status=$?
 fi
 

pkg/router/template/router.go

@@ -2,6 +2,7 @@ package templaterouter
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	"encoding/pem"
 	"fmt"
@@ -16,10 +17,12 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/bcicen/go-haproxy"
 	"github.com/fsnotify/fsnotify"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	routev1 "github.com/openshift/api/route/v1"
 
@@ -649,8 +652,23 @@ func (r *templateRouter) writeCertificates(cfg *ServiceAliasConfig) error {
 	return nil
 }
 
-// reloadRouter executes the router's reload script.
+// reloadRouter reloads HAProxy.
 func (r *templateRouter) reloadRouter(shutdown bool) error {
+	masterSocket := os.Getenv("ROUTER_HAPROXY_MASTER_UNIX_SOCKET")
+	if masterSocket != "" {
+		// We are in master/worker mode, and this can be either embedded or remote.
+		// Currently we only implement master/worker as a sidecar, so there is no local process to handle.
+		if shutdown {
+			// no action for now, master/worker is always a sidecar and haproxy already received SIGUSR1.
+			return nil
+		}
+		return r.reloadRouterExternal(masterSocket)
+	}
+	return r.reloadRouterEmbedded(shutdown)
+}
+
+// reloadRouterEmbedded executes the router's reload script.
+func (r *templateRouter) reloadRouterEmbedded(shutdown bool) error {
 	if r.reloadFn != nil {
 		return r.reloadFn(shutdown)
 	}
@@ -662,7 +680,34 @@ func (r *templateRouter) reloadRouter(shutdown bool) error {
 	if err != nil {
 		return fmt.Errorf("error reloading router: %v\n%s", err, string(out))
 	}
-	log.V(0).Info("router reloaded", "output", string(out))
+	log.V(0).Info("router reloaded", "mode", "embedded", "output", string(out))
+	return nil
+}
+
+// reloadRouterExternal sends a reload command to the external HAProxy.
+func (r *templateRouter) reloadRouterExternal(masterSocket string) error {
+	// TODO missing application's context
+	_ = wait.PollUntilContextCancel(context.Background(), 2*time.Second, true, func(ctx context.Context) (done bool, err error) {
+		_, errstat := os.Lstat(masterSocket)
+		if errstat != nil {
+			log.Info("waiting for haproxy socket", "message", errstat.Error())
+			return false, nil
+		}
+		return true, nil
+	})
+	client := haproxy.HAProxyClient{Addr: "unix://" + masterSocket, Timeout: 10 /*seconds*/}
+	outputBuffer, err := client.RunCommand("reload")
+	if err != nil {
+		return fmt.Errorf("error connecting haproxy: %w", err)
+	}
+	output := outputBuffer.String()
+
+	// `reload` command is synchronous since haproxy 2.7, so it is safe to continue as soon as it returns.
+	// It should return Success=1 in the first line in case everything went well, anything else is considered a failure.
+	if !strings.HasPrefix(output, "Success=1") {
+		return fmt.Errorf("error reloading router: %s", output)
+	}
+	log.Info("router reloaded", "mode", "sidecar")
 	return nil
 }