parameterize versions of python and pip

.circle/docker-compose.sh

@@ -37,6 +37,8 @@ case "$1" in
             -e ST2PKG_RELEASE=${ST2PKG_RELEASE} \
             -e ST2_PACKAGES="${ST2_PACKAGES}" \
             -e ST2_CIRCLE_URL="${CIRCLE_BUILD_URL}" \
+            -e PYTHON_VERSION=${PYTHON_VERSION} \
+            -e PIP_VERSION=${PIP_VERSION} \
             "$2" build
     ;;
     test)

.circleci/config.yml

@@ -19,6 +19,8 @@ jobs:
     environment:
       DISTROS: "jammy jammy el9 el9"
       BASH_ENV: ~/.buildenv
+      PIP_VERSION: "25.3"
+      PYTHON_VERSION: "3.11"
       # These should be set to an empty string, so that st2cd prep tasks are able to replace these
       # with real gitrevs during releases. Note that they are commented out, so that they do not interfere
       # with build parameters. st2cd prep tasks will uncomment these on a branch, and replace with proper

packages/st2/Makefile

@@ -20,15 +20,36 @@ else
 	DEB_DISTRO := unstable
 endif
 
-PYTHON_BINARY := /usr/bin/python3
-PIP_BINARY := pip3
-PYTHON_ALT_BINARY := python3
+PYTHON_VERSION ?= 3
+PYTHON_BINARY := /usr/bin/python$(PYTHON_VERSION)
+PIP_BINARY := pip$(PYTHON_VERSION)
+PYTHON_ALT_BINARY := python$(PYTHON_VERSION)
+PIP_VERSION ?= 25.3
 
 RUNNERS := $(shell ls ../contrib/runners)
 
 # Moved from top of file to handle when only py2 or py3 available
 ST2PKG_VERSION ?= $(shell $(PYTHON_BINARY) -c "from $(ST2_COMPONENT) import __version__; print(__version__),")
 
+.PHONY: ensure-python
+ensure-python:
+ifeq ($(DEBIAN),1)
+	@if [ "$(PYTHON_VERSION)" != "3" ] && [ ! -x "$(PYTHON_BINARY)" ]; then \
+		echo "Installing Python $(PYTHON_VERSION) on Debian/Ubuntu..."; \
+		apt-get update && \
+		apt-get install -y software-properties-common && \
+		add-apt-repository -y ppa:deadsnakes/ppa && \
+		apt-get update && \
+		apt-get install -y python$(PYTHON_VERSION) python$(PYTHON_VERSION)-dev python$(PYTHON_VERSION)-distutils python$(PYTHON_VERSION)-venv; \
+	fi
+else ifeq ($(REDHAT),1)
+	@if [ "$(PYTHON_VERSION)" != "3" ] && [ ! -x "$(PYTHON_BINARY)" ]; then \
+		echo "Installing Python $(PYTHON_VERSION) on RHEL/Rocky..."; \
+		yum install -y python$(PYTHON_VERSION) python$(PYTHON_VERSION)-devel 2>/dev/null || \
+		dnf install -y python$(PYTHON_VERSION) python$(PYTHON_VERSION)-devel; \
+	fi
+endif
+
 # Note: We dynamically obtain the version, this is required because dev
 # build versions don't store correct version identifier in __init__.py
 # and we need setup.py to normalize it (e.g. 1.4dev -> 1.4.dev0)
@@ -40,7 +61,7 @@ ST2PKG_NORMALIZED_VERSION ?= $(shell $(PYTHON_BINARY) setup.py --version || echo
 retry = $(2) $(foreach t,$(shell seq 1 ${1}),|| (echo -e "\033[33m Failed ($$?): '$(2)'\n Retrying $t ... \033[0m"; $(2)))
 
 .PHONY: all install wheelhouse
-all: info install
+all: ensure-python info install
 
 .PHONY: info
 info:
@@ -113,6 +134,8 @@ endif
 wheelhouse: .build-runners .stamp-wheelhouse
 .stamp-wheelhouse: | populate_version requirements inject-deps
 	cat requirements.txt
+	# Upgrade pip to specified version
+	$(PIP_BINARY) install --upgrade pip==$(PIP_VERSION)
 	# Try to install wheels 2x in case the first one fails
 	$(PIP_BINARY) --use-deprecated=legacy-resolver  wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt || \
 		$(PIP_BINARY) --use-deprecated=legacy-resolver wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt
@@ -123,6 +146,8 @@ wheelhouse: .build-runners .stamp-wheelhouse
 	@echo "#            Building Runners            #"
 	@echo "##########################################"
 	@echo $(RUNNERS)
+	# Upgrade pip to specified version
+	$(PIP_BINARY) install --upgrade pip==$(PIP_VERSION)
 	@for runner in $(RUNNERS); do \
 		echo "Installing $$runner"; \
 		$(PIP_BINARY) wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r ../contrib/runners/$$runner/requirements.txt ../contrib/runners/$$runner; \

packages/st2/component.makefile

@@ -19,13 +19,34 @@ else
 	DEB_DISTRO := unstable
 endif
 
-PYTHON_BINARY := /usr/bin/python3
-PIP_BINARY := pip3
-PYTHON_ALT_BINARY := python3
+PYTHON_VERSION ?= 3
+PYTHON_BINARY := /usr/bin/python$(PYTHON_VERSION)
+PIP_BINARY := pip$(PYTHON_VERSION)
+PYTHON_ALT_BINARY := python$(PYTHON_VERSION)
+PIP_VERSION ?= 25.3
 
 # Moved from top of file to handle when only py2 or py3 available
 ST2PKG_VERSION ?= $(shell $(PYTHON_BINARY) -c "from $(ST2_COMPONENT) import __version__; print(__version__),")
 
+.PHONY: ensure-python
+ensure-python:
+ifeq ($(DEBIAN),1)
+	@if [ "$(PYTHON_VERSION)" != "3" ] && [ ! -x "$(PYTHON_BINARY)" ]; then \
+		echo "Installing Python $(PYTHON_VERSION) on Debian/Ubuntu..."; \
+		apt-get update && \
+		apt-get install -y software-properties-common && \
+		add-apt-repository -y ppa:deadsnakes/ppa && \
+		apt-get update && \
+		apt-get install -y python$(PYTHON_VERSION) python$(PYTHON_VERSION)-dev python$(PYTHON_VERSION)-venv; \
+	fi
+else ifeq ($(REDHAT),1)
+	@if [ "$(PYTHON_VERSION)" != "3" ] && [ ! -x "$(PYTHON_BINARY)" ]; then \
+		echo "Installing Python $(PYTHON_VERSION) on RHEL/Rocky..."; \
+		yum install -y python$(PYTHON_VERSION) python$(PYTHON_VERSION)-devel 2>/dev/null || \
+		dnf install -y python$(PYTHON_VERSION) python$(PYTHON_VERSION)-devel; \
+	fi
+endif
+
 # Note: We dynamically obtain the version, this is required because dev
 # build versions don't store correct version identifier in __init__.py
 # and we need setup.py to normalize it (e.g. 1.4dev -> 1.4.dev0)
@@ -43,7 +64,7 @@ info:
 	$(PIP_BINARY) --version
 
 .PHONY: populate_version requirements wheelhouse bdist_wheel
-all: info populate_version requirements bdist_wheel
+all: ensure-python info populate_version requirements bdist_wheel
 
 populate_version: .stamp-populate_version
 .stamp-populate_version:
@@ -60,6 +81,8 @@ wheelhouse: .stamp-wheelhouse
 .stamp-wheelhouse: | populate_version requirements
 	# Install wheels into shared location
 	cat requirements.txt
+	# Upgrade pip to specified version
+	$(PIP_BINARY) install --upgrade pip==$(PIP_VERSION)
 	# Try to install wheels 2x in case the first one fails
 	$(PIP_BINARY) --use-deprecated=legacy-resolver wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt || \
 		$(PIP_BINARY) --use-deprecated=legacy-resolver wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt
@@ -68,6 +91,8 @@ wheelhouse: .stamp-wheelhouse
 bdist_wheel: .stamp-bdist_wheel
 .stamp-bdist_wheel: | populate_version requirements inject-deps
 	cat requirements.txt
+	# Upgrade pip to specified version
+	$(PIP_BINARY) install --upgrade pip==$(PIP_VERSION)
 # We need to install these python packages to handle rpmbuild 4.14 in EL8
 ifeq ($(EL_VERSION),8)
 	$(PIP_BINARY) install wheel setuptools virtualenv

packages/st2/debian/rules

@@ -11,16 +11,18 @@ PATH = /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 WHEELDIR ?= /tmp/wheelhouse
 DH_VIRTUALENV_INSTALL_ROOT := /opt/stackstorm
 export DH_VIRTUALENV_INSTALL_ROOT
-PIP_VERSION = 25.3
+PIP_VERSION ?= 25.3
+export PIP_VERSION
 
 IS_SYSTEMD = $(shell command -v dh_systemd_enable > /dev/null 2>&1 && echo true)
 DEB_DISTRO := $(shell lsb_release -cs)
 
-BUILD_PRE_DEPENDS := python3
-BUILD_DEPENDS := python3-distutils, python3-dev
+PYTHON_VERSION ?= 3
+BUILD_PRE_DEPENDS := python$(PYTHON_VERSION)
+BUILD_DEPENDS := python$(PYTHON_VERSION)-distutils, python$(PYTHON_VERSION)-dev
 
 %:
-	dh $@ --with python-virtualenv --python /usr/bin/python3
+	dh $@ --with python-virtualenv --python /usr/bin/python$(PYTHON_VERSION)
 
 override_dh_installdirs:
 	dh_installdirs

packages/st2/rpm/st2.spec

@@ -14,26 +14,16 @@ Epoch: %{epoch}
 %global _build_id_links none
 %endif
 
+%{!?python_version: %define python_version 3}
+
 Requires: openssl-devel, libffi-devel, git, pam, openssh-server, openssh-clients, bash, setup
-%if 0%{?rhel} == 8
-Requires: python38-devel
-%endif
-%if 0%{?rhel} == 9
-Requires: python3-devel
-%endif
+Requires: python%{python_version}-devel
 
 # EL8 requires a few python packages available within 'BUILDROOT' when outside venv
 # These are in the el8 packagingbuild dockerfile
 # Reference https://fossies.org/linux/ansible/packaging/rpm/ansible.spec
-%if 0%{?rhel} == 8
-# Will use the python3 stdlib venv
-BuildRequires: python38-devel
-BuildRequires: python38-setuptools
-%endif
-%if 0%{?rhel} == 9
-BuildRequires: python3-devel
-BuildRequires: python3-setuptools
-%endif
+BuildRequires: python%{python_version}-devel
+BuildRequires: python%{python_version}-setuptools
 
 # Apply this to both RHEL 8 and RHEL 9
 %if 0%{?rhel} > 7

rake/build/environment.rb

@@ -55,6 +55,10 @@
   envpass :artifact_dir, '/root/build'
   envpass :wheeldir, '/tmp/wheelhouse'
 
+  # Python and pip versions (defaults to 3 and 25.3 if not set)
+  envpass :python_version, '3'
+  envpass :pip_version, '25.3'
+
   # Default hostnames of dependat services (the value can take an address also)
   envpass :rabbitmqhost, 'rabbitmq', proc: convert_to_ipaddr
   envpass :mongodbhost,  'mongodb',  proc: convert_to_ipaddr

rpmspec/package_venv.spec

@@ -7,13 +7,15 @@
 %define venv_dir %{buildroot}/%{venv_install_dir}
 %define venv_bin %{venv_dir}/bin
 
-%define python_binname python3
-%define pip_binname pip3
+%{!?python_version: %define python_version 3}
+%define python_binname python%{python_version}
+%define pip_binname pip%{python_version}
 
 %define venv_python %{venv_bin}/%{python_binname}
 # https://github.com/StackStorm/st2/wiki/Where-all-to-update-pip-and-or-virtualenv
 
-%define pin_pip %{venv_python} %{venv_bin}/%{pip_binname} install pip==25.3
+%{!?pip_version: %define pip_version 25.3}
+%define pin_pip %{venv_python} %{venv_bin}/%{pip_binname} install pip==%{pip_version}
 %define install_venvctrl %{python_binname} -m pip install venvctrl
 
 %define install_crypto %{nil}