From 41a3a8c85e4f753c876ca2f0e636b3e70ded05cc Mon Sep 17 00:00:00 2001 From: ktyagiapphelix2u Date: Wed, 4 Feb 2026 08:39:19 +0000 Subject: [PATCH 1/2] fix: adding new script to run the jenkin job --- util/jenkins/virtualenv_python_tools.sh | 73 +++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 util/jenkins/virtualenv_python_tools.sh diff --git a/util/jenkins/virtualenv_python_tools.sh b/util/jenkins/virtualenv_python_tools.sh new file mode 100644 index 000000000..edb5f507c --- /dev/null +++ b/util/jenkins/virtualenv_python_tools.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# function to create a virtual environment using python's built-in venv module +# instead of the virtualenv command. This is useful for Python 3.12+ where +# distutils has been removed and older versions of pip/setuptools may fail. +# +# Usage: +# . /edx/var/jenkins/jobvenvs/virtualenv_python_tools.sh +# create_virtualenv_python --python=python3.12 --clear +# . "$venvpath/bin/activate" +# +# Optional Environmental Variables: +# +# JOBVENVDIR - where on the system to create the virtualenv +# - e.g. /edx/var/jenkins/jobvenvs +# +# Reason for existence: The original virtualenv_tools.sh uses the 'virtualenv' +# command which doesn't work well with Python 3.12+ due to distutils removal. +# This script uses python's built-in venv module directly, allowing us to +# specify exact python binary paths and avoid compatibility issues. +# +# Why not create virtual environments right in the jenkins workspace +# where the job is run? Because workspaces are so deep in the filesystem +# that the autogenerated shebang line created by virtualenv on things in +# the virtualenv's bin directory will often be too long for the OS to +# parse. + +function create_virtualenv_python () { +if [ -z "${JOBVENVDIR:-}" ] +then + echo "No JOBVENVDIR found. Using default value." >&2 + JOBVENVDIR="/edx/var/jenkins/jobvenvs" +fi + +# Parse arguments +local python_bin="" +local clear_flag="" +while [[ $# -gt 0 ]]; do + case $1 in + --python=*) + python_bin="${1#*=}" + shift + ;; + --clear) + clear_flag="--clear" + shift + ;; + *) + echo "Unknown argument: $1" >&2 + return 1 + ;; + esac +done + +# Default to python3 if no python binary specified +if [ -z "$python_bin" ]; then + python_bin="python3" +fi + +# create a unique hash for the job based location of where job is run +# and the python binary used +venvname="$( (echo "$python_bin"; pwd) | md5sum | cut -d' ' -f1 )" +venvpath="$JOBVENVDIR/$venvname" + +# create the virtualenv using python's venv module +"$python_bin" -m venv $clear_flag "$venvpath" + +# Upgrade pip to avoid distutils issues with Python 3.12+ +"$venvpath/bin/python" -m pip install --upgrade pip setuptools wheel + +# This variable is created in global scope if function is sourced +# so we can access it after running this function. +} From 04a0b7c0c6ca945296953f4881f0484a827ae2d3 Mon Sep 17 00:00:00 2001 From: ktyagiapphelix2u Date: Thu, 5 Feb 2026 05:02:35 +0000 Subject: [PATCH 2/2] fix: adding new script to run the jenkin job --- util/jenkins/virtualenv_python_tools.sh | 74 ++++++++++--------------- 1 file changed, 29 insertions(+), 45 deletions(-) diff --git a/util/jenkins/virtualenv_python_tools.sh b/util/jenkins/virtualenv_python_tools.sh index edb5f507c..f7dea6300 100644 --- a/util/jenkins/virtualenv_python_tools.sh +++ b/util/jenkins/virtualenv_python_tools.sh @@ -6,9 +6,13 @@ # # Usage: # . /edx/var/jenkins/jobvenvs/virtualenv_python_tools.sh -# create_virtualenv_python --python=python3.12 --clear +# create_virtualenv_python python3.12 --clear # . "$venvpath/bin/activate" # +# Arguments: +# python_bin - Path to or name of the python executable (mandatory) +# additional args - Any additional arguments to pass to venv module +# # Optional Environmental Variables: # # JOBVENVDIR - where on the system to create the virtualenv @@ -26,48 +30,28 @@ # parse. function create_virtualenv_python () { -if [ -z "${JOBVENVDIR:-}" ] -then - echo "No JOBVENVDIR found. Using default value." >&2 - JOBVENVDIR="/edx/var/jenkins/jobvenvs" -fi - -# Parse arguments -local python_bin="" -local clear_flag="" -while [[ $# -gt 0 ]]; do - case $1 in - --python=*) - python_bin="${1#*=}" - shift - ;; - --clear) - clear_flag="--clear" - shift - ;; - *) - echo "Unknown argument: $1" >&2 - return 1 - ;; - esac -done - -# Default to python3 if no python binary specified -if [ -z "$python_bin" ]; then - python_bin="python3" -fi - -# create a unique hash for the job based location of where job is run -# and the python binary used -venvname="$( (echo "$python_bin"; pwd) | md5sum | cut -d' ' -f1 )" -venvpath="$JOBVENVDIR/$venvname" - -# create the virtualenv using python's venv module -"$python_bin" -m venv $clear_flag "$venvpath" - -# Upgrade pip to avoid distutils issues with Python 3.12+ -"$venvpath/bin/python" -m pip install --upgrade pip setuptools wheel - -# This variable is created in global scope if function is sourced -# so we can access it after running this function. + if [ -z "${JOBVENVDIR:-}" ] + then + echo "No JOBVENVDIR found. Using default value." >&2 + JOBVENVDIR="/edx/var/jenkins/jobvenvs" + fi + + # First argument is the python executable + local python_bin="$1" + shift + + # create a unique hash for the job based location of where job is run + # and the python binary used + venvname="$( (echo "$python_bin"; pwd) | md5sum | cut -d' ' -f1 )" + venvpath="$JOBVENVDIR/$venvname" + + # create the virtualenv using python's venv module + # Pass any additional arguments directly to venv + "$python_bin" -m venv "$venvpath" "$@" + + # Upgrade pip to avoid distutils issues with Python 3.12+ + "$venvpath/bin/python" -m pip install --upgrade pip setuptools wheel + + # This variable is created in global scope if function is sourced + # so we can access it after running this function. }