From 532a4520a1f8173234deb938458790b6103f1137 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Sat, 7 Feb 2026 17:02:14 +0800 Subject: [PATCH 1/3] nit --- .github/workflows/build_and_test.yml | 4 ++-- dev/spark-test-image/pypy-310/Dockerfile | 15 +++++++++++---- dev/spark-test-image/pypy-311/Dockerfile | 15 +++++++++++---- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 97cb5e922782a..b31794ca4ea09 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -41,14 +41,14 @@ on: description: Additional environment variables to set when running the tests. Should be in JSON format. required: false type: string - default: '{"PYSPARK_IMAGE_TO_TEST": "python-312", "PYTHON_TO_TEST": "python3.12"}' + default: '{"PYSPARK_IMAGE_TO_TEST": "pypy-310", "PYTHON_TO_TEST": "pypy3"}' jobs: description: >- Jobs to run, and should be in JSON format. The values should be matched with the job's key defined in this file, e.g., build. See precondition job below. required: false type: string - default: '' + default: '{"pyspark": "true", "pyspark-pandas": "true"}' secrets: codecov_token: description: The upload token of codecov. diff --git a/dev/spark-test-image/pypy-310/Dockerfile b/dev/spark-test-image/pypy-310/Dockerfile index 9ddec543517b1..2dbb82477a6df 100644 --- a/dev/spark-test-image/pypy-310/Dockerfile +++ b/dev/spark-test-image/pypy-310/Dockerfile @@ -15,16 +15,16 @@ # limitations under the License. # -# Image for building and testing Spark branches. Based on Ubuntu 22.04. +# Image for building and testing Spark branches. Based on Ubuntu 24.04. # See also in https://hub.docker.com/_/ubuntu -FROM ubuntu:jammy-20240911.1 +FROM ubuntu:noble LABEL org.opencontainers.image.authors="Apache Spark project " LABEL org.opencontainers.image.licenses="Apache-2.0" LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark with PyPy 3.10" # Overwrite this label to avoid exposing the underlying Ubuntu OS version label LABEL org.opencontainers.image.version="" -ENV FULL_REFRESH_DATE=20260202 +ENV FULL_REFRESH_DATE=20260207 ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN=true @@ -51,10 +51,17 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* -RUN add-apt-repository ppa:pypy/ppa +# Install PyPy 3.10 RUN mkdir -p /usr/local/pypy/pypy3.10 && \ curl -sqL https://downloads.python.org/pypy/pypy3.10-v7.3.17-linux64.tar.bz2 | tar xjf - -C /usr/local/pypy/pypy3.10 --strip-components=1 && \ ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3.10 && \ ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3 + +# Setup virtual environment +ENV VIRTUAL_ENV=/opt/spark-venv +RUN pypy3 -m venv --without-pip $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Install PyPy 3.10 packages RUN curl -sS https://bootstrap.pypa.io/get-pip.py | pypy3 RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.3.3' scipy coverage matplotlib lxml diff --git a/dev/spark-test-image/pypy-311/Dockerfile b/dev/spark-test-image/pypy-311/Dockerfile index 73d3159b5e5b8..3fc394f539f07 100644 --- a/dev/spark-test-image/pypy-311/Dockerfile +++ b/dev/spark-test-image/pypy-311/Dockerfile @@ -15,16 +15,16 @@ # limitations under the License. # -# Image for building and testing Spark branches. Based on Ubuntu 22.04. +# Image for building and testing Spark branches. Based on Ubuntu 24.04. # See also in https://hub.docker.com/_/ubuntu -FROM ubuntu:jammy-20240911.1 +FROM ubuntu:noble LABEL org.opencontainers.image.authors="Apache Spark project " LABEL org.opencontainers.image.licenses="Apache-2.0" LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark with PyPy 3.11" # Overwrite this label to avoid exposing the underlying Ubuntu OS version label LABEL org.opencontainers.image.version="" -ENV FULL_REFRESH_DATE=20260202 +ENV FULL_REFRESH_DATE=20260207 ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN=true @@ -51,10 +51,17 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* -RUN add-apt-repository ppa:pypy/ppa +# Install PyPy 3.10 RUN mkdir -p /usr/local/pypy/pypy3.11 && \ curl -sqL https://downloads.python.org/pypy/pypy3.11-v7.3.20-linux64.tar.bz2 | tar xjf - -C /usr/local/pypy/pypy3.11 --strip-components=1 && \ ln -sf /usr/local/pypy/pypy3.11/bin/pypy /usr/local/bin/pypy3.11 && \ ln -sf /usr/local/pypy/pypy3.11/bin/pypy /usr/local/bin/pypy3 + +# Setup virtual environment +ENV VIRTUAL_ENV=/opt/spark-venv +RUN pypy3 -m venv --without-pip $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Install PyPy 3.11 packages RUN curl -sS https://bootstrap.pypa.io/get-pip.py | pypy3 RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.3.3' scipy coverage matplotlib lxml From aaa572ca2e80bc59a6b2004fbd9598d69db3c85f Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Sat, 7 Feb 2026 20:38:14 +0800 Subject: [PATCH 2/3] test pypy-311 --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b31794ca4ea09..77ea8535195ac 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -41,7 +41,7 @@ on: description: Additional environment variables to set when running the tests. Should be in JSON format. required: false type: string - default: '{"PYSPARK_IMAGE_TO_TEST": "pypy-310", "PYTHON_TO_TEST": "pypy3"}' + default: '{"PYSPARK_IMAGE_TO_TEST": "pypy-311", "PYTHON_TO_TEST": "pypy3"}' jobs: description: >- Jobs to run, and should be in JSON format. The values should be matched with the job's key defined From 87dd20101be92d46d9501266c140ccf5d43bcd0b Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Sat, 7 Feb 2026 20:38:54 +0800 Subject: [PATCH 3/3] nit --- dev/spark-test-image/pypy-311/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/spark-test-image/pypy-311/Dockerfile b/dev/spark-test-image/pypy-311/Dockerfile index 3fc394f539f07..7760fd8a629bd 100644 --- a/dev/spark-test-image/pypy-311/Dockerfile +++ b/dev/spark-test-image/pypy-311/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* -# Install PyPy 3.10 +# Install PyPy 3.11 RUN mkdir -p /usr/local/pypy/pypy3.11 && \ curl -sqL https://downloads.python.org/pypy/pypy3.11-v7.3.20-linux64.tar.bz2 | tar xjf - -C /usr/local/pypy/pypy3.11 --strip-components=1 && \ ln -sf /usr/local/pypy/pypy3.11/bin/pypy /usr/local/bin/pypy3.11 && \