python_hosted_runner_test.yml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Build and test PySpark on macOS

on:
  workflow_call:
    inputs:
      java:
        required: false
        type: string
        default: 17
      python:
        required: false
        type: string
        default: 3.11
      branch:
        description: Branch to run the build against
        required: false
        type: string
        default: master
      hadoop:
        description: Hadoop version to run with. HADOOP_PROFILE environment variable should accept it.
        required: false
        type: string
        default: hadoop3
      os:
        description: OS to run this build.
        required: false
        type: string
        default: macos-15
      arch:
        description: The target architecture (x86, x64, arm64) of the Python or PyPy interpreter.
        required: false
        type: string
        default: arm64
      envs:
        description: Additional environment variables to set when running the tests. Should be in JSON format.
        required: false
        type: string
        default: '{}'
jobs:
  build:
    name: "Build modules: ${{ matrix.modules }}"
    runs-on: ${{ inputs.os }}
    # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
    # timeout-minutes: 150
    strategy:
      fail-fast: false
      matrix:
        java:
          - ${{ inputs.java }}
        python:
          - ${{inputs.python}}
        modules:
          - >-
            pyspark-sql, pyspark-resource, pyspark-testing
          - >-
            pyspark-core, pyspark-errors, pyspark-streaming
          - >-
            pyspark-mllib, pyspark-ml, pyspark-ml-connect
          - >-
            pyspark-structured-streaming, pyspark-structured-streaming-connect
          - >-
            pyspark-connect
          - >-
            pyspark-pandas
          - >-
            pyspark-pandas-slow
          - >-
            pyspark-pandas-connect
          - >-
            pyspark-pandas-slow-connect
    env:
      MODULES_TO_TEST: ${{ matrix.modules }}
      PYTHON_TO_TEST: python${{inputs.python}}
      HADOOP_PROFILE: ${{ inputs.hadoop }}
      HIVE_PROFILE: hive2.3
      # GitHub Actions' default miniconda to use in pip packaging test.
      CONDA_PREFIX: /usr/share/miniconda
      GITHUB_PREV_SHA: ${{ github.event.before }}
      SPARK_LOCAL_IP: localhost
      SKIP_UNIDOC: true
      SKIP_MIMA: true
      SKIP_PACKAGING: true
      METASPACE_SIZE: 1g
      BRANCH: ${{ inputs.branch }}
    steps:
      - name: Checkout Spark repository
        uses: actions/checkout@v4
        # In order to fetch changed files
        with:
          fetch-depth: 0
          repository: apache/spark
          ref: ${{ inputs.branch }}
      - name: Sync the current branch with the latest in Apache Spark
        if: github.repository != 'apache/spark'
        run: |
          echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
          git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
      # Cache local repositories. Note that GitHub Actions cache has a 10G limit.
      - name: Cache SBT and Maven
        # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
        if: ${{ runner.os != 'macOS' }}
        uses: actions/cache@v4
        with:
          path: |
            build/apache-maven-*
            build/*.jar
            ~/.sbt
          key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
          restore-keys: |
            build-
      - name: Cache Coursier local repository
        # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341
        if: ${{ runner.os != 'macOS' }}
        uses: actions/cache@v4
        with:
          path: ~/.cache/coursier
          key: pyspark-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
          restore-keys: |
            pyspark-coursier-
      - name: Install Java ${{ matrix.java }}
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: ${{ matrix.java }}
      - name: Install Python ${{matrix.python}}
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.python}}
          architecture: ${{ inputs.arch }}
      - name: Install Python packages (Python ${{matrix.python}})
        run: |
          python${{matrix.python}} -m pip install --ignore-installed 'blinker>=1.6.2'
          python${{matrix.python}} -m pip install --ignore-installed 'six==1.16.0'
          python${{matrix.python}} -m pip install numpy 'pyarrow>=22.0.0' 'six==1.16.0' 'pandas==2.3.3' scipy 'plotly<6.0.0' 'mlflow>=2.8.1' coverage matplotlib openpyxl 'memory-profiler>=0.61.0' 'scikit-learn>=1.3.2' unittest-xml-reporting && \
          python${{matrix.python}} -m pip install 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.0' 'googleapis-common-protos==1.71.0' 'zstandard==0.25.0' 'graphviz==0.20.3' && \
          python${{matrix.python}} -m pip cache purge
      - name: List Python packages
        run: python${{matrix.python}} -m pip list
      # Run the tests.
      - name: Run tests
        env: ${{ fromJSON(inputs.envs) }}
        run: |
          if [[ "$MODULES_TO_TEST" == *"pyspark-errors"* ]]; then
            export SKIP_PACKAGING=false
            echo "Python Packaging Tests Enabled!"
          fi
          ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --python-executables "$PYTHON_TO_TEST"
      - name: Upload test results to report
        env: ${{ fromJSON(inputs.envs) }}
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results-${{ inputs.os }}-${{ matrix.modules }}--${{ matrix.java }}-${{ inputs.hadoop }}-hive2.3-${{ env.PYTHON_TO_TEST }}
          path: |
            **/target/test-reports/*.xml
            **/target/surefire-reports/*.xml
      - name: Upload unit tests log files
        env: ${{ fromJSON(inputs.envs) }}
        if: ${{ !success() }}
        uses: actions/upload-artifact@v4
        with:
          name: unit-tests-log-${{ inputs.os }}-${{ matrix.modules }}--${{ matrix.java }}-${{ inputs.hadoop }}-hive2.3-${{ env.PYTHON_TO_TEST }}
          path: "**/target/unit-tests.log"