diff --git a/.github/workflows/r_extra.yml b/.github/workflows/r_extra.yml new file mode 100644 index 00000000000..8c4ec668f49 --- /dev/null +++ b/.github/workflows/r_extra.yml @@ -0,0 +1,219 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: R Extra + +on: + push: + branches: + - '**' + - '!dependabot/**' + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/r_extra.yml' + - '.github/workflows/report_ci.yml' + - "ci/docker/**" + - "ci/etc/rprofile" + - "ci/scripts/PKGBUILD" + - "ci/scripts/cpp_*.sh" + - "ci/scripts/install_minio.sh" + - "ci/scripts/r_*.sh" + - "cpp/**" + - "compose.yaml" + - "r/**" + tags: + - '**' + pull_request: + paths: + - '.dockerignore' + - '.github/workflows/check_labels.yml' + - '.github/workflows/r_extra.yml' + - '.github/workflows/report_ci.yml' + - "ci/docker/**" + - "ci/etc/rprofile" + - "ci/scripts/PKGBUILD" + - "ci/scripts/cpp_*.sh" + - "ci/scripts/install_minio.sh" + - "ci/scripts/r_*.sh" + - "cpp/**" + - "compose.yaml" + - "r/**" + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: | + 0 4 * * * + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-labels: + if: github.event_name != 'schedule' || github.repository == 'apache/arrow' + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + parent-workflow: r_extra + + docker: + needs: check-labels + name: ${{ matrix.title }} + runs-on: ${{ matrix.runs-on }} + if: >- + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: R') + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - envs: + - R_IMAGE=r-ver + - R_ORG=rocker + image: r + runs-on: ubuntu-latest + title: Rocker R-ver + - envs: + - R_IMAGE=r-ver + - R_ORG=rocker + image: r + runs-on: ubuntu-latest + run-options: >- + -e ARROW_DEPENDENCY_SOURCE=BUNDLED + title: Rocker R-ver - Bundled Dependencies + - envs: + - R_IMAGE=r-ver + - R_ORG=rocker + image: r + runs-on: ubuntu-latest + run-options: >- + -e LIBARROW_MINIMAL=TRUE + title: Rocker R-ver - Minimal + - envs: + - R_IMAGE=r-ver + - R_ORG=rocker + image: r + runs-on: ubuntu-latest + run-options: >- + -e ARROW_OFFLINE_BUILD=true + title: Rocker R-ver - Offline build + - envs: + - R_IMAGE=ubuntu-release + - R_ORG=rhub + image: r + runs-on: ubuntu-latest + title: Rhub + - envs: + - R_CUSTOM_CCACHE=true + - R_IMAGE=ubuntu-gcc12 + - R_ORG=rhub + image: r + runs-on: ubuntu-latest + title: Rhub GCC 12 + - envs: + - R_IMAGE=gcc13 + - R_ORG=rhub + image: r + runs-on: ubuntu-latest + run-options: >- + -e INSTALL_ARGS=--use-LTO + title: Rhub GCC 13 with LTO + - envs: + - R_IMAGE=r-base + - R_ORG=rstudio + - R_TAG=4.2-jammy + image: r + runs-on: ubuntu-latest + title: RStudio R-Base 4.2 + env: + ARCHERY_DEBUG: 1 + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker + key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.image }}- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3 + - name: Setup Archery + run: python3 -m pip install -e dev/archery[docker] + - name: Execute Docker Build + continue-on-error: ${{ matrix.continue-on-error || false }} + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + ENVS: ${{ toJSON(matrix.envs) }} + run: | + source ci/scripts/util_enable_core_dumps.sh + if [ "${ENVS}" != "null" ]; then + echo "${ENVS}" | jq -r '.[]' | while read env; do + echo "${env}" >> .env + done + fi + archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} + - name: Docker Push + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push ${{ matrix.image }} + - name: Dump install logs + run: cat r/check/arrow.Rcheck/00install.out + if: always() + continue-on-error: true + - name: Dump test logs + run: cat r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + continue-on-error: true + - name: Save the test output + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-output-{{ "${{ matrix.config.r_image }}" }} + path: r/check/arrow.Rcheck/tests/testthat.Rout* + continue-on-error: true + + report-extra-cpp: + if: github.event_name == 'schedule' && always() + needs: + - docker + uses: ./.github/workflows/report_ci.yml + secrets: inherit diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 6c17eecabb2..01541dcecbc 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -141,30 +141,6 @@ env: {% endif %} {% endmacro %} -{%- macro azure_checkout_arrow() -%} - - script: | - git clone --no-checkout --branch {{ arrow.branch }} {{ arrow.remote }} arrow - git -C arrow checkout {{ arrow.head }} - git -C arrow submodule update --init --recursive - displayName: Clone arrow -{% endmacro %} - -{%- macro azure_free_space() -%} - - script: arrow/ci/scripts/util_free_space.sh - displayName: Free up disk space -{% endmacro %} - -{%- macro azure_upload_anaconda(pattern) -%} - {%- if arrow.is_default_branch() -%} - - bash: | - conda create -y -n azure_upload_anaconda -c conda-forge anaconda-client - source activate azure_upload_anaconda - anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force {{ pattern }} - displayName: Upload packages to Anaconda - - {% endif %} -{% endmacro %} - {%- macro configure_homebrew_arrow(formula) -%} - name: Configure Homebrew formula for testing env: @@ -322,17 +298,3 @@ env: {{ key }}: "{{ value }}" {% endfor %} {% endmacro %} - -{% macro azure_set_sccache_envvars(sccache_key_prefix = "sccache") %} - {% set sccache_vars = { - "AWS_ACCESS_KEY_ID": '$(AWS_ACCESS_KEY_ID)', - "AWS_SECRET_ACCESS_KEY": '$(AWS_SECRET_ACCESS_KEY)', - "SCCACHE_BUCKET": '$(SCCACHE_BUCKET)', - "SCCACHE_REGION": '$(SCCACHE_REGION)', - "SCCACHE_S3_KEY_PREFIX": sccache_key_prefix - } - %} - {% for key, value in sccache_vars.items() %} - {{ key }}: "{{ value }}" - {% endfor %} -{% endmacro %} diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml deleted file mode 100644 index a74208ab254..00000000000 --- a/dev/tasks/r/azure.linux.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -{% import 'macros.jinja' as macros with context %} - -jobs: -- job: linux - pool: - vmImage: ubuntu-latest - timeoutInMinutes: 360 - {% if env is defined %} - variables: - {% for key, value in env.items() %} - {{ key }}: {{ value }} - {% endfor %} - {% endif %} - steps: - {{ macros.azure_checkout_arrow() }} - {{ macros.azure_free_space() }} - - - script: | - set -ex - docker -v - docker compose version - cd arrow - export R_ORG={{ r_org }} - export R_IMAGE={{ r_image }} - export R_TAG={{ r_tag }} - export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }} - docker compose pull --ignore-pull-failures r - docker compose build r - displayName: Docker build - env: - {{ macros.azure_set_sccache_envvars()|indent(4) }} - - - script: | - set -ex - cd arrow - export R_ORG={{ r_org }} - export R_IMAGE={{ r_image }} - export R_TAG={{ r_tag }} - export ARROW_R_DEV={{ not_cran|default("TRUE") }} - # Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true - # if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off. - docker compose run {{ flags|default("") }} r - displayName: Docker run - env: - {{ macros.azure_set_sccache_envvars()|indent(4) }} - - - script: | - set -ex - cat arrow/r/check/arrow.Rcheck/00install.out - displayName: Dump install logs - condition: succeededOrFailed() - - script: | - set -ex - cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* - displayName: Dump test logs - condition: succeededOrFailed() diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 065556c805e..266073daff6 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -690,15 +690,6 @@ tasks: ci: github template: r/github.devdocs.yml - test-r-depsource-bundled: - ci: azure - template: r/azure.linux.yml - params: - r_org: rocker - r_image: r-ver - r_tag: latest - flags: '-e ARROW_DEPENDENCY_SOURCE=BUNDLED' - test-r-depsource-system: ci: github template: docker-tests/github.linux.yml @@ -706,15 +697,6 @@ tasks: flags: '-e ARROW_DEPENDENCY_SOURCE=SYSTEM -e ARROW_GCS=OFF -e xsimd_SOURCE=BUNDLED' image: ubuntu-r-only-r - test-r-offline-minimal: - ci: azure - template: r/azure.linux.yml - params: - r_org: rocker - r_image: r-ver - r_tag: latest - flags: '-e ARROW_OFFLINE_BUILD=true' - test-r-dev-duckdb: ci: github template: docker-tests/github.linux.yml @@ -727,39 +709,6 @@ tasks: ci: github template: r/github.linux.offline.build.yml - test-r-rhub-ubuntu-gcc12-custom-ccache: - ci: azure - template: r/azure.linux.yml - params: - r_org: rhub - r_image: ubuntu-gcc12 - r_tag: latest - r_custom_ccache: true - -{% for r_org, r_image, r_tag in [("rhub", "ubuntu-release", "latest"), - ("rocker", "r-ver", "latest"), - ("rstudio", "r-base", "4.2-jammy")] %} - test-r-{{ r_org }}-{{ r_image }}-{{ r_tag }}: - ci: azure - template: r/azure.linux.yml - params: - r_org: {{ r_org }} - r_image: {{ r_image }} - r_tag: {{ r_tag }} -{% endfor %} - - # This is with R built with --enable-lto - # CRAN also does R CMD INSTALL --use-LTO - # which overrides the UseLTO field in r/DESCRIPTION - test-r-rhub-debian-gcc-devel-lto-latest: - ci: azure - template: r/azure.linux.yml - params: - r_org: rhub - r_image: gcc13 - r_tag: latest - flags: '-e INSTALL_ARGS=--use-LTO' - # This one has -flto=auto test-r-ubuntu-22.04: ci: github @@ -790,15 +739,6 @@ tasks: GCC: 12 image: ubuntu-r-only-r - test-r-minimal-build: - ci: azure - template: r/azure.linux.yml - params: - r_org: rocker - r_image: r-ver - r_tag: latest - flags: "-e LIBARROW_MINIMAL=TRUE" - test-r-linux-sanitizers: ci: github template: r/github.linux.sanitizers.yml