docker-compose.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Usage
# -----
#
# The docker compose file is parametrized using environment variables, the
# defaults are set in .env file.
#
# Example:
# $ ARCH=arm64v8 docker-compose build ubuntu-cpp
# $ ARCH=arm64v8 docker-compose run ubuntu-cpp
#
#
# Coredumps
# ---------
#
# In order to enable coredumps for the C++ tests run by CTest either with
# command `make unittest` or `ctest --output-on-failure` the correct coredump
# patterns must be set.
# The kernel settings are coming from the host, so while it can be enabled from
# a running container using --priviled option the change will affect all other
# containers, so prefer setting it explicitly, directly on the host.
# WARNING: setting this will affect the host machine.
#
# Linux host:
#   $ sudo sysctl -w kernel.core_pattern=core.%e.%p
#
# macOS host running Docker for Mac (won't persist between restarts):
#   $ screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty
#   # echo "core.%e.%p" > /proc/sys/kernel/core_pattern
#
# The setup attempts to generate coredumps by default, but the correct paths
# above must be set. In order to disable the coredump generation set
# ULIMIT_CORE environment variable to 0 before running docker-compose
# (or by setting it in .env file):
#
# ULIMIT_CORE=0 docker-compose run --rm conda-cpp
#
# See more in cpp/build-support/run-test.sh::print_coredumps

version: '3.5'

x-ccache: &ccache
  CCACHE_COMPILERCHECK: content
  CCACHE_COMPRESS: 1
  CCACHE_COMPRESSLEVEL: 6
  CCACHE_MAXSIZE: 500M
  CCACHE_DIR: /ccache

# CPU/memory limit presets to pass to Docker.
#
# Usage: archery docker run --resource-limit=github <image>
#
# Note that exporting ARCHERY_DOCKER_BIN="sudo docker" is likely required,
# unless Docker is configured with cgroups v2 (else Docker will silently
# ignore the limits).
x-limit-presets:
  # These values emulate GitHub Actions:
  # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners
  github:
    # Note we use cpuset and not cpus since Ninja only detects and limits
    # parallelism given the former
    cpuset_cpus: [0, 1]
    memory: 7g

x-with-gpus:
  - ubuntu-cuda-cpp
  - ubuntu-cuda-python

x-hierarchy:
  # This section is used by the archery tool to enable building nested images,
  # so it is enough to call:
  #   archery run debian-ruby
  # instead of a seguence of docker-compose commands:
  #   docker-compose build debian-cpp
  #   docker-compose build debian-c-glib
  #   docker-compose build debian-ruby
  #   docker-compose run --rm debian-ruby
  #
  # Each node must be either a string scalar of a list containing the
  # descendant images if any. Archery checks that all node has a corresponding
  # service entry, so any new image/service must be listed here.
  - almalinux-verify-rc-source
  - conda:
    - conda-cpp:
      - conda-integration
      - conda-cpp-hiveserver2
      - conda-cpp-valgrind
      - conda-python:
        - conda-python-pandas:
          - conda-python-docs
        - conda-python-dask
        - conda-python-hdfs
        - conda-python-java-integration
        - conda-python-jpype
        - conda-python-kartothek
        - conda-python-spark
        - conda-python-turbodbc
  - debian-cpp:
    - debian-c-glib:
      - debian-ruby
    - debian-python
  - debian-go:
    - debian-go-cgo
    - debian-go-cgo-python
  - debian-java:
    - debian-java-jni
  - debian-js
  - fedora-cpp:
    - fedora-python
  - python-sdist
  - ubuntu-cpp:
    - ubuntu-c-glib:
      - ubuntu-ruby
    - ubuntu-lint
    - ubuntu-python:
      - ubuntu-docs
    - ubuntu-python-sdist-test
    - ubuntu-r
    - ubuntu-r-only-r
  - ubuntu-cpp-bundled
  - ubuntu-cuda-cpp:
    - ubuntu-cuda-python
  - ubuntu-csharp
  - ubuntu-cpp-sanitizer
  - ubuntu-cpp-thread-sanitizer
  - ubuntu-r-sanitizer
  - ubuntu-r-valgrind
  - ubuntu-verify-rc-source
  - fedora-r-clang-sanitizer
  - r
  - r-revdepcheck
  # helper services
  - impala
  - postgres
  - python-wheel-manylinux-2010
  - python-wheel-manylinux-2014:
    - java-jni-manylinux-2014
  - python-wheel-manylinux-test-imports
  - python-wheel-manylinux-test-unittests
  - python-wheel-windows-vs2017
  - python-wheel-windows-test

volumes:
  conda-ccache:
    name: ${ARCH}-conda-ccache
  debian-ccache:
    name: ${ARCH}-debian-${DEBIAN}-ccache
  debian-rust:
    name: ${ARCH}-debian-${DEBIAN}-rust
  fedora-ccache:
    name: ${ARCH}-fedora-${FEDORA}-ccache
  maven-cache:
    name: maven-cache
  python-wheel-manylinux2010-ccache:
    name: python-wheel-manylinux2010-ccache
  python-wheel-manylinux2014-ccache:
    name: python-wheel-manylinux2014-ccache
  python-wheel-windows-clcache:
    name: python-wheel-windows-clcache
  ubuntu-ccache:
    name: ${ARCH}-ubuntu-${UBUNTU}-ccache

services:

  ################################# C++ #######################################
  # Release build:
  #   docker-compose run -e ARROW_BUILD_TYPE=release conda-cpp|debian-cpp|...
  # Shared only:
  #   docker-compose run -e ARROW_BUILD_STATIC=OFF conda-cpp|debian-cpp|...
  # Static only:
  #   docker-compose run \
  #     -e ARROW_BUILD_SHARED=OFF \
  #     -e ARROW_TEST_LINKAGE=static \
  #     conda-cpp|debian-cpp|...

  conda:
    # Base image for conda builds.
    #
    # Usage:
    #   docker-compose build con
    #   docker-compose run --rm conda
    # Parameters:
    #   ARCH: amd64, arm32v7
    image: ${REPO}:${ARCH}-conda
    build:
      context: .
      dockerfile: ci/docker/conda.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda
      args:
        arch: ${ARCH}
    volumes:
      - .:/arrow:delegated

  conda-cpp:
    # C++ build in conda environment, including the doxygen docs.
    #
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose run --rm conda-cpp
    # Parameters:
    #   ARCH: amd64, arm32v7
    image: ${REPO}:${ARCH}-conda-cpp
    build:
      context: .
      dockerfile: ci/docker/conda-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-cpp
      args:
        repo: ${REPO}
        arch: ${ARCH}
    shm_size: &shm-size 2G
    ulimits: &ulimits
      core: ${ULIMIT_CORE}
    environment:
      <<: *ccache
      ARROW_BUILD_BENCHMARKS: "ON"
      ARROW_BUILD_EXAMPLES: "ON"
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_EXTRA_ERROR_CONTEXT: "ON"
      ARROW_MIMALLOC: "ON"
      ARROW_USE_LD_GOLD: "ON"
      ARROW_USE_PRECOMPILED_HEADERS: "ON"
      BUILD_DOCS_PYTHON: "ON"
    volumes: &conda-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/cpp_test.sh /arrow /build"]

  conda-cpp-valgrind:
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose run --rm conda-cpp-valgrind
    # Parameters:
    #   ARCH: amd64, arm32v7
    image: ${REPO}:${ARCH}-conda-cpp
    build:
      context: .
      dockerfile: ci/docker/conda-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-cpp
      args:
        repo: ${REPO}
        arch: ${ARCH}
    shm_size: *shm-size
    environment:
      <<: *ccache
      ARROW_CXXFLAGS: "-Og"  # Shrink test runtime by enabling minimal optimizations
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_FLIGHT: "OFF"
      ARROW_GANDIVA: "OFF"
      ARROW_JEMALLOC: "OFF"
      ARROW_RUNTIME_SIMD_LEVEL: "AVX2"  # AVX512 not supported by Valgrind (ARROW-9851)
      ARROW_S3: "OFF"
      ARROW_TEST_MEMCHECK: "ON"
      ARROW_USE_LD_GOLD: "ON"
      BUILD_WARNING_LEVEL: "PRODUCTION"
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/cpp_test.sh /arrow /build"]

  debian-cpp:
    # Usage:
    #   docker-compose build debian-cpp
    #   docker-compose run --rm debian-cpp
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 10, 11
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
    build:
      context: .
      dockerfile: ci/docker/debian-${DEBIAN}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
      args:
        arch: ${ARCH}
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_MIMALLOC: "ON"
    volumes: &debian-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
    # integration_skyhook.sh is a no-op unless skyhook is on.
    command: &cpp-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_skyhook.sh /build &&
        /arrow/ci/scripts/cpp_test.sh /arrow /build"

  ubuntu-cpp:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose run --rm ubuntu-cpp
    # Parameters:
    #   ARCH: amd64, arm64v8, s390x, ...
    #   UBUNTU: 18.04, 20.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
      args:
        arch: ${ARCH}
        base: "${ARCH}/ubuntu:${UBUNTU}"
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
        gcc_version: ${GCC_VERSION}
    shm_size: *shm-size
    cap_add:
      - SYS_ADMIN
    devices:
      - "/dev/fuse:/dev/fuse"
    security_opt:
      - apparmor:unconfined
    ulimits: *ulimits
    environment:
      <<: *ccache
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_MIMALLOC: "ON"
    volumes: &ubuntu-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated
    command: *cpp-command

  ubuntu-cpp-bundled:
    # Arrow build with BUNDLED dependencies
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
      args:
        arch: ${ARCH}
        base: "${ARCH}/ubuntu:${UBUNTU}"
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
      ARROW_DEPENDENCY_SOURCE: BUNDLED
      CMAKE_GENERATOR: "Unix Makefiles"
    volumes: *ubuntu-volumes
    command: *cpp-command

  ubuntu-cuda-cpp:
    # Usage:
    #   docker-compose build cuda-cpp
    #   docker-compose run --rm cuda-cpp
    # Also need to edit the host docker configuration as follows:
    #   https://github.com/docker/compose/issues/6691#issuecomment-561504928
    # Parameters:
    #   ARCH: amd64
    #   CUDA: 9.1, 10.0, 10.1
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp
      args:
        arch: ${ARCH}
        base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
      ARROW_CUDA: "ON"
    volumes: *ubuntu-volumes
    command: *cpp-command

  ubuntu-cpp-sanitizer:
    # Usage:
    #   docker-compose build ubuntu-cpp-sanitizer
    #   docker-compose run --rm ubuntu-cpp-sanitizer
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 18.04, 20.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    cap_add:
      # For LeakSanitizer
      - SYS_PTRACE
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
      args:
        arch: ${ARCH}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    environment:
      <<: *ccache
      CC: clang-${CLANG_TOOLS}
      CXX: clang++-${CLANG_TOOLS}
      ARROW_BUILD_STATIC: "OFF"
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_FUZZING: "ON"  # Check fuzz regressions
      ARROW_JEMALLOC: "OFF"
      ARROW_ORC: "OFF"
      ARROW_S3: "OFF"
      ARROW_USE_ASAN: "ON"
      ARROW_USE_UBSAN: "ON"
      # utf8proc 2.1.0 in Ubuntu Bionic has test failures
      utf8proc_SOURCE: "BUNDLED"
    command: *cpp-command

  ubuntu-cpp-thread-sanitizer:
    # Usage:
    #   docker-compose build ubuntu-cpp-thread-sanitizer
    #   docker-compose run --rm ubuntu-cpp-thread-sanitizer
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 18.04, 20.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
      args:
        arch: ${ARCH}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    environment:
      <<: *ccache
      CC: clang-${CLANG_TOOLS}
      CXX: clang++-${CLANG_TOOLS}
      ARROW_BUILD_STATIC: "OFF"
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_DATASET: "ON"
      ARROW_JEMALLOC: "OFF"
      ARROW_ORC: "OFF"
      ARROW_S3: "OFF"
      ARROW_USE_TSAN: "ON"
    command: *cpp-command

  fedora-cpp:
    # Usage:
    #   docker-compose build fedora-cpp
    #   docker-compose run --rm fedora-cpp
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   FEDORA: 33
    image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
    build:
      context: .
      dockerfile: ci/docker/fedora-${FEDORA}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
      args:
        arch: ${ARCH}
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_MIMALLOC: "ON"
      Protobuf_SOURCE: "BUNDLED"  # Need Protobuf >= 3.15
    volumes: &fedora-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}fedora-ccache:/ccache:delegated
    command: *cpp-command

  ############################### C GLib ######################################

  debian-c-glib:
    # Usage:
    #   docker-compose build debian-cpp
    #   docker-compose build debian-c-glib
    #   docker-compose run --rm debian-c-glib
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 10, 11
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
    build:
      context: .
      dockerfile: ci/docker/linux-apt-c-glib.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
      BUILD_DOCS_C_GLIB: "ON"
    volumes: *debian-volumes
    command: &c-glib-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_test.sh /arrow /build"

  ubuntu-c-glib:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose build ubuntu-c-glib
    #   docker-compose run --rm ubuntu-c-glib
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 18.04, 20.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib
    build:
      context: .
      dockerfile: ci/docker/linux-apt-c-glib.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
    volumes: *ubuntu-volumes
    command: *c-glib-command

  ############################### Ruby ########################################
  # Until Ruby is the only dependent implementation on top of C Glib we can
  # test C Glib and Ruby in one pass. This is an optimization to avoid
  # redundant (one for C GLib and one for Ruby doing the same work twice)
  # builds on CI services.

  debian-ruby:
    # Usage:
    #   docker-compose build debian-cpp
    #   docker-compose build debian-c-glib
    #   docker-compose build debian-ruby
    #   docker-compose run --rm debian-ruby
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 10, 11
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-ruby
    build:
      context: .
      dockerfile: ci/docker/linux-apt-ruby.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-ruby
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
      BUILD_DOCS_C_GLIB: "ON"
    volumes: *debian-volumes
    command: &ruby-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_test.sh /arrow /build &&
        /arrow/ci/scripts/ruby_test.sh /arrow /build"

  ubuntu-ruby:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose build ubuntu-c-glib
    #   docker-compose build ubuntu-ruby
    #   docker-compose run --rm ubuntu-ruby
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 18.04, 20.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby
    build:
      context: .
      dockerfile: ci/docker/linux-apt-ruby.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: *ccache
    volumes: *ubuntu-volumes
    command: *ruby-command

  ############################### Python ######################################

  conda-python:
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose run --rm conda-python
    # Parameters:
    #   ARCH: amd64, arm32v7
    #   PYTHON: 3.6, 3.7, 3.8, 3.9
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}
    build:
      context: .
      dockerfile: ci/docker/conda-python.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *conda-volumes
    command: &python-conda-command
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/python_test.sh /arrow"]

  ubuntu-cuda-python:
    # Usage:
    #   docker-compose build cuda-cpp
    #   docker-compose build cuda-python
    #   docker-compose run --rm cuda-python
    # Parameters:
    #   ARCH: amd64
    #   CUDA: 8.0, 10.0, ...
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp
    shm_size: *shm-size
    environment:
      <<: *ccache
      ARROW_CUDA: "ON"
    volumes: *ubuntu-volumes
    command: &python-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/python_test.sh /arrow"

  debian-python:
    # Usage:
    #   docker-compose build debian-cpp
    #   docker-compose build debian-python
    #   docker-compose run --rm debian-python
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 10, 11
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-python-3
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *debian-volumes
    command: *python-command

  ubuntu-python:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose build ubuntu-python
    #   docker-compose run --rm ubuntu-python
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 18.04, 20.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *ubuntu-volumes
    command: *python-command

  fedora-python:
    # Usage:
    #   docker-compose build fedora-cpp
    #   docker-compose build fedora-python
    #   docker-compose run --rm fedora-python
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   FEDORA: 33
    image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-dnf-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-fedora-${FEDORA}-python-3
      args:
        base: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
    shm_size: *shm-size
    environment:
      <<: *ccache
      Protobuf_SOURCE: "BUNDLED"  # Need Protobuf >= 3.15
    volumes: *fedora-volumes
    command: *python-command

  ############################ Python sdist ###################################

  python-sdist:
    # Usage:
    #   docker-compose build python-sdist
    #   docker-compose run --rm python-sdist
    # Parameters:
    #   PYARROW_VERSION: The pyarrow version for sdist such as "3.0.0"
    image: ${REPO}:python-sdist
    build:
      context: .
      dockerfile: ci/docker/python-sdist.dockerfile
      cache_from:
        - ${REPO}:python-sdist
    environment:
      PYARROW_VERSION: ${PYARROW_VERSION:-}
    volumes:
      - .:/arrow:delegated
    command: /arrow/ci/scripts/python_sdist_build.sh /arrow

  ubuntu-python-sdist-test:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose build ubuntu-python-sdist-test
    #   docker-compose run --rm ubuntu-python-sdist-test
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   PYARROW_VERSION: The test target pyarrow version such as "3.0.0"
    #   UBUNTU: 18.04, 20.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    shm_size: *shm-size
    environment:
      <<: *ccache
      PYARROW_VERSION: ${PYARROW_VERSION:-}
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        apt remove -y git &&
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_sdist_test.sh /arrow"

  ############################ Python wheels ##################################

  # See available versions at:
  #    https://quay.io/repository/pypa/manylinux2010_x86_64?tab=tags
  #    only amd64 arch is supported
  python-wheel-manylinux-2010:
    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2010-vcpkg-${VCPKG}
    build:
      args:
        arch: ${ARCH}
        arch_short: ${ARCH_SHORT}
        base: quay.io/pypa/manylinux2010_${ARCH_ALIAS}:2021-10-11-14ac00e
        vcpkg: ${VCPKG}
        python: ${PYTHON}
        manylinux: 2010
      context: .
      dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2010-vcpkg-${VCPKG}
    environment:
      <<: *ccache
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2010-ccache:/ccache:delegated
    command: /arrow/ci/scripts/python_wheel_manylinux_build.sh

  # See available versions at:
  #    https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags
  python-wheel-manylinux-2014:
    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
    build:
      args:
        arch: ${ARCH}
        arch_short: ${ARCH_SHORT}
        base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
        vcpkg: ${VCPKG}
        python: ${PYTHON}
        manylinux: 2014
      context: .
      dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
    environment:
      <<: *ccache
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
    command: /arrow/ci/scripts/python_wheel_manylinux_build.sh

  python-wheel-manylinux-test-imports:
    image: ${ARCH}/python:${PYTHON}
    shm_size: 2G
    volumes:
      - .:/arrow:delegated
    environment:
      CHECK_IMPORTS: "ON"
      CHECK_UNITTESTS: "OFF"
    command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

  python-wheel-manylinux-test-unittests:
    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test
    build:
      args:
        arch: ${ARCH}
        python: ${PYTHON}
      context: .
      dockerfile: ci/docker/python-wheel-manylinux-test.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test
    shm_size: 2G
    volumes:
      - .:/arrow:delegated
    environment:
      CHECK_IMPORTS: "OFF"
      CHECK_UNITTESTS: "ON"
    command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

  python-wheel-windows-vs2017:
    # The windows images must be built locally and pushed to a remote registry:
    # export REPO=ghcr.io/ursacomputing/arrow
    # PYTHON=3.6 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
    # PYTHON=3.7 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
    # PYTHON=3.8 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
    # PYTHON=3.9 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017
    # PYTHON=3.6 archery docker push python-wheel-windows-vs2017
    # PYTHON=3.7 archery docker push python-wheel-windows-vs2017
    # PYTHON=3.8 archery docker push python-wheel-windows-vs2017
    # PYTHON=3.9 archery docker push python-wheel-windows-vs2017
    image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}
    build:
      args:
        vcpkg: ${VCPKG}
        python: ${PYTHON}
      context: .
      dockerfile: ci/docker/python-wheel-windows-vs2017.dockerfile
      # This should make the pushed images reusable, but the image gets rebuilt.
      # Uncomment if no local cache is available.
      # cache_from:
      #   - mcr.microsoft.com/windows/servercore:ltsc2019
      #   - ${REPO}:wheel-windows-vs2017
    volumes:
      - "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache"
      - type: bind
        source: .
        target: "C:/arrow"
    command: arrow\\ci\\scripts\\python_wheel_windows_build.bat

  python-wheel-windows-test:
    image: python:${PYTHON}-windowsservercore-1809
    volumes:
      - type: bind
        source: .
        target: "C:/arrow"
    command: arrow\\ci\\scripts\\python_wheel_windows_test.bat

  java-jni-manylinux-2014:
    image: ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG}
    build:
      args:
        base: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
        java: 1.8.0
      context: .
      dockerfile: ci/docker/java-jni-manylinux-201x.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG}
    environment:
      <<: *ccache
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
    command:
      ["pip install -e /arrow/dev/archery &&
        /arrow/ci/scripts/java_cdata_build.sh /arrow /java-native-build /arrow/java-dist &&
        /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist"]

  ##############################  Integration #################################

  conda-python-pandas:
    # Possible $PANDAS parameters:
    #  - `latest`: latest release
    #  - `master`: git master branch, use `docker-compose run --no-cache`
    #  - `<version>`: specific version available on conda-forge
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-pandas
    #   docker-compose run --rm conda-python-pandas
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS}
    build:
      context: .
      dockerfile: ci/docker/conda-python-pandas.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        numpy: ${NUMPY}
        pandas: ${PANDAS}
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/python_test.sh /arrow"]

  conda-python-docs:
    # Usage:
    #   archery docker run conda-python-docs
    #
    # Only a single rule is enabled for now to check undocumented arguments.
    # We should extend the list of enabled rules after adding this build to
    # the CI pipeline.
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS}
    environment:
      <<: *ccache
      LC_ALL: "C.UTF-8"
      LANG: "C.UTF-8"
      BUILD_DOCS_CPP: "ON"
      BUILD_DOCS_PYTHON: "ON"
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        pip install -e /arrow/dev/archery[numpydoc] &&
        archery numpydoc --allow-rule PR01"]

  conda-python-dask:
    # Possible $DASK parameters:
    #  - `latest`: latest release
    #  - `master`: git master branch, use `docker-compose run --no-cache`
    #  - `<version>`: specific version available on conda-forge
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-dask
    #   docker-compose run --rm conda-python-dask
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK}
    build:
      context: .
      dockerfile: ci/docker/conda-python-dask.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        dask: ${DASK}
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_dask.sh"]

  conda-python-jpype:
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-jpype
    #   docker-compose run --rm conda-python-jpype
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype
    build:
      context: .
      dockerfile: ci/docker/conda-python-jpype.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
    shm_size: *shm-size
    environment:
      <<: *ccache
      ARROW_FLIGHT: "OFF"
      ARROW_GANDIVA: "OFF"
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/java_build.sh /arrow /build &&
        /arrow/ci/scripts/python_test.sh /arrow"]

  conda-python-java-integration:
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-java-integration
    #   docker-compose run --rm conda-python-java-integration
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration
    build:
      context: .
      dockerfile: ci/docker/conda-python-jpype.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        llvm: ${LLVM}
    shm_size: *shm-size
    environment:
      <<: *ccache
      ARROW_DATASET: "OFF"
      ARROW_FLIGHT: "OFF"
      ARROW_GANDIVA: "OFF"
      ARROW_JAVA_CDATA: "ON"
      ARROW_ORC: "OFF"
      ARROW_PARQUET: "OFF"
      ARROW_PLASMA: "OFF"
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
      - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
    command:
      [ "/arrow/ci/scripts/cpp_build.sh /arrow /build &&
          /arrow/ci/scripts/python_build.sh /arrow /build &&
          /arrow/ci/scripts/java_cdata_build.sh /arrow /build/java/c/build /build/java/c &&
          /arrow/ci/scripts/java_build.sh /arrow /build &&
          /arrow/ci/scripts/java_cdata_integration.sh /arrow" ]

  conda-python-turbodbc:
    # Possible $TURBODBC parameters:
    #  - `latest`: latest release
    #  - `master`: git master branch, use `docker-compose run --no-cache`
    #  - `<version>`: specific version available under github releases
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-turbodbc
    #   docker-compose run --rm conda-python-turbodbc
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-turbodbc-${TURBODBC}
    build:
      context: .
      dockerfile: ci/docker/conda-python-turbodbc.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-turbodbc-${TURBODBC}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        turbodbc: ${TURBODBC}
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_turbodbc.sh /turbodbc /build"]

  conda-python-kartothek:
    # Possible $KARTOTHEK parameters:
    #  - `latest`: latest release
    #  - `master`: git master branch, use `docker-compose run --no-cache`
    #  - `<version>`: specific version available under github releases
    # Usage:
    #   docker-compose build conda
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-kartothek
    #   docker-compose run --rm conda-python-kartothek
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-kartothek-${KARTOTHEK}
    build:
      context: .
      dockerfile: ci/docker/conda-python-kartothek.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-kartothek-${KARTOTHEK}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        kartothek: ${KARTOTHEK}
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_kartothek.sh /kartothek /build"]

  ################################## R ########################################

  ubuntu-r:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose build ubuntu-r
    #   docker-compose run ubuntu-r
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R}
    build:
      context: .
      dockerfile: ci/docker/linux-apt-r.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R}
      args:
        arch: ${ARCH}
        r: ${R}
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
        gcc_version: ${GCC_VERSION}
        tz: ${TZ}
        r_prune_deps: ${R_PRUNE_DEPS}
    shm_size: *shm-size
    environment:
      <<: *ccache
      ARROW_R_CXXFLAGS: '-Werror'
      LIBARROW_BUILD: 'false'
      NOT_CRAN: 'true'
      ARROW_R_DEV: ${ARROW_R_DEV}
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/r_test.sh /arrow"

  ubuntu-r-only-r:
    environment:
      ARROW_DEPENDENCY_SOURCE: ''
    extends: ubuntu-r
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_test.sh /arrow"

  r:
    # This lets you test building/installing the arrow R package
    # (including building the C++ library) on any Docker image that contains R
    #
    # Usage:
    #   R_ORG=rhub R_IMAGE=ubuntu-gcc-release R_TAG=latest docker-compose build r
    #   R_ORG=rhub R_IMAGE=ubuntu-gcc-release R_TAG=latest docker-compose run r
    image: ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG}
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG}
      args:
        base: ${R_ORG}/${R_IMAGE}:${R_TAG}
        r_dev: ${ARROW_R_DEV}
        devtoolset_version: ${DEVTOOLSET_VERSION}
        tz: ${TZ}
        r_prune_deps: ${R_PRUNE_DEPS}
    shm_size: *shm-size
    environment:
      LIBARROW_DOWNLOAD: "false"
      ARROW_SOURCE_HOME: "/arrow"
      ARROW_R_DEV: ${ARROW_R_DEV}
      # To test for CRAN release, delete ^^ these two env vars so we download the Apache release
      ARROW_USE_PKG_CONFIG: "false"
      devtoolset_version: ${DEVTOOLSET_VERSION}
    volumes:
      - .:/arrow:delegated
    command: >
      /bin/bash -c "/arrow/ci/scripts/r_test.sh /arrow"

  ubuntu-r-sanitizer:
    # Only 18.04 and amd64 supported
    # Usage:
    #   docker-compose build ubuntu-r-sanitizer
    #   docker-compose run ubuntu-r-sanitizer
    image: ${REPO}:amd64-ubuntu-18.04-r-sanitizer
    cap_add:
      # LeakSanitizer and gdb requires ptrace(2)
      - SYS_PTRACE
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:amd64-ubuntu-18.04-r-sanitizer
      args:
        base: wch1/r-debug:latest
        r_bin: RDsan
        tz: ${TZ}
        r_prune_deps: ${R_PRUNE_DEPS}
    environment:
      <<: *ccache
      ARROW_SOURCE_HOME: "/arrow"
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_sanitize.sh /arrow"

  fedora-r-clang-sanitizer:
    image: ${REPO}:r-rhub-fedora-clang-devel-latest
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:r-rhub-fedora-clang-devel-latest
      args:
        base: rhub/fedora-clang-devel-san
        r_dev: ${ARROW_R_DEV}
        devtoolset_version: ${DEVTOOLSET_VERSION}
        r_bin: RDsan
        tz: ${TZ}
        r_prune_deps: ${R_PRUNE_DEPS}
    shm_size: *shm-size
    environment:
      LIBARROW_DOWNLOAD: "false"
      ARROW_SOURCE_HOME: "/arrow"
      ARROW_R_DEV: ${ARROW_R_DEV}
      # To test for CRAN release, delete ^^ these two env vars so we download the Apache release
      ARROW_USE_PKG_CONFIG: "false"
    volumes:
      - .:/arrow:delegated
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_sanitize.sh /arrow"

  ubuntu-r-valgrind:
    # Only 18.04 and amd64 supported
    # Usage:
    #   docker-compose build ubuntu-r-valgrind
    #   docker-compose run ubuntu-r-valgrind
    image: ${REPO}:amd64-ubuntu-18.04-r-valgrind
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:amd64-ubuntu-18.04-r-valgrind
      args:
        base: wch1/r-debug:latest
        r_bin: RDvalgrind
        tz: ${TZ}
    environment:
      <<: *ccache
      ARROW_R_DEV: ${ARROW_R_DEV}
      # AVX512 not supported by Valgrind (similar to ARROW-9851) some runners support AVX512 and some do not
      # so some build might pass without this setting, but we want to ensure that we stay to AVX2 regardless of runner.
      EXTRA_CMAKE_FLAGS: "-DARROW_RUNTIME_SIMD_LEVEL=AVX2"
      ARROW_SOURCE_HOME: "/arrow"
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_valgrind.sh /arrow"

  r-revdepcheck:
    # Usage:
    #   docker-compose build r-revdepcheck
    #   docker-compose run r-revdepcheck
    image: ${REPO}:r-rstudio-r-base-4.0-focal-revdepcheck
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:r-rstudio-r-base-4.0-focal-revdepcheck
      args:
        base: rstudio/r-base:4.0-focal
        r_dev: ${ARROW_R_DEV}
        tz: ${TZ}
    shm_size: *shm-size
    environment:
      LIBARROW_DOWNLOAD: "true"
      LIBARROW_MINIMAL: "false"
      ARROW_SOURCE_HOME: "/arrow"
      ARROW_R_DEV: "true"
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "/arrow/ci/scripts/r_revdepcheck.sh /arrow"

  ################################# Go ########################################

  debian-go:
    # Usage:
    #   docker-compose build debian-go
    #   docker-compose run debian-go
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}
    build:
      context: .
      dockerfile: ci/docker/debian-${DEBIAN}-go.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}
      args:
        arch: ${ARCH}
        go: ${GO}
    shm_size: *shm-size
    volumes: *debian-volumes
    command: &go-command >
      /bin/bash -c "
        /arrow/ci/scripts/go_build.sh /arrow &&
        /arrow/ci/scripts/go_test.sh /arrow"

  debian-go-cgo:
    # Usage:
    #   docker-compose build debian-go-cgo
    #   docker-compose run debian-go-cgo
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo
    build:
      context: .
      dockerfile: ci/docker/debian-go-cgo.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}
    shm_size: *shm-size
    volumes: *debian-volumes
    environment:
      ARROW_GO_TESTCGO: "1"
    command: *go-command

  debian-go-cgo-python:
    # Usage:
    #   docker-compose build debian-go-cgo-python
    #   docker-compose run debian-go-cgo-python
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo-python
    build:
      context: .
      dockerfile: ci/docker/debian-${DEBIAN}-go-cgo-python.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo-python
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}
    shm_size: *shm-size
    volumes: *debian-volumes
    command: &go-cgo-python-command >
        /bin/bash -c "
          /arrow/ci/scripts/go_cgo_python_test.sh /arrow"

  ############################# JavaScript ####################################

  debian-js:
    # Usage:
    #   docker-compose build debian-js
    #   docker-compose run debian-js
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE}
    build:
      context: .
      dockerfile: ci/docker/debian-${DEBIAN}-js.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE}
      args:
        arch: ${ARCH}
        node: ${NODE}
    shm_size: *shm-size
    volumes: *debian-volumes
    environment:
      BUILD_DOCS_JS: "ON"
    command: &js-command >
      /bin/bash -c "
        /arrow/ci/scripts/js_build.sh /arrow /build &&
        /arrow/ci/scripts/js_test.sh /arrow"

  #################################### C# #####################################

  ubuntu-csharp:
    # Usage:
    #   docker-compose build ubuntu-csharp
    #   docker-compose run ubuntu-csharp
    image: ${REPO}:${ARCH}-ubuntu-18.04-csharp-${DOTNET}
    build:
      context: .
      dockerfile: ci/docker/ubuntu-18.04-csharp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-18.04-csharp-${DOTNET}
      args:
        dotnet: ${DOTNET}
        platform: bionic  # use bionic-arm64v8 for ARM
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    command: &csharp-command >
      /bin/bash -c "
        /arrow/ci/scripts/csharp_build.sh /arrow &&
        /arrow/ci/scripts/csharp_test.sh /arrow &&
        /arrow/ci/scripts/csharp_pack.sh /arrow"

  ################################ Java #######################################

  debian-java:
    # Usage:
    #   docker-compose build debian-java
    #   docker-compose run debian-java
    image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}
    build:
      context: .
      dockerfile: ci/docker/debian-9-java.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}
      args:
        arch: ${ARCH}
        jdk: ${JDK}
        maven: ${MAVEN}
    shm_size: *shm-size
    volumes: &java-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
    command: &java-command >
      /bin/bash -c "
        /arrow/ci/scripts/java_build.sh /arrow /build &&
        /arrow/ci/scripts/java_test.sh /arrow /build"

  debian-java-jni:
    # Includes plasma test, jni for gandiva and orc, and C data interface.
    # Usage:
    #   docker-compose build debian-java
    #   docker-compose build debian-java-jni
    #   docker-compose run debian-java-jni
    image: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
    build:
      context: .
      dockerfile: ci/docker/linux-apt-jni.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}-jni
      args:
        base: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}
        llvm: ${LLVM}
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
      - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
    command:
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/java_cdata_build.sh /arrow /build/java/c/build /build/java/c &&
        /arrow/ci/scripts/java_build.sh /arrow /build &&
        /arrow/ci/scripts/java_test.sh /arrow /build"

  ############################## Integration ##################################

  conda-integration:
    # Usage:
    #   docker-compose build conda-cpp
    #   docker-compose build conda-integration
    #   docker-compose run conda-integration
    image: ${REPO}:${ARCH}-conda-integration
    build:
      context: .
      dockerfile: ci/docker/conda-integration.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-integration
      args:
        repo: ${REPO}
        arch: ${ARCH}
        jdk: ${JDK}
        # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should
        # be set to ${MAVEN}
        maven: 3.5
        node: ${NODE}
        go: ${GO}
    volumes: *conda-volumes
    environment:
      <<: *ccache
      # tell archery where the arrow binaries are located
      ARROW_CPP_EXE_PATH: /build/cpp/debug
      ARCHERY_INTEGRATION_WITH_RUST: 0
    command:
      ["/arrow/ci/scripts/rust_build.sh /arrow /build &&
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/csharp_build.sh /arrow /build &&
        /arrow/ci/scripts/go_build.sh /arrow &&
        /arrow/ci/scripts/java_build.sh /arrow /build &&
        /arrow/ci/scripts/js_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_arrow.sh /arrow /build"]

  ################################ Docs #######################################

  ubuntu-docs:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose build ubuntu-python
    #   docker-compose build ubuntu-docs
    #   docker-compose run --rm ubuntu-docs
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-docs
    build:
      context: .
      dockerfile: ci/docker/linux-apt-docs.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-docs
      args:
        r: ${R}
        jdk: ${JDK}
        node: ${NODE}
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
    environment:
      <<: *ccache
      ARROW_CUDA: "ON"
      BUILD_DOCS_C_GLIB: "ON"
      BUILD_DOCS_CPP: "ON"
      BUILD_DOCS_JAVA: "ON"
      BUILD_DOCS_JS: "ON"
      BUILD_DOCS_PYTHON: "ON"
      BUILD_DOCS_R: "ON"
      Protobuf_SOURCE: "BUNDLED"  # Need Protobuf >= 3.15
    volumes: *ubuntu-volumes
    command: &docs-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/r_build.sh /arrow /build &&
        /arrow/ci/scripts/js_build.sh /arrow /build &&
        /arrow/ci/scripts/java_build.sh /arrow /build"

  ################################# Tools #####################################

  ubuntu-lint:
    # Usage:
    #   docker-compose build ubuntu-cpp
    #   docker-compose build ubuntu-lint
    #   docker-compose run ubuntu-lint
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-lint
    build:
      context: .
      dockerfile: ci/docker/linux-apt-lint.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-lint
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
        clang_tools: ${CLANG_TOOLS}
    environment:
      <<: *ccache
    volumes: *ubuntu-volumes
    command: archery lint --all --no-clang-tidy --no-iwyu --no-numpydoc

  ######################### Integration Tests #################################

  postgres:
    # required for the impala service
    image: postgres
    ports:
      - 5432:5432
    environment:
      POSTGRES_PASSWORD: postgres

  impala:
    # required for the hiveserver and hdfs tests
    image: ibisproject/impala:latest
    hostname: impala
    links:
      - postgres:postgres
    environment:
      PGPASSWORD: postgres
    ports:
      # HDFS
      - 9020:9020
      - 50070:50070
      - 50075:50075
      - 8020:8020
      - 8042:8042
      # Hive
      - 9083:9083
      # Impala
      - 21000:21000
      - 21050:21050
      - 25000:25000
      - 25010:25010
      - 25020:25020

  conda-cpp-hiveserver2:
    # Usage:
    #   docker-compose build conda-cpp
    #   docker-compose build conda-cpp-hiveserver2
    #   docker-compose run conda-cpp-hiveserver2
    image: ${REPO}:${ARCH}-conda-cpp
    links:
      - impala:impala
    environment:
      <<: *ccache
      ARROW_FLIGHT: "OFF"
      ARROW_GANDIVA: "OFF"
      ARROW_PLASMA: "OFF"
      ARROW_HIVESERVER2: "ON"
      ARROW_HIVESERVER2_TEST_HOST: impala
    shm_size: *shm-size
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_hiveserver2.sh /arrow /build"]

  conda-python-hdfs:
    # Usage:
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-hdfs
    #   docker-compose run conda-python-hdfs
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS}
    build:
      context: .
      dockerfile: ci/docker/conda-python-hdfs.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        jdk: ${JDK}
        # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should
        # be set to ${MAVEN}
        maven: 3.5
        hdfs: ${HDFS}
    links:
      - impala:impala
    environment:
      <<: *ccache
      ARROW_HDFS: "ON"
      ARROW_HDFS_TEST_HOST: impala
      ARROW_HDFS_TEST_PORT: 8020
      ARROW_HDFS_TEST_USER: hdfs
      ARROW_S3: "OFF"
      CMAKE_UNITY_BUILD: "ON"
    shm_size: *shm-size
    volumes: &conda-maven-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
      - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_hdfs.sh /arrow /build"]

  conda-python-spark:
    # Usage:
    #   docker-compose build conda-cpp
    #   docker-compose build conda-python
    #   docker-compose build conda-python-spark
    #   docker-compose run conda-python-spark
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK}
    build:
      context: .
      dockerfile: ci/docker/conda-python-spark.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        jdk: ${JDK}
        # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should
        # be set to ${MAVEN}
        maven: 3.5
        spark: ${SPARK}
    shm_size: *shm-size
    environment:
      <<: *ccache
    volumes: *conda-maven-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/java_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_spark.sh /arrow /spark ${TEST_PYARROW_ONLY:-false}"]

  ################################# Source Verification #####################################

  almalinux-verify-rc-source:
    # Usage:
    #   docker-compose build almalinux-verify-rc-source
    #   docker-compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 almalinux-verify-rc-source
    # Parameters:
    #   ALMALINUX: 8
    image: almalinux:${ALMALINUX}
    volumes:
      - .:/arrow:delegated
    shm_size: '1gb'
    command: >
      /bin/bash -c "
        /arrow/dev/release/setup-rhel-rebuilds.sh &&
        /arrow/dev/release/verify-release-candidate.sh source $${VERIFY_VERSION} $${VERIFY_RC}"

  ubuntu-verify-rc-source:
    # Usage:
    #   docker-compose build ubuntu-verify-rc-source
    #   docker-compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 ubuntu-verify-rc-source
    # Parameters:
    #   UBUNTU: 18.04, 20.04
    image: ubuntu:${UBUNTU}
    volumes:
      - .:/arrow:delegated
    shm_size: '1gb'
    command: >
      /bin/bash -c "
        DEBIAN_FRONTEND=noninteractive /arrow/dev/release/setup-ubuntu.sh &&
        /arrow/dev/release/verify-release-candidate.sh source $${VERIFY_VERSION} $${VERIFY_RC}"