Merge remote-tracking branch 'upstream/main' into cow-index-shallow-copy

Author: jorisvandenbossche

.circleci/config.yml

@@ -46,9 +46,15 @@ jobs:
             fi
       - run:
           name: Build aarch64 wheels
+          no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
-            pip3 install cibuildwheel==2.12.1
-            cibuildwheel --output-dir wheelhouse
+            pip3 install cibuildwheel==2.15.0
+            # When this is a nightly wheel build, allow picking up NumPy 2.0 dev wheels:
+            if [[ "$IS_SCHEDULE_DISPATCH" == "true" || "$IS_PUSH" != 'true' ]]; then
+                export CIBW_ENVIRONMENT="PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+            fi
+            cibuildwheel --prerelease-pythons --output-dir wheelhouse
+
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
 
@@ -91,4 +97,4 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"]
+              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64", "cp312-manylinux_aarch64"]

.circleci/setup_env.sh

@@ -48,10 +48,10 @@ source activate pandas-dev
 # downstream CI jobs that may also build pandas from source.
 export PANDAS_CI=1
 
-if pip list | grep -q ^pandas; then
+if pip show pandas 1>/dev/null; then
     echo
     echo "remove any installed pandas package w/o removing anything else"
-    pip uninstall -y pandas || true
+    pip uninstall -y pandas
 fi
 
 echo "Install pandas"

.gitattributes

@@ -14,3 +14,71 @@
 *.xls binary
 *.xlsx binary
 pandas/_version.py export-subst
+
+
+*.bz2 export-ignore
+*.csv export-ignore
+*.data export-ignore
+*.dta export-ignore
+*.feather export-ignore
+*.tar export-ignore
+*.gz export-ignore
+*.h5 export-ignore
+*.html export-ignore
+*.json export-ignore
+*.jsonl export-ignore
+*.kml export-ignore
+*.msgpack export-ignore
+*.pdf export-ignore
+*.parquet export-ignore
+*.pickle export-ignore
+*.pkl export-ignore
+*.png export-ignore
+*.pptx export-ignore
+*.ods export-ignore
+*.odt export-ignore
+*.orc export-ignore
+*.sas7bdat export-ignore
+*.sav export-ignore
+*.so export-ignore
+*.txt export-ignore
+*.xls export-ignore
+*.xlsb export-ignore
+*.xlsm export-ignore
+*.xlsx export-ignore
+*.xpt export-ignore
+*.cpt export-ignore
+*.xml export-ignore
+*.xsl export-ignore
+*.xz export-ignore
+*.zip export-ignore
+*.zst export-ignore
+*~ export-ignore
+.DS_Store export-ignore
+.git* export-ignore
+
+*.py[ocd] export-ignore
+*.pxi export-ignore
+
+# Ignoring stuff from the top level
+.circleci export-ignore
+.github export-ignore
+asv_bench export-ignore
+ci export-ignore
+doc export-ignore
+gitpod export-ignore
+MANIFEST.in export-ignore
+scripts export-ignore
+typings export-ignore
+web export-ignore
+CITATION.cff export-ignore
+codecov.yml export-ignore
+Dockerfile export-ignore
+environment.yml export-ignore
+setup.py export-ignore
+
+
+# GH 39321
+# csv_dir_path fixture checks the existence of the directory
+# exclude the whole directory to avoid running related tests in sdist
+pandas/tests/io/parser/data export-ignore

.github/CODEOWNERS

@@ -9,6 +9,7 @@ web/                              @datapythonista
 
 # docs
 doc/cheatsheet                    @Dr-Irv
+doc/source/development            @noatamir
 
 # pandas
 pandas/_libs/                     @WillAyd

.github/actions/build_pandas/action.yml

@@ -12,13 +12,21 @@ runs:
       run: |
         micromamba info
         micromamba list
+        pip list --pre
+      shell: bash -el {0}
+
+    - name: Uninstall existing Pandas installation
+      run: |
+        if pip show pandas 1>/dev/null; then
+          pip uninstall -y pandas
+        fi
       shell: bash -el {0}
 
     - name: Build Pandas
       run: |
         if [[ ${{ inputs.editable }} == "true" ]]; then
-          pip install -e . --no-build-isolation -v
+          pip install -e . --no-build-isolation -v --no-deps
         else
-          pip install . --no-build-isolation -v
+          pip install . --no-build-isolation -v --no-deps
         fi
       shell: bash -el {0}

.github/workflows/autoupdate-pre-commit-config.yml

@@ -0,0 +1,38 @@
+name: Linkcheck
+on:
+  schedule:
+  # Run monthly on the 1st day of the month
+    - cron: '0 0 1 * *'
+  pull_request:
+    paths:
+      - ".github/workflows/broken-linkcheck.yml"
+      - "doc/make.py"
+jobs:
+  linkcheck:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Conda
+        uses: ./.github/actions/setup-conda
+
+      - name: Build Pandas
+        uses: ./.github/actions/build_pandas
+
+      - name: Run linkcheck script
+        working-directory: ./doc
+        run: |
+          set -o pipefail
+          python make.py linkcheck | tee linkcheck.txt
+
+      - name: Display broken links
+        if: failure()
+        working-directory: ./doc
+        run: grep broken linkcheck.txt

.github/workflows/broken-linkcheck.yml

@@ -0,0 +1,32 @@
+name: Purge caches once a week
+on:
+  schedule:
+    # 4:10 UTC on Sunday
+    - cron: "10 4 * * 0"
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'pandas-dev'
+    permissions:
+      actions: write
+    steps:
+      - name: Clean Cache
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          echo "Fetching list of cache key"
+          allCaches=$(gh actions-cache list -L 100 -R $REPO | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys.
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $allCaches
+          do
+              gh actions-cache delete $cacheKey -R $REPO --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cache-cleanup-weekly.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
   pull_request:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
 
 env:
   ENV_FILE: environment.yml
@@ -17,28 +17,8 @@ env:
 permissions:
   contents: read
 
+# pre-commit run by https://pre-commit.ci/
 jobs:
-  pre_commit:
-    name: pre-commit
-    runs-on: ubuntu-22.04
-    concurrency:
-      # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
-      cancel-in-progress: true
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: Install Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.9'
-
-    - name: Run pre-commit
-      uses: pre-commit/action@v3.0.0
-      with:
-        extra_args: --verbose --all-files
-
   docstring_typing_manual_hooks:
     name: Docstring validation, typing, and other manual pre-commit hooks
     runs-on: ubuntu-22.04
@@ -53,7 +33,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
@@ -129,7 +109,7 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 
@@ -144,7 +124,7 @@ jobs:
       run: |
         cd asv_bench
         asv machine --yes
-        asv run --quick --dry-run --strict --durations=30 --python=same
+        asv run --quick --dry-run --durations=30 --python=same --show-stderr
 
   build_docker_dev_environment:
     name: Build Docker Dev Environment
@@ -163,7 +143,7 @@ jobs:
         run: docker image prune -f
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -184,7 +164,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/code-checks.yml

@@ -27,7 +27,7 @@ jobs:
           - python
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}