openml-python/.github/workflows/test.yml at b10eb12e593e6605555d989d03ee57504ab999e5 · openml/openml-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
---
name: Tests

on:
  workflow_dispatch:

  push:
    branches:
      - main
      - develop
    tags:
      - "v*.*.*"

  pull_request:
    branches:
      - main
      - develop

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    name: (${{ matrix.os }},Py${{ matrix.python-version }},sk${{ matrix.scikit-learn }},sk-only:${{ matrix.sklearn-only }})
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
        scikit-learn: ["1.3.*", "1.4.*", "1.5.*", "1.6.*", "1.7.*"]
        os: [ubuntu-latest]
        sklearn-only: ["true"]

        exclude:
          # incompatible version combinations
          - python-version: "3.13"
            scikit-learn: "1.3.*"
          - python-version: "3.13"
            scikit-learn: "1.4.*"
          - python-version: "3.14"
            scikit-learn: "1.3.*"
          - python-version: "3.14"
            scikit-learn: "1.4.*"

        include:
          # Full test run on ubuntu, 3.14
          - os: ubuntu-latest
            python-version: "3.14"
            scikit-learn: "1.7.*"
            sklearn-only: "false"

          # Full test run on Windows
          - os: windows-latest
            python-version: "3.12"
            scikit-learn: "1.5.*"
            sklearn-only: "false"

          # Coverage run
          - os: ubuntu-latest
            python-version: "3.12"
            scikit-learn: "1.5.*"
            sklearn-only: "false"
            code-cov: true

    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 2

    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install test dependencies and scikit-learn
      run: |
        python -m pip install --upgrade pip
        pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }}

    - name: Store repository status
      id: status-before
      if: matrix.os != 'windows-latest'
      run: |
        git_status=$(git status --porcelain -b)
        echo "BEFORE=$git_status" >> $GITHUB_ENV
        echo "Repository status before tests: $git_status"

    - name: Show installed dependencies
      run: python -m pip list

    - name: Run tests on Ubuntu Test
      if: matrix.os == 'ubuntu-latest'
      run: |
        if [ "${{ matrix.code-cov }}" = "true" ]; then
          codecov="--cov=openml --long --cov-report=xml"
        fi

        if [ "${{ matrix.sklearn-only }}" = "true" ]; then
          marks="sklearn and not production and not uses_test_server"
        else
          marks="not production and not uses_test_server"
        fi

        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

    - name: Run tests on Ubuntu Production
      if: matrix.os == 'ubuntu-latest'
      run: |
        if [ "${{ matrix.code-cov }}" = "true" ]; then
          codecov="--cov=openml --long --cov-report=xml"
        fi

        if [ "${{ matrix.sklearn-only }}" = "true" ]; then
          marks="sklearn and production and not uses_test_server"
        else
          marks="production and not uses_test_server"
        fi

        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

    - name: Run tests on Windows
      if: matrix.os == 'windows-latest'
      run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
        pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not uses_test_server"

    - name: Check for files left behind by test
      if: matrix.os != 'windows-latest' && always()
      run: |
        before="${{ env.BEFORE }}"
        after="$(git status --porcelain -b)"
        if [[ "$before" != "$after" ]]; then
            echo "git status from before: $before"
            echo "git status from after: $after"
            echo "Not all generated files have been deleted!"
            exit 1
        fi

    - name: Upload coverage
      if: matrix.code-cov && always()
      uses: codecov/codecov-action@v4
      with:
        files: coverage.xml
        token: ${{ secrets.CODECOV_TOKEN }}
        fail_ci_if_error: true
        verbose: true

  dummy_windows_py_sk024:
    name: (windows-latest, Py, sk0.24.*, sk-only:false)
    runs-on: ubuntu-latest
    steps:
      - name: Dummy step
        run: |
          echo "This is a temporary dummy job."
          echo "Always succeeds."

  dummy_windows_py_sk023:
    name: (ubuntu-latest, Py3.8, sk0.23.1, sk-only:false)
    runs-on: ubuntu-latest
    steps:
      - name: Dummy step
        run: |
          echo "This is a temporary dummy job."
          echo "Always succeeds."

  dummy_docker:
    name: docker
    runs-on: ubuntu-latest
    steps:
      - name: Dummy step
        run: |
          echo "This is a temporary dummy docker job."
          echo "Always succeeds."