Refactor azure.py and test_file_utils.py for improved readability #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| pull_request: | |
| branches: [ main, develop ] | |
| jobs: | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| python-version: ["3.11", "3.12"] | |
| extras: ["", "[excel]", "[parquet]", "[documents]", "[all]"] | |
| exclude: | |
| # Run full matrix on ubuntu, simpler on other OSes | |
| - os: macos-latest | |
| extras: "[parquet]" | |
| - os: macos-latest | |
| extras: "[documents]" | |
| - os: windows-latest | |
| extras: "[parquet]" | |
| - os: windows-latest | |
| extras: "[documents]" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniconda-version: 'latest' | |
| python-version: ${{ matrix.python-version }} | |
| environment-file: environment.yaml | |
| activate-environment: fileutils | |
| auto-update-conda: true | |
| auto-activate-base: false | |
| use-only-tar-bz2: false | |
| - name: Install package with extras (Linux/macOS) | |
| if: matrix.os != 'windows-latest' | |
| shell: bash -l {0} | |
| run: | | |
| pip install -e .${{ matrix.extras }} | |
| - name: Install package with extras (Windows) | |
| if: matrix.os == 'windows-latest' | |
| shell: cmd /C call {0} | |
| run: | | |
| pip install -e .${{ matrix.extras }} | |
| - name: Run unit tests (Linux/macOS) | |
| if: matrix.os != 'windows-latest' | |
| shell: bash -l {0} | |
| run: | | |
| pytest tests/unit/ -v --cov=FileUtils --cov-report=xml --cov-report=term-missing | |
| - name: Run unit tests (Windows) | |
| if: matrix.os == 'windows-latest' | |
| shell: cmd /C call {0} | |
| run: | | |
| pytest tests/unit/ -v --cov=FileUtils --cov-report=xml --cov-report=term-missing | |
| - name: Upload coverage to Codecov | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' && matrix.extras == '' | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./coverage.xml | |
| flags: unittests | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniconda-version: 'latest' | |
| python-version: ${{ matrix.python-version }} | |
| environment-file: environment.yaml | |
| activate-environment: fileutils | |
| auto-update-conda: true | |
| auto-activate-base: false | |
| use-only-tar-bz2: false | |
| - name: Install package with all extras | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils pip install -e .[all] | |
| - name: Run integration tests (excluding Azure) | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils pytest tests/integration/ -v -k "not azure" --tb=short | |
| - name: Run Azure integration tests | |
| if: false # Set to true and add Azure credentials if needed | |
| shell: bash -l {0} | |
| env: | |
| AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }} | |
| run: | | |
| conda run -n fileutils pytest tests/integration/test_azure_storage.py -v --tb=short | |
| lint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniconda-version: 'latest' | |
| python-version: ${{ matrix.python-version }} | |
| environment-file: environment.yaml | |
| activate-environment: fileutils | |
| auto-update-conda: true | |
| auto-activate-base: false | |
| use-only-tar-bz2: false | |
| - name: Install linting dependencies (already in environment.yaml) | |
| shell: bash -l {0} | |
| run: | | |
| # Linting dependencies are already in environment.yaml | |
| - name: Check code formatting with black | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils black --check src/ tests/ examples/ | |
| - name: Check import sorting with isort | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils isort --check-only src/ tests/ examples/ | |
| - name: Lint with flake8 | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils flake8 src/ tests/ examples/ --count --select=E9,F63,F7,F82 --show-source --statistics | |
| conda run -n fileutils flake8 src/ tests/ examples/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Type check with mypy | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils mypy src/ --ignore-missing-imports --no-strict-optional || true | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: [test] | |
| if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniconda-version: 'latest' | |
| python-version: "3.12" | |
| environment-file: environment.yaml | |
| activate-environment: fileutils | |
| auto-update-conda: true | |
| auto-activate-base: false | |
| use-only-tar-bz2: false | |
| - name: Install build dependencies (already in environment.yaml) | |
| shell: bash -l {0} | |
| run: | | |
| # Build dependencies are already in environment.yaml | |
| - name: Build package | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils python -m build | |
| - name: Check package | |
| shell: bash -l {0} | |
| run: | | |
| conda run -n fileutils twine check dist/* | |