From 565afc6546fe27b1fbc81db1904f2cce6b29ca9f Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Mon, 10 Mar 2025 16:07:16 -0700 Subject: [PATCH 01/14] Refresh login only when needed --- pori_python/graphkb/util.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pori_python/graphkb/util.py b/pori_python/graphkb/util.py index a42695fa..00b2e749 100644 --- a/pori_python/graphkb/util.py +++ b/pori_python/graphkb/util.py @@ -163,11 +163,16 @@ def request(self, endpoint: str, method: str = "GET", **kwargs) -> Dict: # about catching OSError as well as ConnectionError: # https://stackoverflow.com/questions/74253820 attempts = range(15) + need_refresh_login = False for attempt in attempts: if attempt > 0: time.sleep(2) # wait between retries try: - self.refresh_login() + + if need_refresh_login: + self.refresh_login() + need_refresh_login = False + self.request_count += 1 resp = requests.request( method, url, headers=self.headers, timeout=timeout, **kwargs @@ -175,6 +180,7 @@ def request(self, endpoint: str, method: str = "GET", **kwargs) -> Dict: if resp.status_code == 401 or resp.status_code == 403: logger.debug(f"/{endpoint} - {resp.status_code} - retrying") # try to re-login if the token expired + need_refresh_login = True continue else: break From b90291858137203a13fbf0257e782ba754a22101 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Mon, 17 Mar 2025 14:37:52 -0700 Subject: [PATCH 02/14] Update TMB Signature CategoryVariant matching --- pori_python/ipr/annotate.py | 43 +++++++++++++++++++++++++++--------- pori_python/ipr/constants.py | 7 ++++-- pori_python/ipr/main.py | 27 +++++++++++++--------- 3 files changed, 53 insertions(+), 24 deletions(-) diff --git a/pori_python/ipr/annotate.py b/pori_python/ipr/annotate.py index 771f35b2..90cf5ae9 100644 --- a/pori_python/ipr/annotate.py +++ b/pori_python/ipr/annotate.py @@ -26,7 +26,7 @@ Variant, ) -from .constants import TMB_HIGH_CATEGORY +from .constants import TMB_SIGNATURE, TMB_SIGNATURE_VARIANT_TYPE from .ipr import convert_statements_to_alterations from .util import convert_to_rid_set, logger @@ -354,7 +354,8 @@ def annotate_msi( def annotate_tmb( graphkb_conn: GraphKBConnection, disease_matches: List[str], - category: str = TMB_HIGH_CATEGORY, + reference: str = TMB_SIGNATURE, + type: str = TMB_SIGNATURE_VARIANT_TYPE, ) -> List[KbMatch]: """Annotate Tumour Mutation Burden (tmb) categories from GraphKB in the IPR alterations format. @@ -362,33 +363,53 @@ def annotate_tmb( Args: graphkb_conn (GraphKBConnection): the graphkb api connection object disease_matches (list.str): GraphKB disease RIDs - category: such as 'high mutation burden' + reference: CategoryVariant reference, e.g. Signature 'mutation burden' + type: CategoryVariant type, e.g. Vocabulary 'high signature' Returns: list of kbMatches records for IPR """ gkb_matches = [] - categories = graphkb_conn.query( + categoryVariants = graphkb_conn.query( { "target": { "target": "CategoryVariant", "filters": { - "reference1": { - "target": "Signature", - "filters": {"OR": [{"name": category}, {"displayName": category}]}, - } + "AND": [ + { + "reference1": { + "target": "Signature", + "filters": { + "OR": [ + {"name": reference}, + {"displayName": reference}, + # KBDEV-1246 + # Keep support for 'high mutation burden' until statement datafix + {'name': 'high mutation burden'}, + {'displayName': 'high mutation burden'}, + ] + }, + }, + }, + { + "type": { + "target": "Vocabulary", + "filters": {"OR": [{"name": type}, {"displayName": type}]}, + }, + }, + ], }, }, "queryType": "similarTo", "returnProperties": ["@rid", "displayName"], } ) - if categories: - cat_variants = [cast(Variant, var) for var in categories] + if categoryVariants: + cat_variants = [cast(Variant, var) for var in categoryVariants] for ipr_row in get_ipr_statements_from_variants( graphkb_conn, cat_variants, disease_matches ): - ipr_row["variant"] = category + ipr_row["variant"] = f'{TMB_SIGNATURE} {TMB_SIGNATURE_VARIANT_TYPE}' ipr_row["variantType"] = "tmb" gkb_matches.append(ipr_row) return gkb_matches diff --git a/pori_python/ipr/constants.py b/pori_python/ipr/constants.py index 14bca952..c1dbdf73 100644 --- a/pori_python/ipr/constants.py +++ b/pori_python/ipr/constants.py @@ -8,5 +8,8 @@ # Signatures COSMIC_SIGNATURE_VARIANT_TYPE = "high signature" HLA_SIGNATURE_VARIANT_TYPE = "signature present" -TMB_HIGH = 10.0 # genomic mutations per mb - https://www.bcgsc.ca/jira/browse/GERO-296 -TMB_HIGH_CATEGORY = "high mutation burden" +TMB_SIGNATURE = "mutation burden" +TMB_SIGNATURE_HIGH_THRESHOLD = ( + 10.0 # genomic mutations per mb - https://www.bcgsc.ca/jira/browse/GERO-296 +) +TMB_SIGNATURE_VARIANT_TYPE = "high signature" diff --git a/pori_python/ipr/main.py b/pori_python/ipr/main.py index 9817a983..26c0a9e4 100644 --- a/pori_python/ipr/main.py +++ b/pori_python/ipr/main.py @@ -31,7 +31,12 @@ annotate_tmb, ) from .connection import IprConnection -from .constants import DEFAULT_URL, TMB_HIGH, TMB_HIGH_CATEGORY +from .constants import ( + DEFAULT_URL, + TMB_SIGNATURE, + TMB_SIGNATURE_HIGH_THRESHOLD, + TMB_SIGNATURE_VARIANT_TYPE, +) from .inputs import ( check_comparators, check_variant_links, @@ -402,25 +407,25 @@ def ipr_report( except Exception as err: logger.error(f"tmburMutationBurden parsing failure: {err}") - if tmb_val >= TMB_HIGH: + if tmb_val >= TMB_SIGNATURE_HIGH_THRESHOLD: + tmb_cat = f'{TMB_SIGNATURE} {TMB_SIGNATURE_VARIANT_TYPE}' + logger.warning( - f"GERO-296 - tmburMutationBurden high -checking graphkb matches for {TMB_HIGH_CATEGORY}" + f"GERO-296 - tmburMutationBurden high -checking graphkb matches for {tmb_cat}" ) if not tmb.get("key"): - tmb["key"] = TMB_HIGH_CATEGORY + tmb["key"] = tmb_cat if not tmb.get("kbCategory"): - tmb["kbCategory"] = TMB_HIGH_CATEGORY + tmb["kbCategory"] = tmb_cat # GERO-296 - try matching to graphkb - tmb_matches = annotate_tmb(graphkb_conn, disease_matches, TMB_HIGH_CATEGORY) + tmb_matches = annotate_tmb(graphkb_conn, disease_matches) if tmb_matches: - tmb_variant["kbCategory"] = TMB_HIGH_CATEGORY # type: ignore - tmb_variant["variant"] = TMB_HIGH_CATEGORY + tmb_variant["kbCategory"] = tmb_cat + tmb_variant["variant"] = tmb_cat tmb_variant["key"] = tmb["key"] tmb_variant["variantType"] = "tmb" - logger.info( - f"GERO-296 '{TMB_HIGH_CATEGORY}' matches {len(tmb_matches)} statements." - ) + logger.info(f"GERO-296 '{tmb_cat}' matches {len(tmb_matches)} statements.") gkb_matches.extend([Hashabledict(tmb_statement) for tmb_statement in tmb_matches]) logger.debug(f"\tgkb_matches: {len(gkb_matches)}") From 2f83a8f51d8b6876373c7b8102e3cc5eeb98b839 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Thu, 10 Apr 2025 11:45:34 -0700 Subject: [PATCH 03/14] Add quick-tests.yml to GH workflows --- .github/workflows/quick-tests.yml | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/quick-tests.yml diff --git a/.github/workflows/quick-tests.yml b/.github/workflows/quick-tests.yml new file mode 100644 index 00000000..bd8ff917 --- /dev/null +++ b/.github/workflows/quick-tests.yml @@ -0,0 +1,47 @@ +# Limited testing using Python 3.9 only, EXCLUDING integration tests +# Triggered by a push (new branch, or new commit to any branch), EXCEPT +# if that branch is part of a regular (non-draft) pull request. + +name: quick-tests + +on: + push: + +jobs: + + quick-tests: + if: ${{ !github.event.pull_request || github.event.pull_request.draft == true }} + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -e .[test] # coverage reports need -e to capture properly + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics + + - name: Check with black + run: | + pip install black + black --check -S -l 100 pori_python tests + + - name: Tests with pytest, excluding integration tests + run: pytest --junitxml=junit/test-results-3.9.xml --cov ipr --cov-report term --cov-report xml + env: + IPR_USER: ${{ secrets.IPR_TEST_USER }} + IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} + GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} + GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} + EXCLUDE_INTEGRATION_TESTS: 1 From 96bddf85d14c47244095f56c5d4ddc2b2890d8a0 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Thu, 10 Apr 2025 11:45:56 -0700 Subject: [PATCH 04/14] Add quick-tests-integration.yml to GH workflows --- .github/workflows/quick-tests-integration.yml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/quick-tests-integration.yml diff --git a/.github/workflows/quick-tests-integration.yml b/.github/workflows/quick-tests-integration.yml new file mode 100644 index 00000000..188b3d32 --- /dev/null +++ b/.github/workflows/quick-tests-integration.yml @@ -0,0 +1,49 @@ +# Limited testing using Python 3.9 only, INCLUDING integration tests +# Triggered by a regular (non-draft) pull request created +# Triggered by a regular (non-draft) pull request updated by a new commit +# Triggered by a draft pull request converted into a regular (non-draft) one + +name: quick-tests-integration + +on: + pull_request: + types: [opened, synchronize, ready_for_review] + +jobs: + + quick-tests-integration: + if: ${{ github.event.pull_request.draft == false }} + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -e .[test] # coverage reports need -e to capture properly + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics + + - name: Check with black + run: | + pip install black + black --check -S -l 100 pori_python tests + + - name: Tests with pytest, including integration tests + run: pytest --junitxml=junit/test-results-3.9.xml --cov ipr --cov-report term --cov-report xml + env: + IPR_USER: ${{ secrets.IPR_TEST_USER }} + IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} + GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} + GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} + EXCLUDE_INTEGRATION_TESTS: 0 From 28b3af2cb76fde8988b772836b7fbf59d1b05b06 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Thu, 10 Apr 2025 11:56:37 -0700 Subject: [PATCH 05/14] Small comment edit --- .github/workflows/quick-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quick-tests.yml b/.github/workflows/quick-tests.yml index bd8ff917..0483c42d 100644 --- a/.github/workflows/quick-tests.yml +++ b/.github/workflows/quick-tests.yml @@ -1,6 +1,6 @@ # Limited testing using Python 3.9 only, EXCLUDING integration tests # Triggered by a push (new branch, or new commit to any branch), EXCEPT -# if that branch is part of a regular (non-draft) pull request. +# if that branch is part of a regular (non-draft) pull request name: quick-tests From 9702a2a5032b119a7a24c1de14c8d7792a0a16fd Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Thu, 10 Apr 2025 15:09:19 -0700 Subject: [PATCH 06/14] Update quick-tests.yml & quick-tests-integration.yml --- .github/workflows/quick-tests-integration.yml | 9 +++++---- .github/workflows/quick-tests.yml | 9 ++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/quick-tests-integration.yml b/.github/workflows/quick-tests-integration.yml index 188b3d32..2489c9cb 100644 --- a/.github/workflows/quick-tests-integration.yml +++ b/.github/workflows/quick-tests-integration.yml @@ -1,12 +1,13 @@ -# Limited testing using Python 3.9 only, INCLUDING integration tests -# Triggered by a regular (non-draft) pull request created -# Triggered by a regular (non-draft) pull request updated by a new commit -# Triggered by a draft pull request converted into a regular (non-draft) one +# Limited testing using Python 3.9 only INCLUDING integration tests +# Triggered by a regular (non-draft) pull request created, except to main branch +# Triggered by a regular (non-draft) pull request updated by a new commit, except to main branch +# Triggered by a draft pull request to develop branch converted into a regular (non-draft) one name: quick-tests-integration on: pull_request: + branches: [develop] types: [opened, synchronize, ready_for_review] jobs: diff --git a/.github/workflows/quick-tests.yml b/.github/workflows/quick-tests.yml index 0483c42d..a58ee7fb 100644 --- a/.github/workflows/quick-tests.yml +++ b/.github/workflows/quick-tests.yml @@ -1,5 +1,5 @@ -# Limited testing using Python 3.9 only, EXCLUDING integration tests -# Triggered by a push (new branch, or new commit to any branch), EXCEPT +# Limited testing using Python 3.9 only EXCLUDING integration tests +# Triggered by any push (new branch, or new commit to any branch), EXCEPT # if that branch is part of a regular (non-draft) pull request name: quick-tests @@ -10,7 +10,10 @@ on: jobs: quick-tests: - if: ${{ !github.event.pull_request || github.event.pull_request.draft == true }} + if: ${{ !( + github.event.pull_request && + github.event.pull_request.draft == false + ) }} runs-on: ubuntu-latest steps: - name: Checkout repository From b7c219cb450240eb81c30bf33ee41ec56ede62a3 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Thu, 10 Apr 2025 15:11:02 -0700 Subject: [PATCH 07/14] Add extended-tests-integration.yml to GH workflows --- .../workflows/extended-tests-integration.yml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/extended-tests-integration.yml diff --git a/.github/workflows/extended-tests-integration.yml b/.github/workflows/extended-tests-integration.yml new file mode 100644 index 00000000..83bfefaa --- /dev/null +++ b/.github/workflows/extended-tests-integration.yml @@ -0,0 +1,93 @@ +# Extended testing using Python 3.9, 3.10, 3.11 & 3.12 INCLUDING integration tests +# Triggered by a pull request to main branch created +# Triggered by a pull request to main branch updated by a new commit +# Triggered by a pull request to main branch converted from drat to regular (non-draft) +# EXCEPTION for draft pull request: EXCLUDING integration tests, upload test results & code cov. + +name: extended-tests-integration + +on: + pull_request: + branches: [main] + types: [opened, synchronize, ready_for_review] + +jobs: + + extended-tests-integration: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + steps: + - name: Checkout repository + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -e .[test] # coverage reports need -e to capture properly + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics + + - name: Check with black + run: | + pip install black + black --check -S -l 100 pori_python tests + + - name: Tests with pytest, including integration tests + if: ${{ github.event.pull_request.draft == true }} + run: | + pip list + pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov pori_python --cov-report term --cov-report xml + env: + IPR_USER: ${{ secrets.IPR_TEST_USER }} + IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} + GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} + GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} + EXCLUDE_INTEGRATION_TESTS: 1 + + - name: Tests with pytest, including integration tests + if: ${{ github.event.pull_request.draft == false }} + run: | + pip list + pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov pori_python --cov-report term --cov-report xml + env: + IPR_USER: ${{ secrets.IPR_TEST_USER }} + IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} + GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} + GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} + EXCLUDE_INTEGRATION_TESTS: 0 + + - name: Upload pytest test results + if: ${{ ( + github.event.pull_request.draft == false && + matrix.python-version == '3.9' + ) }} + uses: actions/upload-artifact@v4 + with: + name: pytest-results-${{ matrix.python-version }} + path: junit/test-results-${{ matrix.python-version }}.xml + # Use always() to always run this step to publish test results when there are test failures + + - name: Update code coverage report to CodeCov + if: ${{ ( + github.event.pull_request.draft == false && + matrix.python-version == '3.9' + ) }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: true From 5be64e10daebea766bba05e1c280024c183fdc74 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Fri, 11 Apr 2025 10:10:45 -0700 Subject: [PATCH 08/14] Editing GH workflows --- .../workflows/extended-tests-integration.yml | 45 +++++------- .github/workflows/pytest.yml | 68 ------------------- .github/workflows/quick-pytest.yml | 46 ------------- .github/workflows/quick-tests-integration.yml | 19 ++++-- .github/workflows/quick-tests.yml | 17 +++-- 5 files changed, 43 insertions(+), 152 deletions(-) delete mode 100644 .github/workflows/pytest.yml delete mode 100644 .github/workflows/quick-pytest.yml diff --git a/.github/workflows/extended-tests-integration.yml b/.github/workflows/extended-tests-integration.yml index 83bfefaa..b92fd6c0 100644 --- a/.github/workflows/extended-tests-integration.yml +++ b/.github/workflows/extended-tests-integration.yml @@ -1,15 +1,25 @@ -# Extended testing using Python 3.9, 3.10, 3.11 & 3.12 INCLUDING integration tests -# Triggered by a pull request to main branch created -# Triggered by a pull request to main branch updated by a new commit -# Triggered by a pull request to main branch converted from drat to regular (non-draft) -# EXCEPTION for draft pull request: EXCLUDING integration tests, upload test results & code cov. +# Extended testing, INCLUDDING integration tests +# Python: +# - 3.9 +# - 3.10 +# - 3.11 +# - 3.12 +# Triggered by: +# - Creating a new PR to main branch +# - Addind commit(s) to an existing PR to main branch +# - Addind commit(s) to main branch: +# - Direct commit without PR +# - Updating the out-of-date base branch to an existing PR to main branch name: extended-tests-integration on: + push: + branches: [main] pull_request: branches: [main] - types: [opened, synchronize, ready_for_review] + types: [opened, synchronize] + workflow_dispatch: jobs: @@ -44,19 +54,6 @@ jobs: black --check -S -l 100 pori_python tests - name: Tests with pytest, including integration tests - if: ${{ github.event.pull_request.draft == true }} - run: | - pip list - pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov pori_python --cov-report term --cov-report xml - env: - IPR_USER: ${{ secrets.IPR_TEST_USER }} - IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} - GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} - GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} - EXCLUDE_INTEGRATION_TESTS: 1 - - - name: Tests with pytest, including integration tests - if: ${{ github.event.pull_request.draft == false }} run: | pip list pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov pori_python --cov-report term --cov-report xml @@ -68,10 +65,7 @@ jobs: EXCLUDE_INTEGRATION_TESTS: 0 - name: Upload pytest test results - if: ${{ ( - github.event.pull_request.draft == false && - matrix.python-version == '3.9' - ) }} + if: ${{ matrix.python-version == '3.9' }} uses: actions/upload-artifact@v4 with: name: pytest-results-${{ matrix.python-version }} @@ -79,10 +73,7 @@ jobs: # Use always() to always run this step to publish test results when there are test failures - name: Update code coverage report to CodeCov - if: ${{ ( - github.event.pull_request.draft == false && - matrix.python-version == '3.9' - ) }} + if: ${{ matrix.python-version == '3.9' }} uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index 607c0c6e..00000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,68 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: build - -on: - push: - branches: - - main - - develop - pull_request: - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -e .[test] # coverage reports need -e to capture properly - - name: Lint with flake8 - run: | - pip install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics - - name: Check with black - run: | - pip install black - black --check -S -l 100 pori_python tests - - name: Full Tests with pytest - run: | - pip list - pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov pori_python --cov-report term --cov-report xml - env: - IPR_USER: ${{ secrets.IPR_TEST_USER }} - IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} - GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} - GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} - # SDEV-3381 - Turn off integration tests temporarily, till efficiency is increased - # turn on integration tests for one python version only - EXCLUDE_INTEGRATION_TESTS: ${{ matrix.python-version != '3.11' }} - - name: Upload pytest test results - uses: actions/upload-artifact@master - with: - name: pytest-results-${{ matrix.python-version }} - path: junit/test-results-${{ matrix.python-version }}.xml - # Use always() to always run this step to publish test results when there are test failures - if: matrix.python-version == 3.9 - - name: Update code coverage report to CodeCov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml - flags: unittests - env_vars: OS,PYTHON - name: codecov-umbrella - fail_ci_if_error: true - if: matrix.python-version == 3.9 diff --git a/.github/workflows/quick-pytest.yml b/.github/workflows/quick-pytest.yml deleted file mode 100644 index d4f70223..00000000 --- a/.github/workflows/quick-pytest.yml +++ /dev/null @@ -1,46 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: quick-tests - -on: - push: - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: ['3.11'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -e .[test] # coverage reports need -e to capture properly - - name: Lint with flake8 - run: | - pip install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics - - name: Check with black - run: | - pip install black - black --check -S -l 100 pori_python tests - - name: Short Tests with pytest - run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov ipr --cov-report term --cov-report xml - env: - IPR_USER: ${{ secrets.IPR_TEST_USER }} - IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} - GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} - GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} - EXCLUDE_INTEGRATION_TESTS: 1 - # EXCLUDE_INTEGRATION_TESTS: ${{ matrix.python-version != '3.11' }} - if: github.event_name != 'pull_request' \ No newline at end of file diff --git a/.github/workflows/quick-tests-integration.yml b/.github/workflows/quick-tests-integration.yml index 2489c9cb..0d1a95af 100644 --- a/.github/workflows/quick-tests-integration.yml +++ b/.github/workflows/quick-tests-integration.yml @@ -1,19 +1,26 @@ -# Limited testing using Python 3.9 only INCLUDING integration tests -# Triggered by a regular (non-draft) pull request created, except to main branch -# Triggered by a regular (non-draft) pull request updated by a new commit, except to main branch -# Triggered by a draft pull request to develop branch converted into a regular (non-draft) one +# Limited testing, INCLUDING integration tests +# Python: +# - 3.9 +# Triggered by: +# - Creating a new PR to develop branch +# - Addind commit(s) to an existing PR to develop branch +# - Addind commit(s) to develop branch: +# - Direct commit without PR +# - Updating the out-of-date base branch to an existing PR to develop branch name: quick-tests-integration on: + push: + branches: [develop] pull_request: branches: [develop] - types: [opened, synchronize, ready_for_review] + types: [opened, synchronize] + workflow_dispatch: jobs: quick-tests-integration: - if: ${{ github.event.pull_request.draft == false }} runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/.github/workflows/quick-tests.yml b/.github/workflows/quick-tests.yml index a58ee7fb..458cc6f9 100644 --- a/.github/workflows/quick-tests.yml +++ b/.github/workflows/quick-tests.yml @@ -1,18 +1,25 @@ -# Limited testing using Python 3.9 only EXCLUDING integration tests -# Triggered by any push (new branch, or new commit to any branch), EXCEPT -# if that branch is part of a regular (non-draft) pull request +# Limited testing, EXCLUDING integration tests +# Python: +# - 3.9 +# Triggered by: +# - Pushing a new branch +# - Addind commit(s) to a branch that is not main or develop, and that is not part of a PR to develop or main branch name: quick-tests on: push: + branches-ignore: [main, develop] + workflow_dispatch: jobs: quick-tests: if: ${{ !( - github.event.pull_request && - github.event.pull_request.draft == false + github.event.pull_request && ( + github.event.pull_request.base.ref == 'main' || + github.event.pull_request.base.ref == 'develop' + ) ) }} runs-on: ubuntu-latest steps: From d89d2035cf96141983d74595bd15021708b77d01 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Fri, 11 Apr 2025 10:20:44 -0700 Subject: [PATCH 09/14] Add dummy file for test-triggering purpose --- z.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 z.md diff --git a/z.md b/z.md new file mode 100644 index 00000000..e69de29b From d32f4d21b4d37babe9938be910ca6f69ec2bfad9 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Fri, 11 Apr 2025 10:28:43 -0700 Subject: [PATCH 10/14] test --- z.md | 1 + 1 file changed, 1 insertion(+) diff --git a/z.md b/z.md index e69de29b..83c831f0 100644 --- a/z.md +++ b/z.md @@ -0,0 +1 @@ +# test From 84f7b95fb8a164e48da72b3c28afbff89dcd3858 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Fri, 11 Apr 2025 10:30:33 -0700 Subject: [PATCH 11/14] test 2 --- z.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/z.md b/z.md index 83c831f0..efd9c74c 100644 --- a/z.md +++ b/z.md @@ -1 +1 @@ -# test +# test 2 From 9d6b52427ff3e1d5adf5ba2621bfb28ed8026e8c Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Fri, 11 Apr 2025 10:36:41 -0700 Subject: [PATCH 12/14] commenting out pust trigger for extended-tests-integration --- .github/workflows/extended-tests-integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extended-tests-integration.yml b/.github/workflows/extended-tests-integration.yml index b92fd6c0..03f73c29 100644 --- a/.github/workflows/extended-tests-integration.yml +++ b/.github/workflows/extended-tests-integration.yml @@ -14,8 +14,8 @@ name: extended-tests-integration on: - push: - branches: [main] + # push: + # branches: [main] pull_request: branches: [main] types: [opened, synchronize] From 8c44bf67f51bd613dd38cd88cfe27907eea6b5b1 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Fri, 11 Apr 2025 10:38:15 -0700 Subject: [PATCH 13/14] ... --- .github/workflows/extended-tests-integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/extended-tests-integration.yml b/.github/workflows/extended-tests-integration.yml index 03f73c29..977088d9 100644 --- a/.github/workflows/extended-tests-integration.yml +++ b/.github/workflows/extended-tests-integration.yml @@ -17,7 +17,8 @@ on: # push: # branches: [main] pull_request: - branches: [main] + branches: + - main types: [opened, synchronize] workflow_dispatch: From bbd4c205c9e80de1e6f425b2e260046a20999385 Mon Sep 17 00:00:00 2001 From: mathieulemieux Date: Fri, 11 Apr 2025 10:39:26 -0700 Subject: [PATCH 14/14] ... --- .github/workflows/extended-tests-integration.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/extended-tests-integration.yml b/.github/workflows/extended-tests-integration.yml index 977088d9..9144ee31 100644 --- a/.github/workflows/extended-tests-integration.yml +++ b/.github/workflows/extended-tests-integration.yml @@ -14,13 +14,11 @@ name: extended-tests-integration on: - # push: - # branches: [main] + push: + branches: [main] pull_request: - branches: - - main + branches: [main] types: [opened, synchronize] - workflow_dispatch: jobs: