diff --git a/.github/workflows/pytest.yml b/.github/workflows/extended-tests-integration.yml similarity index 66% rename from .github/workflows/pytest.yml rename to .github/workflows/extended-tests-integration.yml index 607c0c6e..9144ee31 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/extended-tests-integration.yml @@ -1,43 +1,58 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# Extended testing, INCLUDDING integration tests +# Python: +# - 3.9 +# - 3.10 +# - 3.11 +# - 3.12 +# Triggered by: +# - Creating a new PR to main branch +# - Addind commit(s) to an existing PR to main branch +# - Addind commit(s) to main branch: +# - Direct commit without PR +# - Updating the out-of-date base branch to an existing PR to main branch -name: build +name: extended-tests-integration on: push: - branches: - - main - - develop + branches: [main] pull_request: + branches: [main] + types: [opened, synchronize] jobs: - build: + extended-tests-integration: runs-on: ubuntu-latest strategy: matrix: python-version: ['3.9', '3.10', '3.11', '3.12'] - steps: - - uses: actions/checkout@v3 + - name: Checkout repository + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip setuptools pip install -e .[test] # coverage reports need -e to capture properly + - name: Lint with flake8 run: | pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Check with black run: | pip install black black --check -S -l 100 pori_python tests - - name: Full Tests with pytest + + - name: Tests with pytest, including integration tests run: | pip list pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov pori_python --cov-report term --cov-report xml @@ -46,18 +61,19 @@ jobs: IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} - # SDEV-3381 - Turn off integration tests temporarily, till efficiency is increased - # turn on integration tests for one python version only - EXCLUDE_INTEGRATION_TESTS: ${{ matrix.python-version != '3.11' }} + EXCLUDE_INTEGRATION_TESTS: 0 + - name: Upload pytest test results - uses: actions/upload-artifact@master + if: ${{ matrix.python-version == '3.9' }} + uses: actions/upload-artifact@v4 with: name: pytest-results-${{ matrix.python-version }} path: junit/test-results-${{ matrix.python-version }}.xml # Use always() to always run this step to publish test results when there are test failures - if: matrix.python-version == 3.9 + - name: Update code coverage report to CodeCov - uses: codecov/codecov-action@v3 + if: ${{ matrix.python-version == '3.9' }} + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml @@ -65,4 +81,3 @@ jobs: env_vars: OS,PYTHON name: codecov-umbrella fail_ci_if_error: true - if: matrix.python-version == 3.9 diff --git a/.github/workflows/quick-tests-integration.yml b/.github/workflows/quick-tests-integration.yml new file mode 100644 index 00000000..0d1a95af --- /dev/null +++ b/.github/workflows/quick-tests-integration.yml @@ -0,0 +1,57 @@ +# Limited testing, INCLUDING integration tests +# Python: +# - 3.9 +# Triggered by: +# - Creating a new PR to develop branch +# - Addind commit(s) to an existing PR to develop branch +# - Addind commit(s) to develop branch: +# - Direct commit without PR +# - Updating the out-of-date base branch to an existing PR to develop branch + +name: quick-tests-integration + +on: + push: + branches: [develop] + pull_request: + branches: [develop] + types: [opened, synchronize] + workflow_dispatch: + +jobs: + + quick-tests-integration: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -e .[test] # coverage reports need -e to capture properly + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics + + - name: Check with black + run: | + pip install black + black --check -S -l 100 pori_python tests + + - name: Tests with pytest, including integration tests + run: pytest --junitxml=junit/test-results-3.9.xml --cov ipr --cov-report term --cov-report xml + env: + IPR_USER: ${{ secrets.IPR_TEST_USER }} + IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} + GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} + GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} + EXCLUDE_INTEGRATION_TESTS: 0 diff --git a/.github/workflows/quick-pytest.yml b/.github/workflows/quick-tests.yml similarity index 51% rename from .github/workflows/quick-pytest.yml rename to .github/workflows/quick-tests.yml index d4f70223..458cc6f9 100644 --- a/.github/workflows/quick-pytest.yml +++ b/.github/workflows/quick-tests.yml @@ -1,46 +1,57 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# Limited testing, EXCLUDING integration tests +# Python: +# - 3.9 +# Triggered by: +# - Pushing a new branch +# - Addind commit(s) to a branch that is not main or develop, and that is not part of a PR to develop or main branch name: quick-tests on: push: + branches-ignore: [main, develop] + workflow_dispatch: jobs: - build: + quick-tests: + if: ${{ !( + github.event.pull_request && ( + github.event.pull_request.base.ref == 'main' || + github.event.pull_request.base.ref == 'develop' + ) + ) }} runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: ['3.11'] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.9" + - name: Install dependencies run: | python -m pip install --upgrade pip setuptools pip install -e .[test] # coverage reports need -e to capture properly + - name: Lint with flake8 run: | pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Check with black run: | pip install black black --check -S -l 100 pori_python tests - - name: Short Tests with pytest - run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov ipr --cov-report term --cov-report xml + + - name: Tests with pytest, excluding integration tests + run: pytest --junitxml=junit/test-results-3.9.xml --cov ipr --cov-report term --cov-report xml env: IPR_USER: ${{ secrets.IPR_TEST_USER }} IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }} GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }} GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }} EXCLUDE_INTEGRATION_TESTS: 1 - # EXCLUDE_INTEGRATION_TESTS: ${{ matrix.python-version != '3.11' }} - if: github.event_name != 'pull_request' \ No newline at end of file diff --git a/pori_python/graphkb/util.py b/pori_python/graphkb/util.py index a42695fa..00b2e749 100644 --- a/pori_python/graphkb/util.py +++ b/pori_python/graphkb/util.py @@ -163,11 +163,16 @@ def request(self, endpoint: str, method: str = "GET", **kwargs) -> Dict: # about catching OSError as well as ConnectionError: # https://stackoverflow.com/questions/74253820 attempts = range(15) + need_refresh_login = False for attempt in attempts: if attempt > 0: time.sleep(2) # wait between retries try: - self.refresh_login() + + if need_refresh_login: + self.refresh_login() + need_refresh_login = False + self.request_count += 1 resp = requests.request( method, url, headers=self.headers, timeout=timeout, **kwargs @@ -175,6 +180,7 @@ def request(self, endpoint: str, method: str = "GET", **kwargs) -> Dict: if resp.status_code == 401 or resp.status_code == 403: logger.debug(f"/{endpoint} - {resp.status_code} - retrying") # try to re-login if the token expired + need_refresh_login = True continue else: break diff --git a/pori_python/ipr/annotate.py b/pori_python/ipr/annotate.py index 771f35b2..90cf5ae9 100644 --- a/pori_python/ipr/annotate.py +++ b/pori_python/ipr/annotate.py @@ -26,7 +26,7 @@ Variant, ) -from .constants import TMB_HIGH_CATEGORY +from .constants import TMB_SIGNATURE, TMB_SIGNATURE_VARIANT_TYPE from .ipr import convert_statements_to_alterations from .util import convert_to_rid_set, logger @@ -354,7 +354,8 @@ def annotate_msi( def annotate_tmb( graphkb_conn: GraphKBConnection, disease_matches: List[str], - category: str = TMB_HIGH_CATEGORY, + reference: str = TMB_SIGNATURE, + type: str = TMB_SIGNATURE_VARIANT_TYPE, ) -> List[KbMatch]: """Annotate Tumour Mutation Burden (tmb) categories from GraphKB in the IPR alterations format. @@ -362,33 +363,53 @@ def annotate_tmb( Args: graphkb_conn (GraphKBConnection): the graphkb api connection object disease_matches (list.str): GraphKB disease RIDs - category: such as 'high mutation burden' + reference: CategoryVariant reference, e.g. Signature 'mutation burden' + type: CategoryVariant type, e.g. Vocabulary 'high signature' Returns: list of kbMatches records for IPR """ gkb_matches = [] - categories = graphkb_conn.query( + categoryVariants = graphkb_conn.query( { "target": { "target": "CategoryVariant", "filters": { - "reference1": { - "target": "Signature", - "filters": {"OR": [{"name": category}, {"displayName": category}]}, - } + "AND": [ + { + "reference1": { + "target": "Signature", + "filters": { + "OR": [ + {"name": reference}, + {"displayName": reference}, + # KBDEV-1246 + # Keep support for 'high mutation burden' until statement datafix + {'name': 'high mutation burden'}, + {'displayName': 'high mutation burden'}, + ] + }, + }, + }, + { + "type": { + "target": "Vocabulary", + "filters": {"OR": [{"name": type}, {"displayName": type}]}, + }, + }, + ], }, }, "queryType": "similarTo", "returnProperties": ["@rid", "displayName"], } ) - if categories: - cat_variants = [cast(Variant, var) for var in categories] + if categoryVariants: + cat_variants = [cast(Variant, var) for var in categoryVariants] for ipr_row in get_ipr_statements_from_variants( graphkb_conn, cat_variants, disease_matches ): - ipr_row["variant"] = category + ipr_row["variant"] = f'{TMB_SIGNATURE} {TMB_SIGNATURE_VARIANT_TYPE}' ipr_row["variantType"] = "tmb" gkb_matches.append(ipr_row) return gkb_matches diff --git a/pori_python/ipr/constants.py b/pori_python/ipr/constants.py index 14bca952..c1dbdf73 100644 --- a/pori_python/ipr/constants.py +++ b/pori_python/ipr/constants.py @@ -8,5 +8,8 @@ # Signatures COSMIC_SIGNATURE_VARIANT_TYPE = "high signature" HLA_SIGNATURE_VARIANT_TYPE = "signature present" -TMB_HIGH = 10.0 # genomic mutations per mb - https://www.bcgsc.ca/jira/browse/GERO-296 -TMB_HIGH_CATEGORY = "high mutation burden" +TMB_SIGNATURE = "mutation burden" +TMB_SIGNATURE_HIGH_THRESHOLD = ( + 10.0 # genomic mutations per mb - https://www.bcgsc.ca/jira/browse/GERO-296 +) +TMB_SIGNATURE_VARIANT_TYPE = "high signature" diff --git a/pori_python/ipr/main.py b/pori_python/ipr/main.py index 9817a983..26c0a9e4 100644 --- a/pori_python/ipr/main.py +++ b/pori_python/ipr/main.py @@ -31,7 +31,12 @@ annotate_tmb, ) from .connection import IprConnection -from .constants import DEFAULT_URL, TMB_HIGH, TMB_HIGH_CATEGORY +from .constants import ( + DEFAULT_URL, + TMB_SIGNATURE, + TMB_SIGNATURE_HIGH_THRESHOLD, + TMB_SIGNATURE_VARIANT_TYPE, +) from .inputs import ( check_comparators, check_variant_links, @@ -402,25 +407,25 @@ def ipr_report( except Exception as err: logger.error(f"tmburMutationBurden parsing failure: {err}") - if tmb_val >= TMB_HIGH: + if tmb_val >= TMB_SIGNATURE_HIGH_THRESHOLD: + tmb_cat = f'{TMB_SIGNATURE} {TMB_SIGNATURE_VARIANT_TYPE}' + logger.warning( - f"GERO-296 - tmburMutationBurden high -checking graphkb matches for {TMB_HIGH_CATEGORY}" + f"GERO-296 - tmburMutationBurden high -checking graphkb matches for {tmb_cat}" ) if not tmb.get("key"): - tmb["key"] = TMB_HIGH_CATEGORY + tmb["key"] = tmb_cat if not tmb.get("kbCategory"): - tmb["kbCategory"] = TMB_HIGH_CATEGORY + tmb["kbCategory"] = tmb_cat # GERO-296 - try matching to graphkb - tmb_matches = annotate_tmb(graphkb_conn, disease_matches, TMB_HIGH_CATEGORY) + tmb_matches = annotate_tmb(graphkb_conn, disease_matches) if tmb_matches: - tmb_variant["kbCategory"] = TMB_HIGH_CATEGORY # type: ignore - tmb_variant["variant"] = TMB_HIGH_CATEGORY + tmb_variant["kbCategory"] = tmb_cat + tmb_variant["variant"] = tmb_cat tmb_variant["key"] = tmb["key"] tmb_variant["variantType"] = "tmb" - logger.info( - f"GERO-296 '{TMB_HIGH_CATEGORY}' matches {len(tmb_matches)} statements." - ) + logger.info(f"GERO-296 '{tmb_cat}' matches {len(tmb_matches)} statements.") gkb_matches.extend([Hashabledict(tmb_statement) for tmb_statement in tmb_matches]) logger.debug(f"\tgkb_matches: {len(gkb_matches)}") diff --git a/z.md b/z.md new file mode 100644 index 00000000..efd9c74c --- /dev/null +++ b/z.md @@ -0,0 +1 @@ +# test 2