Task/kbdev 1300 add GitHub action 2

.github/workflows/pytest.yml → .github/workflows/extended-tests-integration.yml

@@ -1,43 +1,58 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# Extended testing, INCLUDDING integration tests
+# Python:
+#   - 3.9
+#   - 3.10
+#   - 3.11
+#   - 3.12
+# Triggered by:
+#   - Creating a new PR to main branch
+#   - Addind commit(s) to an existing PR to main branch
+#   - Addind commit(s) to main branch:
+#     - Direct commit without PR
+#     - Updating the out-of-date base branch to an existing PR to main branch
 
-name: build
+name: extended-tests-integration
 
 on:
   push:
-    branches:
-      - main
-      - develop
+    branches: [main]
   pull_request:
+    branches: [main]
+    types: [opened, synchronize]
 
 jobs:
-  build:
 
+  extended-tests-integration:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ['3.9', '3.10', '3.11', '3.12']
-
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout repository
+    - uses: actions/checkout@v4
+
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools
         pip install -e .[test]  # coverage reports need -e to capture properly
+
     - name: Lint with flake8
       run: |
         pip install flake8
         # stop the build if there are Python syntax errors or undefined names
         flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics
+
     - name: Check with black
       run: |
         pip install black
         black --check -S -l 100 pori_python tests
-    - name: Full Tests with pytest
+
+    - name: Tests with pytest, including integration tests
       run: |
         pip list
         pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov pori_python --cov-report term --cov-report xml
@@ -46,23 +61,23 @@ jobs:
         IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }}
         GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }}
         GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }}
-        # SDEV-3381 - Turn off integration tests temporarily, till efficiency is increased
-        # turn on integration tests for one python version only
-        EXCLUDE_INTEGRATION_TESTS: ${{ matrix.python-version != '3.11' }}
+        EXCLUDE_INTEGRATION_TESTS: 0
+
     - name: Upload pytest test results
-      uses: actions/upload-artifact@master
+      if: ${{ matrix.python-version == '3.9' }}
+      uses: actions/upload-artifact@v4
       with:
         name: pytest-results-${{ matrix.python-version }}
         path: junit/test-results-${{ matrix.python-version }}.xml
         # Use always() to always run this step to publish test results when there are test failures
-      if: matrix.python-version == 3.9
+
     - name: Update code coverage report to CodeCov
-      uses: codecov/codecov-action@v3
+      if: ${{ matrix.python-version == '3.9' }}
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./coverage.xml
         flags: unittests
         env_vars: OS,PYTHON
         name: codecov-umbrella
         fail_ci_if_error: true
-      if: matrix.python-version == 3.9

.github/workflows/quick-tests-integration.yml

@@ -0,0 +1,57 @@
+# Limited testing, INCLUDING integration tests
+# Python:
+#   - 3.9
+# Triggered by:
+#   - Creating a new PR to develop branch
+#   - Addind commit(s) to an existing PR to develop branch
+#   - Addind commit(s) to develop branch:
+#     - Direct commit without PR
+#     - Updating the out-of-date base branch to an existing PR to develop branch
+
+name: quick-tests-integration
+
+on:
+  push:
+    branches: [develop]
+  pull_request:
+    branches: [develop]
+    types: [opened, synchronize]
+  workflow_dispatch:
+
+jobs:
+
+  quick-tests-integration:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.9"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools
+        pip install -e .[test]  # coverage reports need -e to capture properly
+
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics
+
+    - name: Check with black
+      run: |
+        pip install black
+        black --check -S -l 100 pori_python tests
+
+    - name: Tests with pytest, including integration tests
+      run: pytest --junitxml=junit/test-results-3.9.xml --cov ipr --cov-report term --cov-report xml
+      env:
+        IPR_USER: ${{ secrets.IPR_TEST_USER }}
+        IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }}
+        GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }}
+        GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }}
+        EXCLUDE_INTEGRATION_TESTS: 0

.github/workflows/quick-pytest.yml → .github/workflows/quick-tests.yml

@@ -1,46 +1,57 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# Limited testing, EXCLUDING integration tests
+# Python:
+#   - 3.9
+# Triggered by:
+#   - Pushing a new branch
+#   - Addind commit(s) to a branch that is not main or develop, and that is not part of a PR to develop or main branch
 
 name: quick-tests
 
 on:
   push:
+    branches-ignore: [main, develop]
+  workflow_dispatch:
 
 jobs:
-  build:
 
+  quick-tests:
+    if: ${{ !(
+      github.event.pull_request && (
+        github.event.pull_request.base.ref == 'main' ||
+        github.event.pull_request.base.ref == 'develop'
+      )
+    ) }}
     runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: ['3.11']
-
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.9"
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools
         pip install -e .[test]  # coverage reports need -e to capture properly
+
     - name: Lint with flake8
       run: |
         pip install flake8
         # stop the build if there are Python syntax errors or undefined names
         flake8 pori_python --count --select=E9,F63,F7,F82 --show-source --statistics
+
     - name: Check with black
       run: |
         pip install black
         black --check -S -l 100 pori_python tests
-    - name: Short Tests with pytest
-      run: pytest --junitxml=junit/test-results-${{ matrix.python-version }}.xml --cov ipr --cov-report term --cov-report xml
+
+    - name: Tests with pytest, excluding integration tests
+      run: pytest --junitxml=junit/test-results-3.9.xml --cov ipr --cov-report term --cov-report xml
       env:
         IPR_USER: ${{ secrets.IPR_TEST_USER }}
         IPR_PASS: ${{ secrets.IPR_TEST_PASSWORD }}
         GRAPHKB_USER: ${{ secrets.GKB_TEST_USER }}
         GRAPHKB_PASS: ${{ secrets.GKB_TEST_PASS }}
         EXCLUDE_INTEGRATION_TESTS: 1
-        # EXCLUDE_INTEGRATION_TESTS: ${{ matrix.python-version != '3.11' }}
-      if: github.event_name != 'pull_request'

pori_python/graphkb/util.py

@@ -163,18 +163,24 @@ def request(self, endpoint: str, method: str = "GET", **kwargs) -> Dict:
         # about catching OSError as well as ConnectionError:
         # https://stackoverflow.com/questions/74253820
         attempts = range(15)
+        need_refresh_login = False
         for attempt in attempts:
             if attempt > 0:
                 time.sleep(2)  # wait between retries
             try:
-                self.refresh_login()
+
+                if need_refresh_login:
+                    self.refresh_login()
+                    need_refresh_login = False
+
                 self.request_count += 1
                 resp = requests.request(
                     method, url, headers=self.headers, timeout=timeout, **kwargs
                 )
                 if resp.status_code == 401 or resp.status_code == 403:
                     logger.debug(f"/{endpoint} - {resp.status_code} - retrying")
                     # try to re-login if the token expired
+                    need_refresh_login = True
                     continue
                 else:
                     break

pori_python/ipr/annotate.py

@@ -26,7 +26,7 @@
     Variant,
 )
 
-from .constants import TMB_HIGH_CATEGORY
+from .constants import TMB_SIGNATURE, TMB_SIGNATURE_VARIANT_TYPE
 from .ipr import convert_statements_to_alterations
 from .util import convert_to_rid_set, logger
 
@@ -354,41 +354,62 @@ def annotate_msi(
 def annotate_tmb(
     graphkb_conn: GraphKBConnection,
     disease_matches: List[str],
-    category: str = TMB_HIGH_CATEGORY,
+    reference: str = TMB_SIGNATURE,
+    type: str = TMB_SIGNATURE_VARIANT_TYPE,
 ) -> List[KbMatch]:
     """Annotate Tumour Mutation Burden (tmb) categories from GraphKB in the IPR alterations format.
 
     Match to GraphKb Category variants with similar names
     Args:
         graphkb_conn (GraphKBConnection): the graphkb api connection object
         disease_matches (list.str): GraphKB disease RIDs
-        category: such as 'high mutation burden'
+        reference: CategoryVariant reference, e.g. Signature 'mutation burden'
+        type: CategoryVariant type, e.g. Vocabulary 'high signature'
 
     Returns:
         list of kbMatches records for IPR
     """
     gkb_matches = []
-    categories = graphkb_conn.query(
+    categoryVariants = graphkb_conn.query(
         {
             "target": {
                 "target": "CategoryVariant",
                 "filters": {
-                    "reference1": {
-                        "target": "Signature",
-                        "filters": {"OR": [{"name": category}, {"displayName": category}]},
-                    }
+                    "AND": [
+                        {
+                            "reference1": {
+                                "target": "Signature",
+                                "filters": {
+                                    "OR": [
+                                        {"name": reference},
+                                        {"displayName": reference},
+                                        # KBDEV-1246
+                                        # Keep support for 'high mutation burden' until statement datafix
+                                        {'name': 'high mutation burden'},
+                                        {'displayName': 'high mutation burden'},
+                                    ]
+                                },
+                            },
+                        },
+                        {
+                            "type": {
+                                "target": "Vocabulary",
+                                "filters": {"OR": [{"name": type}, {"displayName": type}]},
+                            },
+                        },
+                    ],
                 },
             },
             "queryType": "similarTo",
             "returnProperties": ["@rid", "displayName"],
         }
     )
-    if categories:
-        cat_variants = [cast(Variant, var) for var in categories]
+    if categoryVariants:
+        cat_variants = [cast(Variant, var) for var in categoryVariants]
         for ipr_row in get_ipr_statements_from_variants(
             graphkb_conn, cat_variants, disease_matches
         ):
-            ipr_row["variant"] = category
+            ipr_row["variant"] = f'{TMB_SIGNATURE} {TMB_SIGNATURE_VARIANT_TYPE}'
             ipr_row["variantType"] = "tmb"
             gkb_matches.append(ipr_row)
     return gkb_matches

pori_python/ipr/constants.py

@@ -8,5 +8,8 @@
 # Signatures
 COSMIC_SIGNATURE_VARIANT_TYPE = "high signature"
 HLA_SIGNATURE_VARIANT_TYPE = "signature present"
-TMB_HIGH = 10.0  # genomic mutations per mb - https://www.bcgsc.ca/jira/browse/GERO-296
-TMB_HIGH_CATEGORY = "high mutation burden"
+TMB_SIGNATURE = "mutation burden"
+TMB_SIGNATURE_HIGH_THRESHOLD = (
+    10.0  # genomic mutations per mb - https://www.bcgsc.ca/jira/browse/GERO-296
+)
+TMB_SIGNATURE_VARIANT_TYPE = "high signature"