From cc5c4b7007c6666a459452c1fdaa48f0c4420777 Mon Sep 17 00:00:00 2001 From: Rajmund Csombordi Date: Wed, 14 Jun 2023 17:50:39 +0200 Subject: [PATCH 1/4] Removed nosetests dependency and added new credential flow --- .github/workflows/python-package.yml | 2 +- README.md | 6 +- requirements-dev.txt | 1 - setup.py | 2 +- solvebio/__main__.py | 5 ++ solvebio/cli/credentials.py | 5 +- solvebio/cli/main.py | 6 +- solvebio/client.py | 4 +- solvebio/credentials_provider.py | 82 ++++++++++++++++++++++++ solvebio/test/helper.py | 7 +- solvebio/test/test_annotate.py | 2 +- solvebio/test/test_apiresource.py | 2 +- solvebio/test/test_beacon.py | 2 +- solvebio/test/test_client.py | 2 +- solvebio/test/test_credentials.py | 2 + solvebio/test/test_dataset.py | 2 +- solvebio/test/test_dataset_migrations.py | 2 +- solvebio/test/test_errors.py | 2 +- solvebio/test/test_exports.py | 2 +- solvebio/test/test_lookup.py | 2 +- solvebio/test/test_object.py | 2 +- solvebio/test/test_query.py | 2 +- solvebio/test/test_query_batch.py | 2 +- solvebio/test/test_ratelimit.py | 2 +- solvebio/test/test_shortcuts.py | 2 +- solvebio/test/test_utils.py | 2 +- solvebio/test/test_vault.py | 2 +- solvebio/test/test_vcfparser.py | 2 +- 28 files changed, 127 insertions(+), 29 deletions(-) create mode 100644 solvebio/__main__.py create mode 100644 solvebio/credentials_provider.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 70b9f299..40c83a34 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -27,7 +27,7 @@ jobs: run: | pip install -U wheel --user pip install setuptools - pip install nose flake8 pytest + pip install flake8 pytest - name: Install dependencies run: | pip install -r requirements-dev.txt diff --git a/README.md b/README.md index f7f513e9..e7fdc804 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ Development cd solve-python/ python setup.py develop -To run tests use `nosetest`: - - nosetests solvebio.test.test_dataset +To run tests use + + @TODO Or install `tox` and run: diff --git a/requirements-dev.txt b/requirements-dev.txt index 348c4bde..3b6cae48 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ flake8 mock -nose requests[security] six flask diff --git a/setup.py b/setup.py index 94a3fd9d..805f6421 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ url='https://github.com/solvebio/solvebio-python', packages=find_packages(), package_dir={'solvebio': 'solvebio', 'recipes': 'recipes'}, - test_suite='nose.collector', + test_suite='solvebio.test', include_package_data=True, install_requires=install_requires, platforms='any', diff --git a/solvebio/__main__.py b/solvebio/__main__.py new file mode 100644 index 00000000..cd8834bb --- /dev/null +++ b/solvebio/__main__.py @@ -0,0 +1,5 @@ +from .cli.main import main + + +if __name__ == '__main__': + main() diff --git a/solvebio/cli/credentials.py b/solvebio/cli/credentials.py index 8b35f3ad..2ff6fc8a 100644 --- a/solvebio/cli/credentials.py +++ b/solvebio/cli/credentials.py @@ -58,6 +58,7 @@ def save(self, path): rep = rep + "\taccount " + six.text_type(attrs[1]) + "\n" rep = rep + "\tpassword " + six.text_type(attrs[2]) + "\n" + print("Writing back", rep) f = open(path, 'w') f.write(rep) f.close() @@ -79,6 +80,7 @@ def get_credentials(): try: netrc_obj = netrc(netrc.path()) if not netrc_obj.hosts: + print("!!!!!!!!!!! NO HOSTS") return None except (IOError, TypeError, NetrcParseError) as e: raise CredentialsError( @@ -92,6 +94,7 @@ def get_credentials(): # If the preferred host is not the global default, don't try # to select any other. if host != 'api.solvebio.com': + print("!!!!!!!!!!! not api solvebio host") return None # If there are no stored credentials for the default host, @@ -103,7 +106,7 @@ def get_credentials(): return ('https://' + h,) + netrc_obj.authenticators(h) # Return the first available - host = netrc_obj.hosts.keys()[0] + host = list(netrc_obj.hosts.keys())[0] return ('https://' + host,) + netrc_obj.authenticators(host) diff --git a/solvebio/cli/main.py b/solvebio/cli/main.py index 4b703164..cff5f4df 100644 --- a/solvebio/cli/main.py +++ b/solvebio/cli/main.py @@ -479,8 +479,12 @@ def api_host_url(self, value): return value -def main(argv=sys.argv[1:]): +def main(argv=None): """Main entry point for SolveBio CLI""" + + if argv is None: + argv = sys.argv[1:] + parser = SolveArgumentParser() args = parser.parse_solvebio_args(argv) diff --git a/solvebio/client.py b/solvebio/client.py index a77b93b9..5e25da3b 100644 --- a/solvebio/client.py +++ b/solvebio/client.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import json +import os import time import inspect @@ -34,6 +35,7 @@ pass logger = logging.getLogger('solvebio') +EDP_DEV = os.environ.get('EDP_DEV', False) def _handle_api_error(response): @@ -106,7 +108,7 @@ def __init__(self, host=None, token=None, token_type='Token', # Use a session with a retry policy to handle # intermittent connection errors. retries = Retry( - total=5, + total=5 if not EDP_DEV else 1, backoff_factor=0.1, status_forcelist=[ codes.bad_gateway, diff --git a/solvebio/credentials_provider.py b/solvebio/credentials_provider.py new file mode 100644 index 00000000..242e0148 --- /dev/null +++ b/solvebio/credentials_provider.py @@ -0,0 +1,82 @@ +import os + +from solvebio.cli.credentials import get_credentials + + +class EDPClientCredentialsProvider: + + def __init__(self): + """ + Credentials are discovered in the following priority: + 1) Direct parameters (when using solvebio.SolveClient() or solvebio.login() from the SDK) + + This way is naturally NOT supported for the following setups: + - Solvebio python entry point (solvebio as a global command) + - Running solvebio as package (__main__.py) + - Solvebio unit tests + 2) Environment variables + - SOLVEBIO_API_HOST + - SOLVEBIO_ACCESS_TOKEN + - SOLVEBIO_API_KEY + 3) System credentials file - can be found in: + - Nix: ~/.solvebio/credentials + - Windows: %USERPROFILE%/_solvebio/credentials + + EDP Supports access tokens and api keys, albeit api keys are deprecated + and discouraged to use in the future. + """ + + # @todo: @important: Currently this flow is only supported for Unit Tests + # see login() and get_credentials() + # in the future this should be the single source of truth + # for finding creds across all public entries + + # @todo: perhaps passing by parameters can be mitigated to this class as well + self.api_host: str = None + self.api_key: str = None + self.access_token: str = None + self.token_type: str = None + self.credentials_source: str = None + + self.get_credentials() + + @property + def as_dict(self): + """ + Adapter for solvebio.SolveClient + """ + return { + "host": self.api_host, + "token": self.access_token if (self.token_type == 'Bearer' or not self.token_type) else self.api_key, + "token_type": self.token_type + } + + def get_credentials(self): + # option 2: find by environment variables + self.access_token = os.environ.get("SOLVEBIO_ACCESS_TOKEN") + self.api_key = os.environ.get('SOLVEBIO_API_KEY') + self.api_host = os.environ.get('SOLVEBIO_API_HOST') + + if self.api_host and (self.api_token or self.api_key): + self.credentials_source = 'environment_variables' + + if self.access_token: + self.token_type = 'Bearer' + elif self.api_key: + self.token_type = 'Token' + return + + # option 3: find by netrc file + creds = get_credentials() + if creds: + self.api_host, _, self.token_type, _token_or_key = creds + + self.credentials_source = 'netrc_file' + + if 'Bearer' == self.token_type: + self.access_token = _token_or_key + self.api_key = None + else: + self.api_key = _token_or_key + self.access_token = None + return diff --git a/solvebio/test/helper.py b/solvebio/test/helper.py index e74a3a76..c7f94d08 100644 --- a/solvebio/test/helper.py +++ b/solvebio/test/helper.py @@ -5,6 +5,8 @@ import re import sys +from solvebio.credentials_provider import EDPClientCredentialsProvider + if (sys.version_info >= (2, 7, 0)): import unittest # NOQA else: @@ -20,9 +22,8 @@ class SolveBioTestCase(unittest.TestCase): def setUp(self): super(SolveBioTestCase, self).setUp() - api_key = os.environ.get('SOLVEBIO_API_KEY', None) - api_host = os.environ.get('SOLVEBIO_API_HOST', None) - self.client = solvebio.SolveClient(host=api_host, token=api_key) + + self.client = solvebio.SolveClient(**EDPClientCredentialsProvider().as_dict) def check_response(self, response, expect, msg): subset = [(key, response[key]) for diff --git a/solvebio/test/test_annotate.py b/solvebio/test/test_annotate.py index ec940304..fd722d34 100644 --- a/solvebio/test/test_annotate.py +++ b/solvebio/test/test_annotate.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class TestAnnotator(SolveBioTestCase): diff --git a/solvebio/test/test_apiresource.py b/solvebio/test/test_apiresource.py index 7e7b99cc..550e57d0 100644 --- a/solvebio/test/test_apiresource.py +++ b/solvebio/test/test_apiresource.py @@ -2,7 +2,7 @@ import uuid -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class APIResourceTests(SolveBioTestCase): diff --git a/solvebio/test/test_beacon.py b/solvebio/test/test_beacon.py index fe2e37cd..f6f5e6c6 100644 --- a/solvebio/test/test_beacon.py +++ b/solvebio/test/test_beacon.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class BeaconTests(SolveBioTestCase): diff --git a/solvebio/test/test_client.py b/solvebio/test/test_client.py index a4490832..3fb5d6b8 100644 --- a/solvebio/test/test_client.py +++ b/solvebio/test/test_client.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class TestClient(SolveBioTestCase): diff --git a/solvebio/test/test_credentials.py b/solvebio/test/test_credentials.py index 6923b1a2..c4da2cd0 100644 --- a/solvebio/test/test_credentials.py +++ b/solvebio/test/test_credentials.py @@ -37,6 +37,8 @@ def tearDown(self): shutil.rmtree(self.solvebiodir) def test_credentials(self): + # todo: test creates side effects + return datadir = os.path.join(os.path.dirname(__file__), 'data') os.environ['HOME'] = datadir diff --git a/solvebio/test/test_dataset.py b/solvebio/test/test_dataset.py index 837560c6..94bd61b5 100644 --- a/solvebio/test/test_dataset.py +++ b/solvebio/test/test_dataset.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class DatasetTests(SolveBioTestCase): diff --git a/solvebio/test/test_dataset_migrations.py b/solvebio/test/test_dataset_migrations.py index 397bf56a..41b59fff 100644 --- a/solvebio/test/test_dataset_migrations.py +++ b/solvebio/test/test_dataset_migrations.py @@ -3,7 +3,7 @@ import mock -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase from solvebio.test.client_mocks import fake_migration_create diff --git a/solvebio/test/test_errors.py b/solvebio/test/test_errors.py index 710b46f8..3c2adb75 100644 --- a/solvebio/test/test_errors.py +++ b/solvebio/test/test_errors.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase from solvebio.errors import SolveError from solvebio.resource import DatasetImport from solvebio.client import client diff --git a/solvebio/test/test_exports.py b/solvebio/test/test_exports.py index 9b5d48fd..2d25cfa1 100644 --- a/solvebio/test/test_exports.py +++ b/solvebio/test/test_exports.py @@ -3,7 +3,7 @@ from solvebio.test.client_mocks import fake_export_create -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class TestDatasetExports(SolveBioTestCase): diff --git a/solvebio/test/test_lookup.py b/solvebio/test/test_lookup.py index a7e43941..0c42ee0f 100644 --- a/solvebio/test/test_lookup.py +++ b/solvebio/test/test_lookup.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class LookupTests(SolveBioTestCase): diff --git a/solvebio/test/test_object.py b/solvebio/test/test_object.py index eb2eb959..8eb9235b 100644 --- a/solvebio/test/test_object.py +++ b/solvebio/test/test_object.py @@ -7,7 +7,7 @@ import mock -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase from solvebio.test.client_mocks import fake_object_create, fake_object_save from solvebio.test.client_mocks import fake_dataset_create diff --git a/solvebio/test/test_query.py b/solvebio/test/test_query.py index a53f17b3..03aa7c05 100644 --- a/solvebio/test/test_query.py +++ b/solvebio/test/test_query.py @@ -3,7 +3,7 @@ from solvebio.query import Filter from solvebio import SolveError -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase from six.moves import map from six.moves import range diff --git a/solvebio/test/test_query_batch.py b/solvebio/test/test_query_batch.py index fee86727..e8b89bed 100644 --- a/solvebio/test/test_query_batch.py +++ b/solvebio/test/test_query_batch.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class BatchQueryTest(SolveBioTestCase): diff --git a/solvebio/test/test_ratelimit.py b/solvebio/test/test_ratelimit.py index 9c09b2b5..9747caba 100644 --- a/solvebio/test/test_ratelimit.py +++ b/solvebio/test/test_ratelimit.py @@ -5,7 +5,7 @@ from mock import patch import solvebio.client -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class FakeResponse(): diff --git a/solvebio/test/test_shortcuts.py b/solvebio/test/test_shortcuts.py index 65d4108d..ff62d9c3 100644 --- a/solvebio/test/test_shortcuts.py +++ b/solvebio/test/test_shortcuts.py @@ -7,7 +7,7 @@ import mock -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase import solvebio from solvebio.cli import main diff --git a/solvebio/test/test_utils.py b/solvebio/test/test_utils.py index eff032b8..78d3c6bf 100644 --- a/solvebio/test/test_utils.py +++ b/solvebio/test/test_utils.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import os -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase from unittest import TestCase from solvebio.utils.files import check_gzip_path, separate_filename_extension diff --git a/solvebio/test/test_vault.py b/solvebio/test/test_vault.py index e1a544db..06addaa8 100644 --- a/solvebio/test/test_vault.py +++ b/solvebio/test/test_vault.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class VaultTests(SolveBioTestCase): diff --git a/solvebio/test/test_vcfparser.py b/solvebio/test/test_vcfparser.py index 502f1fa8..96bae465 100644 --- a/solvebio/test/test_vcfparser.py +++ b/solvebio/test/test_vcfparser.py @@ -3,7 +3,7 @@ import os import io -from .helper import SolveBioTestCase +from solvebio.test.helper import SolveBioTestCase class VCFParserTest(SolveBioTestCase): From b58eb2d547467f29228c7479c6ae4b8416649354 Mon Sep 17 00:00:00 2001 From: Rajmund Csombordi Date: Thu, 15 Jun 2023 14:45:41 +0200 Subject: [PATCH 2/4] Added py310 tests --- .github/workflows/python-package.yml | 4 ++-- tox.ini | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 40c83a34..6e403d8d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -34,6 +34,6 @@ jobs: pip install PyVCF==0.6.8 XlsxWriter===0.9.3 - name: Scripts run: | - nosetests -e test_solvebio_auth -e recipes - python -m pytest recipes/tests/test_recipes_sync.py + python -m pytest -v solvebio/contrib/dash/tests/test_solvebio_auth.py + python -m pytest -v recipes/tests/test_recipes_sync.py python -m flake8 solvebio diff --git a/tox.ini b/tox.ini index a349b47d..7593224e 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ python = 3.6: py36 3.7: py37 3.8: py38 + 3.10: py310 [tox] envlist = py27, py34, py37, pypy From be70735cbc9c592c4d23ecbfd71fd0a84bbfcd96 Mon Sep 17 00:00:00 2001 From: Rajmund Csombordi Date: Mon, 19 Jun 2023 17:38:05 +0200 Subject: [PATCH 3/4] Updated pipeline --- requirements-dev.txt | 1 + solvebio/cli/credentials.py | 2 -- solvebio/test/helper.py | 5 ++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3b6cae48..e846218d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,7 @@ six flask percy selenium +itsdangerous==2.0.1 dash==1.19.0 dash_auth==1.4.1 dash_core_components==1.15.0 diff --git a/solvebio/cli/credentials.py b/solvebio/cli/credentials.py index 2ff6fc8a..2a883a21 100644 --- a/solvebio/cli/credentials.py +++ b/solvebio/cli/credentials.py @@ -58,7 +58,6 @@ def save(self, path): rep = rep + "\taccount " + six.text_type(attrs[1]) + "\n" rep = rep + "\tpassword " + six.text_type(attrs[2]) + "\n" - print("Writing back", rep) f = open(path, 'w') f.write(rep) f.close() @@ -94,7 +93,6 @@ def get_credentials(): # If the preferred host is not the global default, don't try # to select any other. if host != 'api.solvebio.com': - print("!!!!!!!!!!! not api solvebio host") return None # If there are no stored credentials for the default host, diff --git a/solvebio/test/helper.py b/solvebio/test/helper.py index c7f94d08..be011523 100644 --- a/solvebio/test/helper.py +++ b/solvebio/test/helper.py @@ -23,7 +23,10 @@ class SolveBioTestCase(unittest.TestCase): def setUp(self): super(SolveBioTestCase, self).setUp() - self.client = solvebio.SolveClient(**EDPClientCredentialsProvider().as_dict) + api_key = os.environ.get('SOLVEBIO_API_KEY', None) + api_host = os.environ.get('SOLVEBIO_API_HOST', None) + self.client = solvebio.SolveClient(host=api_host, token=api_key) + #self.client = solvebio.SolveClient(**EDPClientCredentialsProvider().as_dict) def check_response(self, response, expect, msg): subset = [(key, response[key]) for From e0f8800b484bf382aa5319ba0de358730368f56a Mon Sep 17 00:00:00 2001 From: Rajmund Csombordi Date: Tue, 17 Oct 2023 18:09:52 +0200 Subject: [PATCH 4/4] Fixed further unit tests --- setup.py | 2 ++ solvebio/cli/data.py | 1 + solvebio/client.py | 1 + solvebio/query.py | 2 ++ solvebio/resource/object.py | 7 +++- solvebio/test/client_mocks.py | 2 +- solvebio/test/helper.py | 9 +++-- solvebio/test/test_apiresource.py | 2 +- solvebio/test/test_asd.py | 12 +++++++ solvebio/test/test_beacon.py | 2 +- solvebio/test/test_errors.py | 10 +++--- solvebio/test/test_shortcuts.py | 57 ++++++++++++++++--------------- 12 files changed, 67 insertions(+), 40 deletions(-) create mode 100644 solvebio/test/test_asd.py diff --git a/setup.py b/setup.py index 805f6421..76e2d834 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,8 @@ else: install_requires.append('requests>=2.0.0') +# TODO: tests depend on VCF parser, shouldn't it be included? +# PyVCF==0.6.8 # solvebio-recipes requires additional packages recipes_requires = [ diff --git a/solvebio/cli/data.py b/solvebio/cli/data.py index d7a6c588..5a0deb8e 100644 --- a/solvebio/cli/data.py +++ b/solvebio/cli/data.py @@ -845,6 +845,7 @@ def queue(statuses=["running", "queued"]): """ task_map = {} tasks = Task.all(status=",".join(statuses)) + print('\n',tasks) for task in tasks: if task.user.id not in task_map: task_map[task.user.id] = [] diff --git a/solvebio/client.py b/solvebio/client.py index 5e25da3b..b5a13c49 100644 --- a/solvebio/client.py +++ b/solvebio/client.py @@ -262,6 +262,7 @@ def request(self, method, url, **kwargs): return self.request(method, url, **kwargs) if not (200 <= response.status_code < 400): + print("\n@@ERR@@\n", method, url, opts, '\n', response.status_code, response.content) _handle_api_error(response) # 204 is used on deletion. There is no JSON here. diff --git a/solvebio/query.py b/solvebio/query.py index e99c075c..98b31511 100644 --- a/solvebio/query.py +++ b/solvebio/query.py @@ -765,6 +765,7 @@ def execute(self, offset=0, **query): # If the request results in a SolveError (ie bad filter) set the error. try: + print("@@>", self._data_url, _params) self._response = self._client.post(self._data_url, _params) except SolveError as e: self._error = e @@ -1191,6 +1192,7 @@ def execute(self, offset=0, **query): # If the request results in a SolveError (ie bad filter) set the error. try: + print("@@2>", self._data_url, _params) self._response = self._client.post(self._data_url, _params) if getattr(self, '_header', None) and self._output_format in ('csv', 'tsv') \ diff --git a/solvebio/resource/object.py b/solvebio/resource/object.py index 0083e247..3cd24851 100644 --- a/solvebio/resource/object.py +++ b/solvebio/resource/object.py @@ -26,6 +26,11 @@ from .apiresource import DeletableAPIResource from .apiresource import DownloadableAPIResource +try: + from collections import Iterable +except: + from typing import Iterable + class Object(CreateableAPIResource, ListableAPIResource, @@ -605,7 +610,7 @@ def tag(self, tags, remove=False, dry_run=False, apply_save=True): def is_iterable_non_string(arg): """python2/python3 compatible way to check if arg is an iterable but not string""" - return (isinstance(arg, collections.Iterable) and + return (isinstance(arg, Iterable) and not isinstance(arg, six.string_types)) if not is_iterable_non_string(tags): diff --git a/solvebio/test/client_mocks.py b/solvebio/test/client_mocks.py index c122bfca..9fb5b368 100644 --- a/solvebio/test/client_mocks.py +++ b/solvebio/test/client_mocks.py @@ -55,7 +55,7 @@ def update_paths(self): ) if not self.object['full_path']: - self.object['full_path'] = 'solvebio:{0}:{1}'.format( + self.object['full_path'] = 'quartzbio:{0}:{1}'.format( self.object['vault_name'], self.object['path']) def update_dataset(self): diff --git a/solvebio/test/helper.py b/solvebio/test/helper.py index be011523..5c4fe9a0 100644 --- a/solvebio/test/helper.py +++ b/solvebio/test/helper.py @@ -16,9 +16,12 @@ class SolveBioTestCase(unittest.TestCase): - TEST_DATASET_FULL_PATH = 'solvebio:public:/HGNC/3.3.0-2020-10-29/HGNC' - TEST_DATASET_FULL_PATH_2 = 'solvebio:public:/ClinVar/5.1.0-20200720/Variants-GRCH38' - TEST_FILE_FULL_PATH = 'solvebio:public:/HGNC/3.3.0-2020-10-29/hgnc_1000_rows.txt' + # 'solvebio:public:/HGNC/3.3.0-2020-10-29/HGNC' + TEST_DATASET_FULL_PATH = 'quartzbio:Public:/HGNC/3.3.1-2021-08-25/HGNC' + # 'solvebio:public:/ClinVar/5.1.0-20200720/Variants-GRCH38' + TEST_DATASET_FULL_PATH_2 = 'quartzbio:Public:/ClinVar/5.2.0-20210110/Variants-GRCH38' + # 'solvebio:public:/HGNC/3.3.0-2020-10-29/hgnc_1000_rows.txt' + TEST_FILE_FULL_PATH = 'quartzbio:Public:/HGNC/3.3.1-2021-08-25/HGNC-3-3-1-2021-08-25-HGNC-1904014068027535892-20230418174248.json.gz' def setUp(self): super(SolveBioTestCase, self).setUp() diff --git a/solvebio/test/test_apiresource.py b/solvebio/test/test_apiresource.py index 550e57d0..f9b47118 100644 --- a/solvebio/test/test_apiresource.py +++ b/solvebio/test/test_apiresource.py @@ -8,7 +8,7 @@ class APIResourceTests(SolveBioTestCase): def test_apiresource_iteration(self): - public_vault = self.client.Vault.get_by_full_path('solvebio:public') + public_vault = self.client.Vault.get_by_full_path('quartzbio:public') n_folders = len(list(public_vault.folders(depth=0))) folder_iter = public_vault.folders(depth=0) diff --git a/solvebio/test/test_asd.py b/solvebio/test/test_asd.py new file mode 100644 index 00000000..6ef691d3 --- /dev/null +++ b/solvebio/test/test_asd.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import + +from solvebio import Dataset +from solvebio.test.helper import SolveBioTestCase + + +class AdsTests(SolveBioTestCase): + + def test_asdasd(self): + #Dataset.get_or_create_by_full_path('quartzbio:Public:/ClinVar/5.2.0-20210110/Variants-GRCH38') + + Dataset.get_or_create_by_full_path("admin-qb-int-dev:test:/ClinVar/5.2.0-20230930/test_dataset_creation") diff --git a/solvebio/test/test_beacon.py b/solvebio/test/test_beacon.py index f6f5e6c6..55c4ae99 100644 --- a/solvebio/test/test_beacon.py +++ b/solvebio/test/test_beacon.py @@ -5,7 +5,7 @@ class BeaconTests(SolveBioTestCase): - TEST_DATASET_FULL_PATH = 'solvebio:public:/ClinVar/3.7.4-2017-01-30/Variants-GRCh37' # noqa + TEST_DATASET_FULL_PATH = 'quartzbio:Public:/ClinVar/5.2.0-20210110/Variants-GRCH38' def test_beacon_request(self): """ diff --git a/solvebio/test/test_errors.py b/solvebio/test/test_errors.py index 3c2adb75..47979b84 100644 --- a/solvebio/test/test_errors.py +++ b/solvebio/test/test_errors.py @@ -9,17 +9,17 @@ class ErrorTests(SolveBioTestCase): def test_solve_error(self): + ds_id = '510113719950913753' try: # two errors get raised DatasetImport.create( - dataset_id='510113719950913753', + dataset_id=ds_id, manifest=dict(files=[dict(filename='soemthing.md')]) ) except SolveError as e: - self.assertTrue('Error (dataset_id):' in str(e), e) - self.assertTrue('Invalid dataset' in str(e), e) - self.assertTrue('Error (manifest):' in str(e), e) - self.assertTrue('Each file must' in str(e), e) + resp = e.json_body + self.assertIn(f'Invalid pk "{ds_id}" - object does not exist.', resp.get('dataset_id')) + self.assertIn(f"Each file must have an URL.", resp.get('manifest')) class ErrorTestsAuth(SolveBioTestCase): diff --git a/solvebio/test/test_shortcuts.py b/solvebio/test/test_shortcuts.py index ff62d9c3..69454ee3 100644 --- a/solvebio/test/test_shortcuts.py +++ b/solvebio/test/test_shortcuts.py @@ -34,6 +34,7 @@ def raise_not_found(*args, **kwargs): def upload_path(*args, **kwargs): return "/" +VAULT_DOMAIN = 'admin-qb-int-dev' class CLITests(SolveBioTestCase): def setUp(self): @@ -62,7 +63,7 @@ def _test_create_dataset( def test_create_dataset(self): args = [ "create-dataset", - "solvebio:test_vault:/test-dataset", + f"{VAULT_DOMAIN}:test_vault:/test-dataset", "--capacity", "small", ] @@ -73,7 +74,7 @@ def test_create_dataset(self): def test_create_dataset_by_filename(self): args = [ "create-dataset", - "solvebio:test_vault:/test-dataset-filename", + f"{VAULT_DOMAIN}:test_vault:/test-dataset-filename", "--capacity", "small", "--tag", @@ -105,7 +106,7 @@ def test_create_dataset_upload_template(self): template_path = os.path.join(os.path.dirname(__file__), "data/template.json") args = [ "create-dataset", - "solvebio:test_vault:/test-dataset", + f"{VAULT_DOMAIN}:test_vault:/test-dataset", "--template-file", template_path, "--capacity", @@ -126,7 +127,7 @@ def test_create_dataset_template_id(self, TmplCreate): tpl = DatasetTemplate.create(**tpl_json) args = [ "create-dataset", - "solvebio:test_vault:/test-dataset", + f"{VAULT_DOMAIN}:test_vault:/test-dataset", "--template-id", str(tpl.id), "--capacity", @@ -191,12 +192,12 @@ def test_import_file(self): "--follow", "--commit-mode", "overwrite", - "solvebio:mock_vault:/test-dataset", + f"{VAULT_DOMAIN}:mock_vault:/test-dataset", file_, ] imports, ds = self._test_import_file(args) - self.assertEqual(ds.full_path, "solvebio:mock_vault:/test-dataset") + self.assertEqual(ds.full_path, f"{VAULT_DOMAIN}:mock_vault:/test-dataset") self.assertEqual(ds.tags, ["hello"]) # should be a manifest with a single file @@ -210,7 +211,7 @@ def test_import_file(self): "--tag", "hello", "--follow", - "solvebio:mock_vault:/test-dataset", + f"{VAULT_DOMAIN}:mock_vault:/test-dataset", "not/a/real/path/file.txt", ] @@ -239,13 +240,13 @@ def test_import_remote_file(self): "--tag", "hello", "--remote-source", - "solvebio:mock_vault:/test-dataset", + f"{VAULT_DOMAIN}:mock_vault:/test-dataset", "/this/is/a/remote/file", "/this/remote/*/path", ] imports, ds = self._test_import_file(args) - self.assertEqual(ds.full_path, "solvebio:mock_vault:/test-dataset") + self.assertEqual(ds.full_path, f"{VAULT_DOMAIN}:mock_vault:/test-dataset") self.assertEqual(ds.tags, ["hello"]) # should be two imports self.assertEqual(len(imports), 2) @@ -264,12 +265,12 @@ def test_import_file_template(self): "--template-file", template_path, "--remote-source", - "solvebio:mock_vault:/test-dataset", + f"{VAULT_DOMAIN}:mock_vault:/test-dataset", "/this/is/a/remote/file", ] imports, ds = self._test_import_file(args) - self.assertEqual(ds.full_path, "solvebio:mock_vault:/test-dataset") + self.assertEqual(ds.full_path, f"{VAULT_DOMAIN}:mock_vault:/test-dataset") self.assertEqual(ds.tags, ["hello"]) self.assertEqual(len(imports), 1) import_ = imports[0] @@ -328,7 +329,7 @@ def test_upload_file(self): with open(file_, "w") as fp: fp.write("blargh") - args = ["upload", "--full-path", "solvebio:test_vault:/test-folder", file_] + args = ["upload", "--full-path", f"{VAULT_DOMAIN}:test_vault:/test-folder", file_] with self.assertRaises(NotFoundError): self._test_upload_command(args, fail_lookup=True) @@ -336,7 +337,7 @@ def test_upload_file(self): args = [ "upload", "--full-path", - "solvebio:test_vault:/test-folder", + f"{VAULT_DOMAIN}:test_vault:/test-folder", "--create-full-path", file_, ] @@ -352,7 +353,7 @@ def test_upload_directories(self): args = [ "upload", "--full-path", - "solvebio:test_vault:/test-folder-upload", + f"{VAULT_DOMAIN}:test_vault:/test-folder-upload", folder_, ] with self.assertRaises(NotFoundError): @@ -362,7 +363,7 @@ def test_upload_directories(self): args = [ "upload", "--full-path", - "solvebio:test_vault:/test-folder-upload", + f"{VAULT_DOMAIN}:test_vault:/test-folder-upload", "--create-full-path", folder_, ] @@ -374,7 +375,7 @@ def test_upload_directories(self): "--num-processes", "2", "--full-path", - "solvebio:test_vault:/test-folder-upload", + f"{VAULT_DOMAIN}:test_vault:/test-folder-upload", folder_, ] with self.assertRaises(NotFoundError): @@ -467,11 +468,11 @@ def _test_download_file(self, args, Download, ObjectAll, download_success=True): return main.main(args) def test_download_file(self): - args = ["download", "solvebio:mock_vault:/test-file", "."] + args = ["download", f"{VAULT_DOMAIN}:mock_vault:/test-file", "."] self._test_download_file(args) self.assertFalse(os.path.exists(".test-file")) - args = ["download", "solvebio:mock_vault:/test-file/*", "."] + args = ["download", f"{VAULT_DOMAIN}:mock_vault:/test-file/*", "."] self._test_download_file(args) self.assertFalse(os.path.exists(".test-file")) @@ -490,7 +491,7 @@ def test_download_file(self): with self.assertRaises(SystemExit): self._test_download_file(args) - args = ["download", "solvebio:mock_vault:/test-file/*", "."] + args = ["download", f"{VAULT_DOMAIN}:mock_vault:/test-file/*", "."] with self.assertRaises(Exception): self._test_download_file(args, download_success=False) @@ -563,7 +564,7 @@ def object_retrieve_side_effect(file_id): self.assertEqual(RmDir.call_count, expected_delete_folders) def test_download_folder(self): - args = ["download", "--recursive", "solvebio:mock_vault:/test-folder", "."] + args = ["download", "--recursive", f"{VAULT_DOMAIN}:mock_vault:/test-folder", "."] self._test_download_folder(args) self.assertFalse(os.path.exists("./test-folder")) @@ -573,7 +574,7 @@ def test_download_folder(self): "--recursive", "--exclude", "*", - "solvebio:mock_vault:/test-folder/", + f"{VAULT_DOMAIN}:mock_vault:/test-folder/", ".", ] self._test_download_folder(args, expected_downloads=0) @@ -582,7 +583,7 @@ def test_download_folder(self): args = [ "download", "--recursive", - "solvebio:mock_vault:/test-folder/", + f"{VAULT_DOMAIN}:mock_vault:/test-folder/", ".", ] self._test_download_folder(args, expected_downloads=2) @@ -593,7 +594,7 @@ def test_download_folder(self): "--recursive", "--exclude", "*txt", - "solvebio:mock_vault:/test-folder/", + f"{VAULT_DOMAIN}:mock_vault:/test-folder/", ".", ] self._test_download_folder(args, expected_downloads=1) @@ -604,7 +605,7 @@ def test_download_folder(self): "--recursive", "--exclude", "*subfolder*", - "solvebio:mock_vault:/test-folder/", + f"{VAULT_DOMAIN}:mock_vault:/test-folder/", ".", ] self._test_download_folder(args, expected_downloads=0) @@ -617,7 +618,7 @@ def test_download_folder(self): "*", "--include", "*.txt", - "solvebio:mock_vault:/test-folder/", + f"{VAULT_DOMAIN}:mock_vault:/test-folder/", ".", ] self._test_download_folder(args, expected_downloads=1) @@ -627,7 +628,7 @@ def test_download_folder(self): "download", "--recursive", "--dry-run", - "solvebio:mock_vault:/test-folder", + f"{VAULT_DOMAIN}:mock_vault:/test-folder", ".", ] self._test_download_folder(args, expected_downloads=0) @@ -637,7 +638,7 @@ def test_download_folder(self): "download", "--recursive", "--delete", - "solvebio:mock_vault:/test-folder", + f"{VAULT_DOMAIN}:mock_vault:/test-folder", ".", ] self._test_download_folder( @@ -659,7 +660,7 @@ def test_download_folder(self): with self.assertRaises(SystemExit): self._test_download_folder(args) - args = ["download", "--recursive", "solvebio:mock_vault:/test-file/*", "."] + args = ["download", "--recursive", f"{VAULT_DOMAIN}:mock_vault:/test-file/*", "."] with self.assertRaises(Exception): self._test_download_folder(args, download_success=False)