diff --git a/solvebio/__init__.py b/solvebio/__init__.py index 53732340..3f31722b 100644 --- a/solvebio/__init__.py +++ b/solvebio/__init__.py @@ -118,8 +118,7 @@ def emit(self, record): def login(**kwargs): - """ - Sets up the auth credentials using the provided key/token, + """Sets up the auth credentials using the provided key/token, or checks the credentials file (if no token provided). Lookup order: diff --git a/solvebio/errors.py b/solvebio/errors.py index a1cc2318..ff6aac4d 100644 --- a/solvebio/errors.py +++ b/solvebio/errors.py @@ -46,15 +46,16 @@ def __init__(self, message=None, response=None): elif response.status_code == 404: self.message = '404 Not Found ({})'.format(response.url) - # Handle other keys - for k, v in list(self.json_body.items()): - if k in ["non_field_errors", "detail"]: - self.message += '\nError: ' - else: - self.message += '\nError (%s): ' % k - - # can be a list, dict, string - self.message += str(v) + if self.json_body: + # Handle other keys + for k, v in list(self.json_body.items()): + if k in ["non_field_errors", "detail"]: + self.message += '\nError: ' + else: + self.message += '\nError (%s): ' % k + + # can be a list, dict, string + self.message += str(v) def __str__(self): return self.message diff --git a/solvebio/resource/apiresource.py b/solvebio/resource/apiresource.py index c84c13b8..b6fcfa3d 100644 --- a/solvebio/resource/apiresource.py +++ b/solvebio/resource/apiresource.py @@ -45,16 +45,10 @@ def class_name(cls): 'actions on its subclasses (e.g. Vault, Object, Dataset)') return str(quote_plus(cls.__name__)) - @classmethod - def class_url(cls): - """Returns a versioned URI string for this class""" - base = 'v{0}'.format(getattr(cls, 'RESOURCE_VERSION', '1')) - return "/{0}/{1}".format(base, class_to_api_name(cls.class_name())) - def instance_url(self): """Get instance URL by ID""" id_ = self.get(self.ID_ATTR) - base = self.class_url() + base = self.RESOURCE if id_: return '/'.join([base, six.text_type(id_)]) @@ -68,7 +62,7 @@ def instance_url(self): class ListObject(SolveObject): def all(self, **params): - """Lists all items in a class that you have access to""" + """Lists all resources that you have access to""" return self.request('get', self['url'], params=params) def create(self, **params): @@ -142,18 +136,8 @@ def retrieve(cls, **kwargs): _client = kwargs.pop('client', None) or cls._client or client return super(SingletonAPIResource, cls).retrieve(None, client=_client) - @classmethod - def class_url(cls): - """ - Returns a versioned URI string for this class, - and don't pluralize the class name. - """ - base = 'v{0}'.format(getattr(cls, 'RESOURCE_VERSION', '1')) - return "/{0}/{1}".format(base, class_to_api_name( - cls.class_name(), pluralize=False)) - def instance_url(self): - return self.class_url() + return self.RESOURCE class CreateableAPIResource(APIResource): @@ -161,8 +145,7 @@ class CreateableAPIResource(APIResource): @classmethod def create(cls, **params): _client = params.pop('client', None) or cls._client or client - url = cls.class_url() - response = _client.post(url, data=params) + response = _client.post(cls.RESOURCE, data=params) return convert_to_solve_object(response, client=_client) @@ -241,8 +224,7 @@ class ListableAPIResource(APIResource): @classmethod def all(cls, **params): _client = params.pop('client', None) or cls._client or client - url = cls.class_url() - response = _client.get(url, params) + response = _client.get(cls.RESOURCE, params) results = convert_to_solve_object(response, client=_client) # If the object has LIST_FIELDS, setup tabulate @@ -257,8 +239,7 @@ def all(cls, **params): @classmethod def _retrieve_helper(cls, model_name, field_name, error_value, **params): _client = params.pop('client', None) or cls._client or client - url = cls.class_url() - response = _client.get(url, params) + response = _client.get(cls.RESOURCE, params) results = convert_to_solve_object(response, client=_client) objects = results.data allow_multiple = params.pop('allow_multiple', None) @@ -289,8 +270,7 @@ class SearchableAPIResource(APIResource): def search(cls, query='', **params): _client = params.pop('client', None) or cls._client or client params.update({'q': query}) - url = cls.class_url() - response = _client.get(url, params) + response = _client.get(cls.RESOURCE, params) results = convert_to_solve_object(response, client=_client) # If the object has LIST_FIELDS, setup tabulate @@ -306,8 +286,14 @@ def search(cls, query='', **params): class UpdateableAPIResource(APIResource): def save(self): - self.refresh_from(self.request('patch', self.instance_url(), - data=self.serialize(self))) + # If the class is "createable" and has no ID, try to create it + if self.get(self.ID_ATTR) is None and getattr(self, 'create'): + self.refresh_from(self.request('post', self.RESOURCE, + data=self.serialize(self))) + else: + self.refresh_from(self.request('patch', self.instance_url(), + data=self.serialize(self))) + return self def serialize(self, obj): diff --git a/solvebio/resource/application.py b/solvebio/resource/application.py index e747316a..7d674dec 100644 --- a/solvebio/resource/application.py +++ b/solvebio/resource/application.py @@ -12,7 +12,7 @@ class Application(CreateableAPIResource, SearchableAPIResource, UpdateableAPIResource): ID_ATTR = 'client_id' - RESOURCE_VERSION = 2 + RESOURCE = '/v2/applications' LIST_FIELDS = ( ('client_id', 'Client ID'), diff --git a/solvebio/resource/beacon.py b/solvebio/resource/beacon.py index 9d895e6d..2fb6aa85 100644 --- a/solvebio/resource/beacon.py +++ b/solvebio/resource/beacon.py @@ -12,7 +12,7 @@ class Beacon(CreateableAPIResource, Beacons provide entity-based search endpoints for datasets. Beacons must be created within Beacon Sets. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/beacons' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/beaconset.py b/solvebio/resource/beaconset.py index f6ffba29..cbf2e29f 100644 --- a/solvebio/resource/beaconset.py +++ b/solvebio/resource/beaconset.py @@ -13,7 +13,7 @@ class BeaconSet(CreateableAPIResource, entity-based search endpoints for datasets. Beacon sets can be used to query a group of related datasets in a single API request. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/beacon_sets' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/dataset.py b/solvebio/resource/dataset.py index 51998df3..c379c904 100644 --- a/solvebio/resource/dataset.py +++ b/solvebio/resource/dataset.py @@ -24,7 +24,7 @@ class Dataset(CreateableAPIResource, Datasets are access points to data. Dataset names are unique within a vault folder. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/datasets' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/datasetcommit.py b/solvebio/resource/datasetcommit.py index 8d49d9a2..18d29f98 100644 --- a/solvebio/resource/datasetcommit.py +++ b/solvebio/resource/datasetcommit.py @@ -38,7 +38,7 @@ class DatasetCommit(CreateableAPIResource, ListableAPIResource, """ DatasetCommits represent a change made to a Dataset. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/dataset_commits' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/datasetexport.py b/solvebio/resource/datasetexport.py index fd20a166..9fd49519 100644 --- a/solvebio/resource/datasetexport.py +++ b/solvebio/resource/datasetexport.py @@ -17,7 +17,7 @@ class DatasetExport(CreateableAPIResource, ListableAPIResource, For interactive use, DatasetExport can be "followed" to watch the progression of the task. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/dataset_exports' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/datasetfield.py b/solvebio/resource/datasetfield.py index 22ab7d58..21b62ad1 100644 --- a/solvebio/resource/datasetfield.py +++ b/solvebio/resource/datasetfield.py @@ -5,6 +5,8 @@ from .apiresource import UpdateableAPIResource from .apiresource import DeletableAPIResource +from ..annotate import Expression + class DatasetField(CreateableAPIResource, ListableAPIResource, @@ -15,7 +17,7 @@ class DatasetField(CreateableAPIResource, which can be used as filters. Dataset field resources provide users with documentation about each field. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/dataset_fields' def facets(self, **params): response = self._client.get(self.facets_url, params) @@ -23,3 +25,10 @@ def facets(self, **params): def help(self): return self.facets() + + def evaluate(self): + if not self.get('expression'): + return None + + return Expression(self['expression'], client=self._client).evaluate( + data_type=self.get('data_type', 'string')) diff --git a/solvebio/resource/datasetimport.py b/solvebio/resource/datasetimport.py index 8b1e4593..ed5a3e9b 100644 --- a/solvebio/resource/datasetimport.py +++ b/solvebio/resource/datasetimport.py @@ -20,7 +20,7 @@ class DatasetImport(CreateableAPIResource, ListableAPIResource, For interactive use, DatasetImport can be "followed" to watch the progression of an import job. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/dataset_imports' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/datasetmigration.py b/solvebio/resource/datasetmigration.py index 43a26b68..2d5b4e08 100644 --- a/solvebio/resource/datasetmigration.py +++ b/solvebio/resource/datasetmigration.py @@ -17,7 +17,7 @@ class DatasetMigration(CreateableAPIResource, ListableAPIResource, For interactive use, DatasetMigration can be "followed" to watch the progression of the task. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/dataset_migrations' LIST_FIELDS = ( ('id', 'ID'), @@ -55,9 +55,8 @@ def follow(self, loop=True, sleep_seconds=Task.SLEEP_WAIT_DEFAULT): processed_records = self.metadata\ .get('progress', {})\ .get('processed_records', 0) - print("Migration '{0}' is {1}: {2}/{3} records migrated" + print("Migration '{}' is running: {}/{} records migrated" .format(self.id, - self.status, processed_records, self.documents_count)) diff --git a/solvebio/resource/datasettemplate.py b/solvebio/resource/datasettemplate.py index b5008162..9bcddcc3 100644 --- a/solvebio/resource/datasettemplate.py +++ b/solvebio/resource/datasettemplate.py @@ -3,6 +3,12 @@ from .apiresource import UpdateableAPIResource from .apiresource import DeletableAPIResource +from . import DatasetField + +import re +import inspect +from itertools import dropwhile + class DatasetTemplate(CreateableAPIResource, ListableAPIResource, UpdateableAPIResource, DeletableAPIResource): @@ -10,7 +16,7 @@ class DatasetTemplate(CreateableAPIResource, ListableAPIResource, DatasetTemplates contain the schema of a Dataset, including some properties and all the fields. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/dataset_templates' LIST_FIELDS = ( ('id', 'ID'), @@ -18,6 +24,36 @@ class DatasetTemplate(CreateableAPIResource, ListableAPIResource, ('description', 'Description'), ) + def __init__(self, *args, **kwargs): + super(DatasetTemplate, self).__init__(*args, **kwargs) + self.fields = self.__init_fields(**kwargs) + + @classmethod + def __init_fields(cls, *args, **kwargs): + fields = kwargs.get("fields") or [] + for attr in dir(cls): + if attr.startswith("__"): + continue + + func = getattr(cls, attr, None) + if getattr(func, "field", None): + fields.append(func.field) + + return fields + + @classmethod + def create(cls, **params): + params["fields"] = cls.__init_fields(**params) + return super(DatasetTemplate, cls).create(**params) + + @classmethod + def get_or_create_by_name(cls, name, **params): + objects = cls.all(name=name, **params).solve_objects() + if objects: + return objects[0] + + return cls.create(name=name, **params) + @property def import_params(self): """ @@ -31,3 +67,36 @@ def import_params(self): 'annotator_params': self.annotator_params, 'validation_params': self.validation_params } + + @staticmethod + def field(*args, **kwargs): + def get_function_body(func): + source_lines = inspect.getsourcelines(func)[0] + source_lines = dropwhile(lambda x: x.startswith('@'), source_lines) + source = ''.join(source_lines) + # Remove comments + source = re.sub(r'(?m)^ *#.*\n?', '', source) + pattern = re.compile( + r'(async\s+)?def\s+\w+\s*\(.*?\)\s*:\s*(.*)', + flags=re.S) + lines = pattern.search(source).group(2).splitlines() + if len(lines) == 1: + body = lines[0] + else: + # Dedent the code + indentation = len(lines[1]) - len(lines[1].lstrip()) + body = '\n'.join( + [lines[0]] + + [line[indentation:] for line in lines[1:]] + ) + + # Remove any return statement + return body.replace('return ', '') + + def decorator(f): + kwargs['name'] = f.__name__ + kwargs['expression'] = get_function_body(f) + f.field = DatasetField(*args, **kwargs) + return f + + return decorator diff --git a/solvebio/resource/group.py b/solvebio/resource/group.py index 6c065a58..45ef9b80 100644 --- a/solvebio/resource/group.py +++ b/solvebio/resource/group.py @@ -10,7 +10,7 @@ class Group(CreateableAPIResource, ListableAPIResource, """ A Group represents a group of users with shared permissions for a vault. """ - RESOURCE_VERSION = 1 + RESOURCE = '/v1/groups' LIST_FIELDS = ( ('id', 'ID'), @@ -47,7 +47,7 @@ def vaults(self, **params): def datasets(self, **params): from . import Object vaults = self._get_vaults(**params) - objects_url = Object.class_url() + '?object_type=dataset&' + \ + objects_url = Object.RESOURCE + '?object_type=dataset&' + \ '&'.join(['vault_id={0}'.format(v.id) for v in vaults]) response = self._client.get(objects_url, params) datasets = convert_to_solve_object(response, client=self._client) diff --git a/solvebio/resource/object.py b/solvebio/resource/object.py index c4a21691..5fc8c099 100644 --- a/solvebio/resource/object.py +++ b/solvebio/resource/object.py @@ -35,7 +35,7 @@ class Object(CreateableAPIResource, An object is a resource in a Vault. It has three possible types, though more may be added later: folder, file, and SolveBio Dataset. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/objects' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/object_copy_task.py b/solvebio/resource/object_copy_task.py index e49b9725..8b19ad56 100644 --- a/solvebio/resource/object_copy_task.py +++ b/solvebio/resource/object_copy_task.py @@ -9,7 +9,7 @@ class ObjectCopyTask(CreateableAPIResource, ListableAPIResource, UpdateableAPIResource): - RESOURCE_VERSION = 2 + RESOURCE = '/v2/object_copy_tasks' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/savedquery.py b/solvebio/resource/savedquery.py index 49c1e728..ce84fd15 100644 --- a/solvebio/resource/savedquery.py +++ b/solvebio/resource/savedquery.py @@ -14,7 +14,7 @@ class SavedQuery(CreateableAPIResource, ListableAPIResource, with ease. A dataset is said to be compatible with a saved query if it contains all the fields found in said saved query. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/saved_queries' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/solveobject.py b/solvebio/resource/solveobject.py index ac82ebe5..00560abf 100644 --- a/solvebio/resource/solveobject.py +++ b/solvebio/resource/solveobject.py @@ -30,6 +30,7 @@ def convert_to_solve_object(resp, **kwargs): class SolveObject(dict): """Base class for all SolveBio API resource objects""" ID_ATTR = 'id' + RESOURCE = None # Allows pre-setting a SolveClient _client = None @@ -45,6 +46,9 @@ def __init__(self, id=None, **params): if id: self[self.ID_ATTR] = id + for k, v in list(params.items()): + self[k] = v + def __setattr__(self, k, v): if k[0] == '_' or k in self.__dict__: return super(SolveObject, self).__setattr__(k, v) diff --git a/solvebio/resource/task.py b/solvebio/resource/task.py index 607c6d4d..aa1eb254 100644 --- a/solvebio/resource/task.py +++ b/solvebio/resource/task.py @@ -7,7 +7,7 @@ class Task(ListableAPIResource, UpdateableAPIResource): """ Tasks are operations on datasets or vaults. """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/tasks' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/user.py b/solvebio/resource/user.py index 84a734e3..d831dcc5 100644 --- a/solvebio/resource/user.py +++ b/solvebio/resource/user.py @@ -3,4 +3,4 @@ class User(SingletonAPIResource): - pass + RESOURCE = '/v1/user' diff --git a/solvebio/resource/vault.py b/solvebio/resource/vault.py index a834efb2..b4f608b9 100644 --- a/solvebio/resource/vault.py +++ b/solvebio/resource/vault.py @@ -23,7 +23,7 @@ class Vault(CreateableAPIResource, Typically, vaults contain a series of datasets that are compatible with each other (i.e. they come from the same data source or project). """ - RESOURCE_VERSION = 2 + RESOURCE = '/v2/vaults' LIST_FIELDS = ( ('id', 'ID'), diff --git a/solvebio/resource/vault_sync_task.py b/solvebio/resource/vault_sync_task.py index 8dff79cd..38325ef9 100644 --- a/solvebio/resource/vault_sync_task.py +++ b/solvebio/resource/vault_sync_task.py @@ -9,7 +9,7 @@ class VaultSyncTask(CreateableAPIResource, ListableAPIResource, UpdateableAPIResource): - RESOURCE_VERSION = 2 + RESOURCE = '/v2/vault_sync_tasks' LIST_FIELDS = ( ('id', 'ID'),