Merge pull request #53 from scrapinghub/sc1467-1-fixes

Minor improvements for new python-client

Author: chekunkov

README.rst

@@ -100,6 +100,29 @@ Project instance also has the following fields:
 - spiders - access to spiders collection (see ``Spiders`` section)
 
 
+Settings
+--------
+
+To get a list of the project settings::
+
+    >>> project.settings.list()
+    [(u'default_job_units', 2), (u'job_runtime_limit', 24)]]
+
+To get a project setting value by name::
+
+    >>> project.settings.get('job_runtime_limit')
+    24
+
+To update a project setting value by name::
+
+    >>> project.settings.set('job_runtime_limit', 20)
+
+Or update a few project settings at once::
+
+    >>> project.settings.update({'default_job_units': 1,
+    ...                          'job_runtime_limit': 20})
+
+
 Spiders
 -------
 
@@ -181,7 +204,7 @@ count
 
 It's also possible to count jobs for a given project/spider::
 
-    >> spider.jobs.count()
+    >>> spider.jobs.count()
     5
 
 Count logic supports different filters, as described for `count endpoint`_.
@@ -314,9 +337,9 @@ Metadata
 
 Job details can be found in jobs metadata and it's scrapystats::
 
-    >>> job.metadata['version']
+    >>> job.metadata.get('version')
     '5123a86-master'
-    >>> job.metadata['scrapystats']
+    >>> job.metadata.get('scrapystats')
     ...
     'downloader/response_count': 104,
     'downloader/response_status_count/200': 104,
@@ -332,7 +355,7 @@ Job details can be found in jobs metadata and it's scrapystats::
 
 Anything can be stored in metadata, here is example how to add tags::
 
-    >>> job.update_metadata({'tags': 'obsolete'})
+    >>> job.metadata.set('tags', ['obsolete'])
 
 Items
 ^^^^^

scrapinghub/__init__.py

@@ -1,5 +1,7 @@
-__all__ = ["APIError", "Connection", "HubstorageClient", "ScrapinghubClient"]
-
+__all__ = ["APIError", "Connection", "HubstorageClient",
+           "ScrapinghubClient", "ScrapinghubAPIError",
+           "DuplicateJobError", "BadRequest", "NotFound",
+           "Unauthorized", "ValueTooLarge"]
 
 import pkgutil
 __version__ = pkgutil.get_data(__package__, 'VERSION')
@@ -10,3 +12,11 @@
 from .legacy import *
 from .hubstorage import HubstorageClient
 from .client import ScrapinghubClient
+from .client.exceptions import (
+    ScrapinghubAPIError,
+    DuplicateJobError,
+    BadRequest,
+    NotFound,
+    Unauthorized,
+    ValueTooLarge,
+)

scrapinghub/client/exceptions.py

@@ -32,7 +32,7 @@ def __init__(self, message=None, http_error=None):
         super(ScrapinghubAPIError, self).__init__(message)
 
 
-class InvalidUsage(ScrapinghubAPIError):
+class BadRequest(ScrapinghubAPIError):
     pass
 
 
@@ -60,7 +60,7 @@ def wrapped(*args, **kwargs):
         except HTTPError as exc:
             status_code = exc.response.status_code
             if status_code == 400:
-                raise InvalidUsage(http_error=exc)
+                raise BadRequest(http_error=exc)
             elif status_code == 401:
                 raise Unauthorized(http_error=exc)
             elif status_code == 404:
@@ -76,8 +76,8 @@ def wrapped(*args, **kwargs):
                 raise NotFound(msg)
             elif exc._type == APIError.ERR_VALUE_ERROR:
                 raise ValueError(msg)
-            elif exc._type == APIError.ERR_INVALID_USAGE:
-                raise InvalidUsage(msg)
+            elif exc._type == APIError.ERR_BAD_REQUEST:
+                raise BadRequest(msg)
             elif exc._type == APIError.ERR_AUTH_ERROR:
                 raise Unauthorized(http_error=exc)
             raise ScrapinghubAPIError(msg)

scrapinghub/client/jobs.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import json
 
-from ..hubstorage.job import JobMeta
+from ..hubstorage.job import JobMeta as _JobMeta
 from ..hubstorage.job import Items as _Items
 from ..hubstorage.job import Logs as _Logs
 from ..hubstorage.job import Samples as _Samples
@@ -11,9 +11,8 @@
 from .logs import Logs
 from .requests import Requests
 from .samples import Samples
-from .exceptions import NotFound, InvalidUsage, DuplicateJobError
-from .utils import get_tags_for_update
-from .utils import parse_job_key
+from .exceptions import NotFound, BadRequest, DuplicateJobError
+from .utils import _MappingProxy, get_tags_for_update, parse_job_key
 
 
 class Jobs(object):
@@ -150,7 +149,7 @@ def schedule(self, spidername=None, **params):
         try:
             response = self._client._connection._post(
                 'schedule', 'json', params)
-        except InvalidUsage as exc:
+        except BadRequest as exc:
             if 'already scheduled' in str(exc):
                 raise DuplicateJobError(exc)
             raise
@@ -302,10 +301,10 @@ class Job(object):
         >>> job = project.job('123/1/2')
         >>> job.key
         '123/1/2'
-        >>> job.metadata['state']
+        >>> job.metadata.get('state')
         'finished'
     """
-    def __init__(self, client, jobkey, metadata=None):
+    def __init__(self, client, jobkey):
         self.projectid = parse_job_key(jobkey).projectid
         self.key = jobkey
 
@@ -319,24 +318,7 @@ def __init__(self, client, jobkey, metadata=None):
         self.requests = Requests(_Requests, client, jobkey)
         self.samples = Samples(_Samples, client, jobkey)
 
-        self.metadata = JobMeta(client._hsclient, jobkey, cached=metadata)
-
-    def update_metadata(self, *args, **kwargs):
-        """Update job metadata.
-
-        :param \*\*kwargs: keyword arguments representing job metadata
-
-        Usage:
-
-        - update job outcome::
-
-            >>> job.update_metadata(close_reason='custom reason')
-
-        - change job tags::
-
-            >>> job.update_metadata({'tags': 'obsolete'})
-        """
-        self._job.update_metadata(*args, **kwargs)
+        self.metadata = JobMeta(_JobMeta, client, jobkey)
 
     def update_tags(self, add=None, remove=None):
         """Partially update job tags.
@@ -426,19 +408,49 @@ def cancel(self):
         Usage::
 
             >>> job.cancel()
-            >>> job.metadata['cancelled_by']
+            >>> job.metadata.get('cancelled_by')
             'John'
         """
         self._project.jobq.request_cancel(self)
 
-    def purge(self):
-        """Delete job and expire its local metadata.
 
-        Usage::
+class JobMeta(_MappingProxy):
+    """Class representing job metadata.
 
-            >>> job.purge()
-            >>> job.metadata['state']
-            'deleted'
-        """
-        self.delete()
-        self.metadata.expire()
+    Not a public constructor: use :class:`Job` instance to get a
+    :class:`Jobmeta` instance. See :attr:`Job.metadata` attribute.
+
+    Usage::
+
+    - get job metadata instance
+
+        >>> job.metadata
+        <scrapinghub.client.jobs.JobMeta at 0x10494f198>
+
+    - iterate through job metadata
+
+        >>> job.metadata.iter()
+        <dict_itemiterator at 0x104adbd18>
+
+    - list job metadata
+
+        >>> job.metadata.list()
+        [('project', 123), ('units', 1), ('state', 'finished'), ...]
+
+    - get meta field value by name
+
+        >>> job.metadata.get('version')
+        'test'
+
+    - update job meta field value (some meta fields are read-only)
+
+        >>> job.metadata.set('my-meta', 'test')
+
+    - update multiple meta fields at once
+
+        >>> job.metadata.update({'my-meta1': 'test1', 'my-meta2': 'test2})
+
+    - delete meta field by name
+
+        >>> job.metadata.delete('my-meta')
+    """

scrapinghub/client/projects.py

@@ -1,15 +1,17 @@
 from __future__ import absolute_import
 
+import six
+
 from ..hubstorage.activity import Activity as _Activity
 from ..hubstorage.collectionsrt import Collections as _Collections
-from ..hubstorage.project import Settings
+from ..hubstorage.project import Settings as _Settings
 
 from .activity import Activity
 from .collections import Collections
 from .frontiers import _HSFrontier, Frontiers
 from .jobs import Jobs
 from .spiders import Spiders
-from .utils import parse_project_id
+from .utils import _MappingProxy, parse_project_id
 
 
 class Projects(object):
@@ -121,4 +123,53 @@ def __init__(self, client, projectid):
         self.activity = Activity(_Activity, client, projectid)
         self.collections = Collections(_Collections, client, projectid)
         self.frontiers = Frontiers(_HSFrontier, client, projectid)
-        self.settings = Settings(client._hsclient, projectid)
+        self.settings = Settings(_Settings, client, projectid)
+
+
+class Settings(_MappingProxy):
+    """Class representing job metadata.
+
+    Not a public constructor: use :class:`Project` instance to get a
+    :class:`Settings` instance. See :attr:`Project.settings` attribute.
+
+    Usage::
+
+    - get project settings instance
+
+        >>> project.settings
+        <scrapinghub.client.projects.Settings at 0x10ecf1250>
+
+    - iterate through project settings
+
+        >>> project.settings.iter()
+        <dictionary-itemiterator at 0x10ed11578>
+
+    - list project settings
+
+        >>> project.settings.list()
+        [(u'default_job_units', 2),
+         (u'job_runtime_limit', 20)]
+
+    - get setting value by name
+
+        >>> project.settings.get('default_job_units')
+        2
+
+    - update setting value (some settings are read-only)
+
+        >>> project.settings.set('default_job_units', 2)
+
+    - update multiple settings at once
+
+        >>> project.settings.update({'default_job_units': 1,
+        ...                          'job_runtime_limit': 20})
+
+    - delete project setting by name
+
+        >>> project.settings.delete('job_runtime_limit')
+    """
+    def set(self, key, value):
+        # FIXME drop the method when post-by-key is implemented on server side
+        if not isinstance(key, six.string_types):
+            raise TypeError("key should be a string")
+        self.update({key: value})

scrapinghub/client/utils.py

@@ -4,9 +4,9 @@
 import json
 import logging
 import binascii
-
 from codecs import decode
-from six import string_types
+
+import six
 
 from ..hubstorage.resourcetype import DownloadableResource
 from ..hubstorage.resourcetype import ItemsResourceType
@@ -46,7 +46,7 @@ def parse_project_id(projectid):
 def parse_job_key(jobkey):
     if isinstance(jobkey, tuple):
         parts = jobkey
-    elif isinstance(jobkey, string_types):
+    elif isinstance(jobkey, six.string_types):
         parts = jobkey.split('/')
     else:
         raise ValueError("Job key should be a string or a tuple")
@@ -125,6 +125,30 @@ def list(self, *args, **kwargs):
         return list(self.iter(*args, **kwargs))
 
 
+class _MappingProxy(_Proxy):
+
+    def get(self, key):
+        return next(self._origin.apiget(key))
+
+    def set(self, key, value):
+        self._origin.apipost(key, data=json.dumps(value), is_idempotent=True)
+
+    def update(self, values):
+        if not isinstance(values, dict):
+            raise TypeError("values should be a dict")
+        data = next(self._origin.apiget())
+        data.update(values)
+        self._origin.apipost(jl={k: v for k, v in six.iteritems(data)
+                                 if k not in self._origin.ignore_fields},
+                             is_idempotent=True)
+
+    def delete(self, key):
+        self._origin.apidelete(key)
+
+    def iter(self):
+        return six.iteritems(next(self._origin.apiget()))
+
+
 def wrap_kwargs(fn, kwargs_fn):
     """Tiny wrapper to prepare modified version of function kwargs"""
     def wrapped(*args, **kwargs):
@@ -160,7 +184,7 @@ def format_iter_filters(params):
     if filters and isinstance(filters, list):
         filter_data = []
         for elem in params.pop('filter'):
-            if isinstance(elem, string_types):
+            if isinstance(elem, six.string_types):
                 filter_data.append(elem)
             elif isinstance(elem, (list, tuple)):
                 filter_data.append(json.dumps(elem))
@@ -195,12 +219,12 @@ def parse_auth(auth):
         return (apikey, '')
 
     if isinstance(auth, tuple):
-        all_strings = all(isinstance(k, string_types) for k in auth)
+        all_strings = all(isinstance(k, six.string_types) for k in auth)
         if len(auth) != 2 or not all_strings:
             raise ValueError("Wrong authentication credentials")
         return auth
 
-    if not isinstance(auth, string_types):
+    if not isinstance(auth, six.string_types):
         raise ValueError("Wrong authentication credentials")
 
     jwt_auth = _search_for_jwt_credentials(auth)
@@ -217,7 +241,7 @@ def _search_for_jwt_credentials(auth):
     except (binascii.Error, TypeError):
         return
     try:
-        if not isinstance(decoded_auth, string_types):
+        if not isinstance(decoded_auth, six.string_types):
             decoded_auth = decoded_auth.decode('ascii')
         login, _, password = decoded_auth.partition(':')
         if password and parse_job_key(login):