Merge pull request #38 from scrapinghub/sc1467-1

SH Python client update

Author: vshlapakov

README.rst

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --doctest-glob='scrapinghub/*.py'

pytest.ini

@@ -1 +1 @@
-1.9.0
+1.10.0.dev1

scrapinghub/VERSION

@@ -1,5 +1,7 @@
-__all__ = ["APIError", "Connection", "HubstorageClient"]
-
+__all__ = ["APIError", "Connection", "HubstorageClient",
+           "ScrapinghubClient", "ScrapinghubAPIError",
+           "DuplicateJobError", "BadRequest", "NotFound",
+           "Unauthorized", "ValueTooLarge"]
 
 import pkgutil
 __version__ = pkgutil.get_data(__package__, 'VERSION')
@@ -9,3 +11,12 @@
 
 from .legacy import *
 from .hubstorage import HubstorageClient
+from .client import ScrapinghubClient
+from .client.exceptions import (
+    ScrapinghubAPIError,
+    DuplicateJobError,
+    BadRequest,
+    NotFound,
+    Unauthorized,
+    ValueTooLarge,
+)

scrapinghub/__init__.py

@@ -0,0 +1,93 @@
+from scrapinghub import Connection as _Connection
+from scrapinghub import HubstorageClient as _HubstorageClient
+
+from .projects import Projects
+from .exceptions import wrap_http_errors
+
+from .utils import parse_auth
+from .utils import parse_project_id, parse_job_key
+
+
+__all__ = ['ScrapinghubClient']
+
+
+class Connection(_Connection):
+
+    @wrap_http_errors
+    def _request(self, *args, **kwargs):
+        return super(Connection, self)._request(*args, **kwargs)
+
+
+class HubstorageClient(_HubstorageClient):
+
+    @wrap_http_errors
+    def request(self, *args, **kwargs):
+        return super(HubstorageClient, self).request(*args, **kwargs)
+
+
+class ScrapinghubClient(object):
+    """Main class to work with Scrapinghub API.
+
+    :param auth: Scrapinghub APIKEY or other SH auth credentials.
+    :param dash_endpoint: (optional) Scrapinghub Dash panel url.
+    :param \*\*kwargs: (optional) Additional arguments for
+        :class:`scrapinghub.hubstorage.HubstorageClient` constructor.
+
+    :ivar projects: projects collection, :class:`Projects` instance.
+
+    Usage::
+
+        >>> from scrapinghub import ScrapinghubClient
+        >>> client = ScrapinghubClient('APIKEY')
+        >>> client
+        <scrapinghub.client.ScrapinghubClient at 0x1047af2e8>
+    """
+
+    def __init__(self, auth=None, dash_endpoint=None, **kwargs):
+        self.projects = Projects(self)
+        login, password = parse_auth(auth)
+        self._connection = Connection(apikey=login,
+                                      password=password,
+                                      url=dash_endpoint)
+        self._hsclient = HubstorageClient(auth=(login, password), **kwargs)
+
+    def get_project(self, project_id):
+        """Get :class:`Project` instance with a given project id.
+
+        The method is a shortcut for client.projects.get().
+
+        :param project_id: integer or string numeric project id.
+        :return: :class:`Project` object.
+        :rtype: scrapinghub.client.projects.Project
+
+        Usage::
+
+            >>> project = client.get_project(123)
+            >>> project
+            <scrapinghub.client.projects.Project at 0x106cdd6a0>
+        """
+        return self.projects.get(parse_project_id(project_id))
+
+    def get_job(self, job_key):
+        """Get Job with a given job key.
+
+        :param job_key: job key string in format 'project_id/spider_id/job_id',
+            where all the components are integers.
+        :return: :class:`Job` object.
+        :rtype: scrapinghub.client.jobs.Job
+
+        Usage::
+
+            >>> job = client.get_job('123/1/1')
+            >>> job
+            <scrapinghub.client.jobs.Job at 0x10afe2eb1>
+        """
+        project_id = parse_job_key(job_key).project_id
+        return self.projects.get(project_id).jobs.get(job_key)
+
+    def close(self, timeout=None):
+        """Close client instance.
+
+        :param timeout: (optional) float timeout secs to stop gracefully.
+        """
+        self._hsclient.close(timeout=timeout)

scrapinghub/client/__init__.py

@@ -0,0 +1,65 @@
+from __future__ import absolute_import
+
+from .utils import _Proxy
+from .utils import parse_job_key
+
+
+class Activity(_Proxy):
+    """Representation of collection of job activity events.
+
+    Not a public constructor: use :class:`Project` instance to get a
+    :class:`Activity` instance. See :attr:`Project.activity` attribute.
+
+    Please note that list() method can use a lot of memory and for a large
+    amount of activities it's recommended to iterate through it via iter()
+    method (all params and available filters are same for both methods).
+
+    Usage:
+
+    - get all activity from a project::
+
+        >>> project.activity.iter()
+        <generator object jldecode at 0x1049ee990>
+
+    - get only last 2 events from a project::
+
+        >>> project.activity.list(count=2)
+        [{'event': 'job:completed', 'job': '123/2/3', 'user': 'jobrunner'},
+         {'event': 'job:started', 'job': '123/2/3', 'user': 'john'}]
+
+    - post a new event::
+
+        >>> event = {'event': 'job:completed',
+                     'job': '123/2/4',
+                     'user': 'jobrunner'}
+        >>> project.activity.add(event)
+
+    - post multiple events at once::
+
+        >>> events = [
+            {'event': 'job:completed', 'job': '123/2/5', 'user': 'jobrunner'},
+            {'event': 'job:cancelled', 'job': '123/2/6', 'user': 'john'},
+        ]
+        >>> project.activity.add(events)
+
+    """
+    def __init__(self, *args, **kwargs):
+        super(Activity, self).__init__(*args, **kwargs)
+        self._proxy_methods([('iter', 'list')])
+        self._wrap_iter_methods(['iter'])
+
+    def add(self, values, **kwargs):
+        """Add new event to the project activity.
+
+        :param values: a single event or a list of events, where event is
+            represented with a dictionary of ('event', 'job', 'user') keys.
+        """
+        if not isinstance(values, list):
+            values = list(values)
+        for activity in values:
+            if not isinstance(activity, dict):
+                raise ValueError("Please pass events as dictionaries")
+            job_key = activity.get('job')
+            if job_key and parse_job_key(job_key).project_id != self.key:
+                raise ValueError('Please use same project id')
+        self._origin.post(values, **kwargs)