Add count param for iter* methods

Author: vshlapakov

scrapinghub/client/activity.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 
 from .proxy import _Proxy
-from .utils import parse_job_key
+from .utils import parse_job_key, update_kwargs
 
 
 class Activity(_Proxy):
@@ -44,13 +44,14 @@ class Activity(_Proxy):
         >>> project.activity.add(events)
 
     """
-    def iter(self, **params):
+    def iter(self, count=None, **params):
         """Iterate over activity events.
 
+        :param count: limit amount of elements.
         :return: a generator object over a list of activity event dicts.
         :rtype: :class:`types.GeneratorType[dict]`
         """
-        # TODO describe allowable params
+        update_kwargs(params, count=count)
         params = self._modify_iter_params(params)
         return self._origin.list(**params)
 

scrapinghub/client/collections.py

@@ -284,12 +284,22 @@ def create_writer(self, start=0, auth=None, size=1000, interval=15,
                       maxitemsize=1024 ** 2, callback=None):
         """Create a new writer for a collection.
 
+        :param start: (optional) initial offset for writer thread.
+        :param auth: (optional) set auth credentials for the request.
+        :param size: (optional) set initial queue size.
+        :param interval: (optional) set interval for writer thread.
+        :param qsize: (optional) setup max queue size for the writer.
+        :param content_encoding: (optional) set different Content-Encoding header.
+        :param maxitemsize: (optional) max item size in bytes.
+        :param callback: (optional) some callback function.
         :return: a new writer object.
         :rtype: :class:`~scrapinghub.hubstorage.batchuploader._BatchWriter`
+
+        If provided - calllback shouldn't try to inject more items in the queue,
+        otherwise it can lead to deadlocks.
         """
-        # TODO describe allowable params
         kwargs = {}
-        update_kwargs(start=start, auth=auth, size=size, interval=interval,
+        update_kwargs(kwargs, start=start, auth=auth, size=size, interval=interval,
                       qsize=qsize, content_encoding=content_encoding,
                       maxitemsize=maxitemsize, callback=callback)
         return self._origin._collections.create_writer(

scrapinghub/client/proxy.py

@@ -4,6 +4,7 @@
 import json
 
 from ..hubstorage import ValueTooLarge as _ValueTooLarge
+from .utils import update_kwargs
 from .exceptions import ValueTooLarge
 
 
@@ -61,13 +62,14 @@ def write(self, item):
         except _ValueTooLarge as exc:
             raise ValueTooLarge(str(exc))
 
-    def iter(self, _key=None, **params):
+    def iter(self, _key=None, count=None, **params):
         """Iterate over elements in collection.
 
+        :param count: limit amount of elements.
         :return: a generator object over a list of element dictionaries.
         :rtype: :class:`types.GeneratorType[dict]`
         """
-        # TODO describe allowable params
+        update_kwargs(params or {}, count=count)
         params = self._modify_iter_params(params)
         return self._origin.list(_key, **params)
 
@@ -90,35 +92,40 @@ def close(self, block=True):
 
 class _DownloadableProxyMixin(object):
 
-    def iter(self, _path=None, requests_params=None, **apiparams):
+    def iter(self, _path=None, count=None, requests_params=None, **apiparams):
         """A general method to iterate through elements.
 
+        :param count: limit amount of elements.
         :return: an iterator over elements list.
         :rtype: :class:`collections.Iterable`
         """
-        # TODO describe allowable params
+        update_kwargs(apiparams, count=count)
         apiparams = self._modify_iter_params(apiparams)
         return self._origin.iter_values(_path, requests_params, **apiparams)
 
-    def iter_raw_json(self, _path=None, requests_params=None, **apiparams):
+    def iter_raw_json(self, _path=None, count=None, requests_params=None,
+                      **apiparams):
         """A method to iterate through raw json-packed elements.
         Can be convenient if data is needed in raw json format.
 
+        :param count: limit amount of elements.
         :return: an iterator over elements list packed with json.
         :rtype: :class:`collections.Iterable[str]`
         """
-        # TODO describe allowable params
+        update_kwargs(apiparams, count=count)
         apiparams = self._modify_iter_params(apiparams)
         return self._origin.iter_json(_path, requests_params, **apiparams)
 
-    def iter_raw_msgpack(self, _path=None, requests_params=None, **apiparams):
+    def iter_raw_msgpack(self, _path=None, count=None, requests_params=None,
+                         **apiparams):
         """A method to iterate through raw msgpack-ed elements.
         Can be convenient if data is needed in same msgpack format.
 
+        :param count: limit amount of elements.
         :return: an iterator over elements list packed with msgpack.
         :rtype: :class:`collections.Iterable[bytes]`
         """
-        # TODO describe allowable params
+        update_kwargs(apiparams, count=count)
         apiparams = self._modify_iter_params(apiparams)
         return self._origin.iter_msgpack(_path, requests_params, **apiparams)