Drop iter_raw_* methods

Author: vshlapakov

scrapinghub/client/collections.py

@@ -211,51 +211,6 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None,
         return self._origin._collections.iter_values(
             self._origin.coltype, self._origin.colname, **params)
 
-    def iter_raw_json(self, key=None, prefix=None, prefixcount=None,
-                      startts=None, endts=None, requests_params=None, **params):
-        """A method to iterate through json pack-ed items.
-        Can be convenient if data is needed in the json format.
-
-        :param key: a string key or a list of keys to filter with.
-        :param prefix: a string prefix to filter items.
-        :param prefixcount: maximum number of values to return per prefix.
-        :param startts: UNIX timestamp at which to begin results.
-        :param endts: UNIX timestamp at which to end results.
-        :param requests_params: (optional) a dict with optional requests params.
-        :param \*\*params: (optional) additional query params for the request.
-        :return: an iterator over items list packed with json.
-        :rtype: :class:`collections.Iterable[str]`
-        """
-        update_kwargs(params, key=key, prefix=prefix, prefixcount=prefixcount,
-                      startts=startts, endts=endts,
-                      requests_params=requests_params)
-        params = self._collections._modify_iter_params(params)
-        return self._origin._collections.iter_json(
-            self._origin.coltype, self._origin.colname, **params)
-
-    def iter_raw_msgpack(self, key=None, prefix=None, prefixcount=None,
-                         startts=None, endts=None, requests_params=None,
-                         **params):
-        """A method to iterate through raw msgpack-ed items.
-        Can be convenient if data is needed in same msgpack format.
-
-        :param key: a string key or a list of keys to filter with.
-        :param prefix: a string prefix to filter items.
-        :param prefixcount: maximum number of values to return per prefix.
-        :param startts: UNIX timestamp at which to begin results.
-        :param endts: UNIX timestamp at which to end results.
-        :param requests_params: (optional) a dict with optional requests params.
-        :param \*\*params: (optional) additional query params for the request.
-        :return: an iterator over items list packed with msgpack.
-        :rtype: :class:`collections.Iterable[bytes]`
-        """
-        update_kwargs(params, key=key, prefix=prefix, prefixcount=prefixcount,
-                      startts=startts, endts=endts,
-                      requests_params=requests_params)
-        params = self._collections._modify_iter_params(params)
-        return self._origin._collections.iter_msgpack(
-            self._origin.coltype, self._origin.colname, **params)
-
     def list(self, key=None, prefix=None, prefixcount=None, startts=None,
              endts=None, requests_params=None, **params):
         """Convenient shortcut to list iter results.

scrapinghub/client/proxy.py

@@ -112,32 +112,6 @@ def iter(self, _path=None, count=None, requests_params=None, **apiparams):
         apiparams = self._modify_iter_params(apiparams)
         return self._origin.iter_values(_path, requests_params, **apiparams)
 
-    def iter_raw_json(self, _path=None, count=None, requests_params=None,
-                      **apiparams):
-        """A method to iterate through raw json-packed elements.
-        Can be convenient if data is needed in raw json format.
-
-        :param count: limit amount of elements.
-        :return: an iterator over elements list packed with json.
-        :rtype: :class:`collections.Iterable[str]`
-        """
-        update_kwargs(apiparams, count=count)
-        apiparams = self._modify_iter_params(apiparams)
-        return self._origin.iter_json(_path, requests_params, **apiparams)
-
-    def iter_raw_msgpack(self, _path=None, count=None, requests_params=None,
-                         **apiparams):
-        """A method to iterate through raw msgpack-ed elements.
-        Can be convenient if data is needed in same msgpack format.
-
-        :param count: limit amount of elements.
-        :return: an iterator over elements list packed with msgpack.
-        :rtype: :class:`collections.Iterable[bytes]`
-        """
-        update_kwargs(apiparams, count=count)
-        apiparams = self._modify_iter_params(apiparams)
-        return self._origin.iter_msgpack(_path, requests_params, **apiparams)
-
 
 class _MappingProxy(_Proxy):
     """A helper class to support basic get/set interface for dict-like

tests/client/cassetes/test_logs/test_logs_iter_raw_json.gz

@@ -1,10 +1,6 @@
-import json
-
 import pytest
 from six.moves import range
 
-from scrapinghub.hubstorage.serialization import mpdecode
-
 
 def _add_test_items(job):
     for i in range(3):
@@ -29,18 +25,6 @@ def test_items_iter(spider):
     with pytest.raises(StopIteration):
         next(o)
 
-    o = job.items.iter_raw_json(offset=2)
-    item = json.loads(next(o))
-    assert item['id'] == 2
-    assert item['data'] == 'data2'
-    with pytest.raises(StopIteration):
-        next(o)
-
-    msgpacked_o = job.items.iter_raw_msgpack(offset=2)
-    o = mpdecode(msgpacked_o)
-    assert item['id'] == 2
-    assert item['data'] == 'data2'
-
 
 def test_items_list(spider):
     job = spider.jobs.run(meta={'state': 'running'})

tests/client/cassetes/test_logs/test_logs_iter_raw_msgpack.gz

@@ -1,11 +1,9 @@
-import json
 import types
 from numbers import Integral
 
 import pytest
 
 from scrapinghub.client.utils import LogLevel
-from scrapinghub.hubstorage.serialization import mpdecode
 
 from .conftest import TEST_TS
 
@@ -103,35 +101,3 @@ def test_logs_list_filter(spider):
 
     logs3 = job.logs.list(filter=[('message', 'contains', ['simple'])])
     assert len(logs3) == 3
-
-
-def test_logs_iter_raw_json(spider):
-    job = spider.jobs.run()
-    _add_test_logs(job)
-
-    logs0 = job.logs.iter_raw_json(offset=2)
-    raw_log0 = next(logs0)
-    log0 = json.loads(raw_log0)
-    assert log0.get('message') == 'simple-msg3'
-    assert log0.get('_key')
-    assert isinstance(log0.get('time'), Integral)
-    assert log0.get('level') == 10
-
-    logs1 = job.logs.iter_raw_json(level='ERROR')
-    raw_log1 = next(logs1)
-    log1 = json.loads(raw_log1)
-    assert log1.get('message') == 'error-msg'
-
-
-def test_logs_iter_raw_msgpack(spider):
-    job = spider.jobs.run()
-    _add_test_logs(job)
-
-    logs1 = job.logs.iter_raw_msgpack(offset=2)
-    assert isinstance(logs1, types.GeneratorType)
-    unpacked_logs1 = list(mpdecode(logs1))
-    assert unpacked_logs1[0].get('message') == 'simple-msg3'
-
-    logs2 = job.logs.iter_raw_msgpack(level='ERROR')
-    unpacked_logs2 = list(mpdecode(logs2))
-    assert unpacked_logs2[0].get('message') == 'error-msg'

tests/client/cassetes/test_requests/test_requests_iter_raw_json.gz

@@ -1,5 +1,3 @@
-import json
-
 import pytest
 
 from .conftest import TEST_TS
@@ -39,18 +37,3 @@ def test_requests_iter(spider):
     }
     with pytest.raises(StopIteration):
         next(rr)
-
-
-def test_requests_iter_raw_json(spider):
-    job = spider.jobs.run()
-    _add_test_requests(job)
-    job.requests.close()
-
-    rr = job.requests.iter_raw_json()
-    raw_req = next(rr)
-    req = json.loads(raw_req)
-    assert req.get('url') == 'http://test.com/'
-    assert req.get('status') == 200
-    next(rr), next(rr)
-    with pytest.raises(StopIteration):
-        next(rr)