Split test_proxy, rename others according to modules

Author: vshlapakov

tests/client/conftest.py

@@ -22,6 +22,9 @@
 TEST_HS_ENDPOINT = os.getenv('HS_ENDPOINT',
                              'http://storage.vm.scrapinghub.com')
 
+# use some fixed timestamp to represent current time
+TEST_TS = 1476803148638
+
 # vcrpy creates the cassetes automatically under VCR_CASSETES_DIR
 VCR_CASSETES_DIR = 'tests/client/cassetes'
 

tests/client/test_activity.py

@@ -0,0 +1,42 @@
+import types
+
+import pytest
+
+from .conftest import TEST_PROJECT_ID
+
+
+def _add_test_activity(project):
+    activity = project.activity
+    jobkey = TEST_PROJECT_ID + '/2/3'
+    events = [{'event': 'job:completed', 'job': jobkey, 'user': 'jobrunner'},
+              {'event': 'job:cancelled', 'job': jobkey, 'user': 'john'}]
+    activity.add(events)
+
+
+def test_activity_wrong_project(project):
+    event = {'event': 'job:completed', 'job': '123/1/1', 'user': 'user'}
+    with pytest.raises(ValueError):
+        project.activity.add(event)
+
+
+def test_activity_iter(project):
+    _add_test_activity(project)
+    activity = project.activity.iter()
+    assert isinstance(activity, types.GeneratorType)
+    activity_item = next(activity)
+    assert activity_item == {'event': 'job:cancelled',
+                             'job': TEST_PROJECT_ID + '/2/3',
+                             'user': 'john'}
+
+
+def test_activity_list(project):
+    _add_test_activity(project)
+    activity = project.activity.list(count=2)
+    assert isinstance(activity, list)
+    assert len(activity) == 2
+    assert activity[0] == {'event': 'job:cancelled',
+                           'job': TEST_PROJECT_ID + '/2/3',
+                           'user': 'john'}
+    assert activity[1] == {'event': 'job:completed',
+                           'job': TEST_PROJECT_ID + '/2/3',
+                           'user': 'jobrunner'}

tests/client/test_client.py

@@ -5,10 +5,7 @@
 from scrapinghub.client.jobs import Job
 from scrapinghub.client.projects import Projects, Project
 
-from scrapinghub.hubstorage.utils import apipoll
-
-from .conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME
-from .conftest import TEST_USER_AUTH, TEST_DASH_ENDPOINT
+from .conftest import TEST_PROJECT_ID
 
 
 # ScrapinghubClient class tests
@@ -33,41 +30,3 @@ def test_client_get_project(client):
 def test_client_get_job(client):
     fake_job = client.get_job('1/2/3')
     assert isinstance(fake_job, Job)
-
-
-# Projects class tests
-
-
-def test_client_projects_get_project(client):
-    projects = client.projects
-    # testing with int project id
-    p1 = projects.get(int(TEST_PROJECT_ID))
-    assert isinstance(p1, Project)
-    # testing with string project id
-    p2 = projects.get(TEST_PROJECT_ID)
-    assert isinstance(p2, Project)
-    assert p1.key == p2.key
-
-
-def test_client_projects_list_projects(client):
-    projects = client.projects.list()
-    assert client.projects.list() == []
-
-    # use user apikey to list test projects
-    client = ScrapinghubClient(TEST_USER_AUTH, TEST_DASH_ENDPOINT)
-    projects = client.projects.list()
-    assert isinstance(projects, list)
-    assert int(TEST_PROJECT_ID) in projects
-
-
-def test_client_projects_summary(client, project):
-    # add at least one running or pending job to ensure summary is returned
-    project.jobs.schedule(TEST_SPIDER_NAME, meta={'state': 'running'})
-
-    def _get_summary():
-        summaries = {str(js['project']): js
-                     for js in client.projects.summary()}
-        return summaries.get(TEST_PROJECT_ID)
-
-    summary = apipoll(_get_summary)
-    assert summary is not None

tests/client/test_frontier.py tests/client/test_frontiers.pytests/client/test_frontier.py renamed to tests/client/test_frontiers.py

@@ -0,0 +1,54 @@
+import json
+
+import pytest
+from six.moves import range
+
+from scrapinghub.hubstorage.serialization import mpdecode
+
+
+def _add_test_items(job):
+    for i in range(3):
+        job.items.write({'id': i, 'data': 'data' + str(i)})
+    job.items.flush()
+    job.items.close()
+
+
+def test_items_iter(spider):
+    job = spider.jobs.schedule(meta={'state': 'running'})
+    _add_test_items(job)
+
+    o = job.items.iter()
+    assert next(o) == {'id': 0, 'data': 'data0'}
+    assert next(o) == {'id': 1, 'data': 'data1'}
+    next(o)
+    with pytest.raises(StopIteration):
+        next(o)
+
+    o = job.items.iter(offset=2)
+    assert next(o) == {'id': 2, 'data': 'data2'}
+    with pytest.raises(StopIteration):
+        next(o)
+
+    o = job.items.iter_raw_json(offset=2)
+    item = json.loads(next(o))
+    assert item['id'] == 2
+    assert item['data'] == 'data2'
+    with pytest.raises(StopIteration):
+        next(o)
+
+    msgpacked_o = job.items.iter_raw_msgpack(offset=2)
+    o = mpdecode(msgpacked_o)
+    assert item['id'] == 2
+    assert item['data'] == 'data2'
+
+
+def test_items_list(spider):
+    job = spider.jobs.schedule(meta={'state': 'running'})
+    _add_test_items(job)
+
+    o = job.items.list()
+    assert isinstance(o, list)
+    assert len(o) == 3
+    assert o[0] == {'id': 0, 'data': 'data0'}
+    assert o[1] == {'id': 1, 'data': 'data1'}
+    assert o[2] == {'id': 2, 'data': 'data2'}

tests/client/test_items.py

@@ -0,0 +1,137 @@
+import json
+import types
+from numbers import Integral
+
+import pytest
+
+from scrapinghub.client.utils import LogLevel
+from scrapinghub.hubstorage.serialization import mpdecode
+
+from .conftest import TEST_TS
+
+
+def _add_test_logs(job):
+    job.logs.log('simple-msg1')
+    job.logs.log('simple-msg2', ts=TEST_TS)
+    job.logs.log('simple-msg3', level=LogLevel.DEBUG)
+
+    job.logs.debug('debug-msg')
+    job.logs.info('info-msg')
+    job.logs.warning('warning-msg')
+    job.logs.warn('warn-msg')
+    job.logs.error('error-msg')
+
+    # test raw write interface
+    job.logs.write({'message': 'test',
+                    'level': LogLevel.INFO,
+                    'time': TEST_TS})
+    job.logs.flush()
+
+
+def test_logs_base(spider):
+    job = spider.jobs.schedule()
+    assert list(job.logs.iter()) == []
+    assert job.logs.batch_write_start() == 0
+    _add_test_logs(job)
+    log1 = job.logs.get(0)
+    assert log1['level'] == 20
+    assert log1['message'] == 'simple-msg1'
+    assert isinstance(log1['time'], Integral) and log1['time'] > 0
+    assert job.logs.stats() == {
+        'counts': {'10': 2, '20': 4, '30': 2, '40': 1},
+        'totals': {'input_bytes': 91, 'input_values': 9}
+    }
+    job.logs.close()
+
+
+def test_logs_iter(spider):
+    job = spider.jobs.schedule()
+    _add_test_logs(job)
+
+    logs1 = job.logs.iter()
+    assert isinstance(logs1, types.GeneratorType)
+    assert next(logs1).get('message') == 'simple-msg1'
+    assert next(logs1).get('message') == 'simple-msg2'
+    assert next(logs1).get('message') == 'simple-msg3'
+
+    # testing offset
+    logs2 = job.logs.iter(offset=3)
+    assert next(logs2).get('message') == 'debug-msg'
+    assert next(logs2).get('message') == 'info-msg'
+
+    # testing level
+    logs3 = job.logs.iter(level='ERROR')
+    assert next(logs3).get('message') == 'error-msg'
+    with pytest.raises(StopIteration):
+        next(logs3).get('message')
+
+
+def test_logs_list(spider):
+    job = spider.jobs.schedule()
+    _add_test_logs(job)
+
+    logs1 = job.logs.list()
+    assert isinstance(logs1, list)
+    assert len(logs1) == 9
+    assert logs1[0].get('message') == 'simple-msg1'
+    assert logs1[1].get('message') == 'simple-msg2'
+    assert logs1[2].get('message') == 'simple-msg3'
+
+    # testing offset
+    logs2 = job.logs.list(offset=3)
+    assert len(logs2) == 6
+    assert logs2[0].get('message') == 'debug-msg'
+    assert logs2[1].get('message') == 'info-msg'
+
+    # testing level
+    logs3 = job.logs.list(level='ERROR')
+    assert len(logs3) == 1
+    assert logs3[0].get('message') == 'error-msg'
+
+
+def test_logs_list_filter(spider):
+    job = spider.jobs.schedule()
+    _add_test_logs(job)
+
+    logs1 = job.logs.list(filter='["message", "contains", ["simple"]]')
+    assert isinstance(logs1, list)
+    assert len(logs1) == 3
+    assert logs1[0].get('message') == 'simple-msg1'
+
+    logs2 = job.logs.list(filter=[['message', 'contains', ['simple']]])
+    assert len(logs2) == 3
+
+    logs3 = job.logs.list(filter=[('message', 'contains', ['simple'])])
+    assert len(logs3) == 3
+
+
+def test_logs_iter_raw_json(spider):
+    job = spider.jobs.schedule()
+    _add_test_logs(job)
+
+    logs0 = job.logs.iter_raw_json(offset=2)
+    raw_log0 = next(logs0)
+    log0 = json.loads(raw_log0)
+    assert log0.get('message') == 'simple-msg3'
+    assert log0.get('_key')
+    assert isinstance(log0.get('time'), Integral)
+    assert log0.get('level') == 10
+
+    logs1 = job.logs.iter_raw_json(level='ERROR')
+    raw_log1 = next(logs1)
+    log1 = json.loads(raw_log1)
+    assert log1.get('message') == 'error-msg'
+
+
+def test_logs_iter_raw_msgpack(spider):
+    job = spider.jobs.schedule()
+    _add_test_logs(job)
+
+    logs1 = job.logs.iter_raw_msgpack(offset=2)
+    assert isinstance(logs1, types.GeneratorType)
+    unpacked_logs1 = list(mpdecode(logs1))
+    assert unpacked_logs1[0].get('message') == 'simple-msg3'
+
+    logs2 = job.logs.iter_raw_msgpack(level='ERROR')
+    unpacked_logs2 = list(mpdecode(logs2))
+    assert unpacked_logs2[0].get('message') == 'error-msg'

tests/client/test_logs.py

@@ -4,18 +4,62 @@
 import pytest
 from six.moves import range
 
+from scrapinghub import ScrapinghubClient
 from scrapinghub.client.activity import Activity
 from scrapinghub.client.collections import Collections
 from scrapinghub.client.exceptions import DuplicateJobError
 from scrapinghub.client.frontiers import Frontiers
 from scrapinghub.client.jobs import Jobs, Job
-from scrapinghub.client.projects import Settings
+from scrapinghub.client.projects import Project, Settings
 from scrapinghub.client.spiders import Spiders
 
+from scrapinghub.hubstorage.utils import apipoll
+
 from .conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME
+from .conftest import TEST_USER_AUTH, TEST_DASH_ENDPOINT
 from .utils import validate_default_meta
 
 
+# Projects class tests
+
+
+def test_client_projects_get_project(client):
+    projects = client.projects
+    # testing with int project id
+    p1 = projects.get(int(TEST_PROJECT_ID))
+    assert isinstance(p1, Project)
+    # testing with string project id
+    p2 = projects.get(TEST_PROJECT_ID)
+    assert isinstance(p2, Project)
+    assert p1.key == p2.key
+
+
+def test_client_projects_list_projects(client):
+    projects = client.projects.list()
+    assert client.projects.list() == []
+
+    # use user apikey to list test projects
+    client = ScrapinghubClient(TEST_USER_AUTH, TEST_DASH_ENDPOINT)
+    projects = client.projects.list()
+    assert isinstance(projects, list)
+    assert int(TEST_PROJECT_ID) in projects
+
+
+def test_client_projects_summary(client, project):
+    # add at least one running or pending job to ensure summary is returned
+    project.jobs.schedule(TEST_SPIDER_NAME, meta={'state': 'running'})
+
+    def _get_summary():
+        summaries = {str(js['project']): js
+                     for js in client.projects.summary()}
+        return summaries.get(TEST_PROJECT_ID)
+
+    summary = apipoll(_get_summary)
+    assert summary is not None
+
+
+#  Project class tests
+
 def test_project_subresources(project):
     assert project.key == TEST_PROJECT_ID
     assert isinstance(project.collections, Collections)