Drop job.purge, fix tests

Author: vshlapakov

scrapinghub/client/jobs.py

@@ -413,18 +413,6 @@ def cancel(self):
         """
         self._project.jobq.request_cancel(self)
 
-    def purge(self):
-        """Delete job and expire its local metadata.
-
-        Usage::
-
-            >>> job.purge()
-            >>> job.metadata.get('state')
-            'deleted'
-        """
-        self.delete()
-        self.metadata.expire()
-
 
 class JobMeta(_MappingProxy):
     """Class representing job metadata.

tests/client/conftest.py

@@ -141,10 +141,9 @@ def setup_vcrpy(request, project):
 
 
 def remove_all_jobs(project):
-    for k in list(project.settings.keys()):
+    for k, _ in project.settings.iter():
         if k != 'botgroups':
-            del project.settings[k]
-    project.settings.save()
+            project.settings.delete(k)
 
     # Cleanup JobQ: run 2 times to ensure we covered all jobs
     for queuename in ('pending', 'running', 'finished')*2:

tests/client/test_job.py

@@ -1,3 +1,5 @@
+from collections import Iterator
+
 from scrapinghub.client.items import Items
 from scrapinghub.client.jobs import Job
 from scrapinghub.client.jobs import JobMeta
@@ -22,15 +24,6 @@ def test_job_base(client, spider):
     assert isinstance(job.metadata, JobMeta)
 
 
-def test_job_update_metadata(spider):
-    job = spider.jobs.schedule(meta={'meta1': 'data1'})
-    assert job.metadata['meta1'] == 'data1'
-    job.update_metadata(meta1='data2', meta2='data3')
-    job.metadata.expire()
-    assert job.metadata['meta1'] == 'data2'
-    assert job.metadata['meta2'] == 'data3'
-
-
 def test_job_update_tags(spider):
     job1 = spider.jobs.schedule(spider_args={'subid': 'tags-1'},
                                 add_tag=['tag1'])
@@ -39,27 +32,26 @@ def test_job_update_tags(spider):
     # FIXME the endpoint normalises tags so it's impossible to send tags
     # having upper-cased symbols, let's add more tests when it's fixed
     assert job1.update_tags(add=['tag11', 'tag12']) == 1
-    assert job1.metadata.liveget('tags') == ['tag1', 'tag11', 'tag12']
+    assert job1.metadata.get('tags') == ['tag1', 'tag11', 'tag12']
 
     assert job1.update_tags(remove=['tag1', 'tagx']) == 1
-    assert job1.metadata.liveget('tags') == ['tag11', 'tag12']
+    assert job1.metadata.get('tags') == ['tag11', 'tag12']
 
     # assert that 2nd job tags weren't changed
-    assert job2.metadata.liveget('tags') == ['tag2']
+    assert job2.metadata.get('tags') == ['tag2']
     # FIXME adding and removing tags at the same time doesn't work neither:
     # remove-tag field is ignored if there's non-void add-tag field
 
 
 def test_job_start(spider):
     job = spider.jobs.schedule()
-    assert job.metadata['state'] == 'pending'
+    assert job.metadata.get('state') == 'pending'
     job.start()
-    job.metadata.expire()
-    assert job.metadata['state'] == 'running'
-    assert isinstance(job.metadata['pending_time'], int)
-    assert isinstance(job.metadata['running_time'], int)
-    assert job.metadata['spider'] == TEST_SPIDER_NAME
-    assert job.metadata['priority'] == 2
+    assert job.metadata.get('state') == 'running'
+    assert isinstance(job.metadata.get('pending_time'), int)
+    assert isinstance(job.metadata.get('running_time'), int)
+    assert job.metadata.get('spider') == TEST_SPIDER_NAME
+    assert job.metadata.get('priority') == 2
 
 
 def test_job_start_extras(spider):
@@ -86,75 +78,94 @@ def test_job_start_extras(spider):
 
 def test_job_update(spider):
     job = spider.jobs.schedule()
-    assert job.metadata['state'] == 'pending'
+    assert job.metadata.get('state') == 'pending'
     job.update(state='running', foo='bar')
 
     job = spider.jobs.get(job.key)
-    assert job.metadata['state'] == 'running'
-    assert job.metadata['foo'] == 'bar'
+    assert job.metadata.get('state') == 'running'
+    assert job.metadata.get('foo') == 'bar'
 
 
 def test_job_cancel_pending(spider):
     job = spider.jobs.schedule()
-    assert job.metadata['state'] == 'pending'
-    job.metadata.expire()
+    assert job.metadata.get('state') == 'pending'
     job.cancel()
-    assert job.metadata['state'] == 'finished'
+    assert job.metadata.get('state') == 'finished'
 
 
 def test_job_cancel_running(spider):
     job = spider.jobs.schedule()
     job.start()
-    assert job.metadata['state'] == 'running'
-    job.metadata.expire()
+    assert job.metadata.get('state') == 'running'
     job.cancel()
     # still running as should be stopped by scheduler
-    assert job.metadata['state'] == 'running'
+    assert job.metadata.get('state') == 'running'
 
 
 def test_job_finish(spider):
     job = spider.jobs.schedule()
-    assert job.metadata['state'] == 'pending'
-    job.metadata.expire()
+    assert job.metadata.get('state') == 'pending'
     job.finish()
     assert job.metadata.get('state') == 'finished'
 
 
 def test_job_finish_with_metadata(spider):
     job = spider.jobs.schedule(meta={'meta1': 'val1', 'meta2': 'val3'})
-    assert job.metadata['state'] == 'pending'
-    job.metadata.expire()
+    assert job.metadata.get('state') == 'pending'
     job.finish(meta2='val2', meta3='val3')
     assert job.metadata.get('state') == 'finished'
-    assert job.metadata['meta1'] == 'val1'
-    assert job.metadata['meta2'] == 'val2'
-    assert job.metadata['meta3'] == 'val3'
+    assert job.metadata.get('meta1') == 'val1'
+    assert job.metadata.get('meta2') == 'val2'
+    assert job.metadata.get('meta3') == 'val3'
 
 
 def test_job_delete(spider):
     job = spider.jobs.schedule(meta={'state': 'finished'})
-    assert job.metadata['state'] == 'finished'
-    job.metadata.expire()
+    assert job.metadata.get('state') == 'finished'
     job.delete()
     assert job.metadata.get('state') == 'deleted'
 
 
 def test_job_delete_with_metadata(spider):
     meta = {'state': 'finished', 'meta1': 'val1', 'meta2': 'val3'}
     job = spider.jobs.schedule(meta=meta)
-    assert job.metadata['state'] == 'finished'
-    job.metadata.expire()
+    assert job.metadata.get('state') == 'finished'
     job.delete(meta2='val2', meta3='val3')
     assert job.metadata.get('state') == 'deleted'
-    assert job.metadata['meta1'] == 'val1'
-    assert job.metadata['meta2'] == 'val2'
-    assert job.metadata['meta3'] == 'val3'
+    assert job.metadata.get('meta1') == 'val1'
+    assert job.metadata.get('meta2') == 'val2'
+    assert job.metadata.get('meta3') == 'val3'
 
 
-def test_job_purge(spider):
-    meta = {'state': 'finished', 'meta1': 'val1'}
-    job = spider.jobs.schedule(meta=meta)
-    assert job.metadata['state'] == 'finished'
-    job.purge()
-    assert job.metadata['state'] == 'deleted'
-    assert job.metadata['meta1'] == 'val1'
+def test_metadata_update(spider):
+    job = spider.jobs.schedule(meta={'meta1': 'data1'})
+    assert job.metadata.get('meta1') == 'data1'
+    job.metadata.update({'meta1': 'data2', 'meta2': 'data3'})
+    assert job.metadata.get('meta1') == 'data2'
+    assert job.metadata.get('meta2') == 'data3'
+
+
+def test_metadata_set(spider):
+    job = spider.jobs.schedule(meta={'meta1': 'data1'})
+    assert job.metadata.get('meta1') == 'data1'
+    job.metadata.set('meta1', 'data2')
+    job.metadata.set('meta2', 123)
+    assert job.metadata.get('meta1') == 'data2'
+    assert job.metadata.get('meta2') == 123
+
+
+def test_metadata_delete(spider):
+    job = spider.jobs.schedule(meta={'meta1': 'data1', 'meta2': 'data2'})
+    job.metadata.delete('meta1')
+    assert job.metadata.get('meta1') is None
+    assert job.metadata.get('meta2') == 'data2'
+
+
+def test_metadata_iter_list(spider):
+    job = spider.jobs.schedule(meta={'meta1': 'data1', 'meta2': 'data2'})
+    meta_iter = job.metadata.iter()
+    assert isinstance(meta_iter, Iterator)
+    meta_list = job.metadata.list()
+    assert ('meta1', 'data1') in meta_list
+    assert ('meta2', 'data2') in meta_list
+    assert meta_list == list(meta_iter)

tests/client/test_projects.py

@@ -162,7 +162,7 @@ def test_project_jobs_schedule(project):
     assert isinstance(job0, Job)
     validate_default_meta(job0.metadata, state='pending')
     assert isinstance(job0.metadata.get('pending_time'), int)
-    assert job0.metadata['pending_time'] > 0
+    assert job0.metadata.get('pending_time') > 0
     assert job0.metadata.get('scheduled_by')
 
     # running the same spider with same args leads to duplicate error
@@ -181,7 +181,7 @@ def test_project_jobs_schedule(project):
     assert meta.get('meta1') == 'val1'
     assert meta.get('spider_args') == {'arg1': 'val1', 'arg2': 'val2'}
     assert isinstance(meta.get('running_time'), int)
-    assert meta['running_time'] > 0
+    assert meta.get('running_time') > 0
     assert meta.get('started_by')
 
 

tests/client/test_spiders.py

@@ -156,7 +156,7 @@ def test_spider_jobs_schedule(spider):
     assert job1.metadata.get('meta1') == 'val1'
     assert job1.metadata.get('spider_args') == {'arg1': 'val1', 'arg2': 'val2'}
     assert isinstance(job1.metadata.get('running_time'), int)
-    assert job1.metadata['running_time'] > 0
+    assert job1.metadata.get('running_time') > 0
     assert job1.metadata.get('started_by')