Skip to content

Commit d231d29

Browse files
committed
Minor refactoring after CR
1 parent 2ad3679 commit d231d29

File tree

2 files changed

+41
-22
lines changed

2 files changed

+41
-22
lines changed

README.rst

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -451,25 +451,32 @@ Get a frontier slot by name::
451451

452452
Add a request to the slot::
453453

454-
>>> slot.add([{'fp': '/some/path.html'}])
454+
>>> slot.queue.add([{'fp': '/some/path.html'}])
455455
>>> slot.flush()
456456

457+
Add a fingerprint only to the slot::
458+
459+
>>> slot.fingerprints.add(['fp1', 'fp2'])
460+
>>> slot.flush()
461+
462+
There are convenient shortcuts: ``f`` for ``fingerprints`` and ``q`` for ``queue``.
463+
457464
Add requests with additional parameters::
458465

459-
>>> slot.add([{'fp': '/'}, {'fp': 'page1.html', 'p': 1, 'qdata': {'depth': 1}}])
466+
>>> slot.q.add([{'fp': '/'}, {'fp': 'page1.html', 'p': 1, 'qdata': {'depth': 1}}])
460467
>>> slot.flush()
461468

462469
To list requests for a given slot::
463470

464-
>>> reqs = slot.list()
471+
>>> reqs = slot.q.list()
465472

466473
To retrieve fingerprints for a given slot::
467474

468-
>>> fps = [req['requests'] for req in slot.iter()]
475+
>>> fps = [req['requests'] for req in slot.q.iter()]
469476

470477
To delete a batch of requests::
471478

472-
>>> slot.delete('00013967d8af7b0001')
479+
>>> slot.q.delete('00013967d8af7b0001')
473480

474481
To delete the whole slot from the frontier::
475482

scrapinghub/client.py

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1154,22 +1154,22 @@ class FrontierSlot(object):
11541154
11551155
Usage:
11561156
1157-
- add request to a slot
1157+
- add request to a queue
11581158
>>> data = [{'fp': 'page1.html', 'p': 1, 'qdata': {'depth': 1}}]
1159-
>>> slot.add('example.com', data)
1159+
>>> slot.q.add('example.com', data)
11601160
11611161
- flush data for a slot
11621162
>>> slot.flush()
11631163
11641164
- read requests from a slot
1165-
>>> slot.iter()
1165+
>>> slot.q.iter()
11661166
<generator object jldecode at 0x1049aa9e8>
1167-
>>> slot.list()
1167+
>>> slot.q.list()
11681168
[{'id': '0115a8579633600006',
11691169
'requests': [['page1.html', {'depth': 1}]]}]
11701170
11711171
- delete a batch with requests from a slot
1172-
>>> slot.delete('0115a8579633600006')
1172+
>>> slot.q.delete('0115a8579633600006')
11731173
11741174
- delete a whole slot
11751175
>>> slot.delete()
@@ -1181,21 +1181,19 @@ def __init__(self, client, frontier, slot):
11811181
self._frontier = frontier
11821182
self.fingerprints = FrontierSlotFingerprints(self)
11831183
self.queue = FrontierSlotQueue(self)
1184-
# proxy iter & list methods to FrontierSlotQueue instance
1185-
self.iter = self.queue.iter
1186-
self.list = self.queue.list
11871184

1188-
def add(self, fps):
1189-
"""Add requests to slot."""
1190-
origin = self._frontier._frontiers._origin
1191-
return origin.add(self._frontier.key, self.key, fps)
1185+
@property
1186+
def f(self):
1187+
return self.fingerprints
1188+
1189+
@property
1190+
def q(self):
1191+
return self.queue
11921192

1193-
def delete(self, ids=None):
1194-
"""Delete slot or some specific requests."""
1193+
def delete(self):
1194+
"""Delete the slot."""
11951195
origin = self._frontier._frontiers._origin
1196-
if ids is None:
1197-
return origin.delete_slot(self._frontier.key, self.key)
1198-
return self.queue.delete(ids)
1196+
return origin.delete_slot(self._frontier.key, self.key)
11991197

12001198
def flush(self):
12011199
"""Flush data for the slot."""
@@ -1212,6 +1210,15 @@ def __init__(self, slot):
12121210
self._frontier = slot._frontier
12131211
self._slot = slot
12141212

1213+
def add(self, fps):
1214+
origin = self._frontier._frontiers._origin
1215+
writer = origin._get_writer(self._frontier.key, self.key)
1216+
fps = list(fps) if not isinstance(fps, list) else fps
1217+
if not all(isinstance(fp, string_types) for fp in fps):
1218+
raise ValueError('Fingerprint should be of a string type')
1219+
for fp in fps:
1220+
writer.write({'fp': fp})
1221+
12151222
def iter(self, **kwargs):
12161223
"""Iterate through fingerprints."""
12171224
origin = self._frontier._frontiers._origin
@@ -1231,6 +1238,11 @@ def __init__(self, slot):
12311238
self._frontier = slot._frontier
12321239
self._slot = slot
12331240

1241+
def add(self, fps):
1242+
"""Add requests to slot."""
1243+
origin = self._frontier._frontiers._origin
1244+
return origin.add(self._frontier.key, self.key, fps)
1245+
12341246
def iter(self, **kwargs):
12351247
"""Iterate through batches in queue."""
12361248
origin = self._frontier._frontiers._origin

0 commit comments

Comments
 (0)