aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorbnewbold <bnewbold@archive.org>2020-03-12 17:02:43 +0000
committerbnewbold <bnewbold@archive.org>2020-03-12 17:02:43 +0000
commit30ba490bb65d195b14f5b06aea2de5b4eb1d23d2 (patch)
treef5a2ee19b6fe6b3cedd89f124ce46429ed646a97
parent770675b8104191dfbb013064020116c374b23e4d (diff)
parent7a03b73f331cd47be63ce55643cd884c9cf32b40 (diff)
downloadsandcrawler-30ba490bb65d195b14f5b06aea2de5b4eb1d23d2.tar.gz
sandcrawler-30ba490bb65d195b14f5b06aea2de5b4eb1d23d2.zip
Merge branch 'martin-abstract-class-process' into 'master'
workers: add explicit process to base class See merge request webgroup/sandcrawler!25
-rw-r--r--python/sandcrawler/workers.py6
1 files changed, 6 insertions, 0 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py
index c290421..54bd581 100644
--- a/python/sandcrawler/workers.py
+++ b/python/sandcrawler/workers.py
@@ -52,6 +52,12 @@ class SandcrawlerWorker(object):
print("Worker: {}".format(self.counts), file=sys.stderr)
return self.counts
+ def process(self, task):
+ """
+ Derived workers need to implement business logic here.
+ """
+ raise NotImplementedError('implementation required')
+
class MultiprocessWrapper(SandcrawlerWorker):
def __init__(self, worker, sink, jobs=None):