diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-06-17 21:23:08 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-06-17 21:23:08 -0700 |
| commit | b4a40d99b23a83eabeed490c0dce52dba31dc7b8 (patch) | |
| tree | 325f3fb3b4cf389f2e82d458346be0100f06f8bc /python/sandcrawler/workers.py | |
| parent | 83fc37ecff0176032542b2eee24b4f09d5c21db2 (diff) | |
| download | sandcrawler-b4a40d99b23a83eabeed490c0dce52dba31dc7b8.tar.gz sandcrawler-b4a40d99b23a83eabeed490c0dce52dba31dc7b8.zip | |
fixes and tweaks from testing locally
Diffstat (limited to 'python/sandcrawler/workers.py')
| -rw-r--r-- | python/sandcrawler/workers.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index 8115ee3..4a1d7a4 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -233,7 +233,7 @@ class BlackholeSink(SandcrawlerWorker): Useful for tests. """ - def push_record(self, task): + def push_record(self, task, key=None): return def push_batch(self, tasks): @@ -528,7 +528,7 @@ class KafkaJsonPusher(RecordPusher): # without decoding as JSON. Eg, for thumbnails (where # message bytes are JPEG, and we need # the sha1hex key # from the message) - record = msg + record = msg.value() else: record = json.loads(msg.value().decode('utf-8')) # This complex bit of code implements backoff/backpressure |