diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-04-29 14:44:49 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-04-29 14:44:49 -0700 |
| commit | b306927e721349302d9a30511e8eb0c0676d4e04 (patch) | |
| tree | 6f65069d54344c06b8f3f924ec7dbfae1e8345ac | |
| parent | f2177d5e30190dfc1e55f1b08fd21c2ce917ee86 (diff) | |
| download | sandcrawler-b306927e721349302d9a30511e8eb0c0676d4e04.tar.gz sandcrawler-b306927e721349302d9a30511e8eb0c0676d4e04.zip | |
timeouts: don't push through None error messages
| -rw-r--r-- | python/sandcrawler/workers.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index 9f1b55c..96aef3f 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -69,10 +69,10 @@ class SandcrawlerWorker(object): self.counts['timeout'] += 1 resp = self.timeout_response(task) # pylint: disable=assignment-from-none # TODO: what if it is this push_record() itself that is timing out? - if self.sink: + if resp and self.sink: self.sink.push_record(resp) self.counts['pushed'] += 1 - else: + elif resp: print(json.dumps(resp)) finally: signal.alarm(0) |