aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-04-29 14:44:49 -0700
committerBryan Newbold <bnewbold@archive.org>2020-04-29 14:44:49 -0700
commitb306927e721349302d9a30511e8eb0c0676d4e04 (patch)
tree6f65069d54344c06b8f3f924ec7dbfae1e8345ac
parentf2177d5e30190dfc1e55f1b08fd21c2ce917ee86 (diff)
downloadsandcrawler-b306927e721349302d9a30511e8eb0c0676d4e04.tar.gz
sandcrawler-b306927e721349302d9a30511e8eb0c0676d4e04.zip
timeouts: don't push through None error messages
-rw-r--r--python/sandcrawler/workers.py4
1 files changed, 2 insertions, 2 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py
index 9f1b55c..96aef3f 100644
--- a/python/sandcrawler/workers.py
+++ b/python/sandcrawler/workers.py
@@ -69,10 +69,10 @@ class SandcrawlerWorker(object):
self.counts['timeout'] += 1
resp = self.timeout_response(task) # pylint: disable=assignment-from-none
# TODO: what if it is this push_record() itself that is timing out?
- if self.sink:
+ if resp and self.sink:
self.sink.push_record(resp)
self.counts['pushed'] += 1
- else:
+ elif resp:
print(json.dumps(resp))
finally:
signal.alarm(0)