aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/workers.py6
1 files changed, 5 insertions, 1 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py
index a8b03c7..a110754 100644
--- a/python/sandcrawler/workers.py
+++ b/python/sandcrawler/workers.py
@@ -219,7 +219,11 @@ class JsonLinePusher(RecordPusher):
if not line:
continue
self.counts['total'] += 1
- record = json.loads(line)
+ try:
+ record = json.loads(line)
+ except json.decoder.JSONDecodeError:
+ self.counts['error-json-decode'] += 1
+ continue
if self.batch_size:
batch.append(record)
if len(batch) >= self.batch_size: