diff options
| -rw-r--r-- | python/sandcrawler/workers.py | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index a8b03c7..a110754 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -219,7 +219,11 @@ class JsonLinePusher(RecordPusher): if not line: continue self.counts['total'] += 1 - record = json.loads(line) + try: + record = json.loads(line) + except json.decoder.JSONDecodeError: + self.counts['error-json-decode'] += 1 + continue if self.batch_size: batch.append(record) if len(batch) >= self.batch_size: |