diff options
Diffstat (limited to 'python/fatcat_tools/importers/common.py')
| -rw-r--r-- | python/fatcat_tools/importers/common.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 4a3cd648..13b1e5b8 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -6,6 +6,7 @@ import json import ftfy import base64 import sqlite3 +import datetime import subprocess import unicodedata from collections import Counter @@ -756,10 +757,11 @@ class KafkaJsonPusher(RecordPusher): print("... got {} kafka messages ({}sec poll interval)".format( len(batch), self.poll_interval)) if not batch: - if datetime.datetime.now() - last_push > datetime.timedelta(minutes=5): + if datetime.datetime.now() - last_push > datetime.timedelta(seconds=30): #XXX minutes=5 # it has been some time, so flush any current editgroup self.importer.finish() last_push = datetime.datetime.now() + #print("Flushed any partial import batch: {}".format(self.importer.counts)) continue # first check errors on entire batch... for msg in batch: |