summaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/importers/common.py
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-12-12 17:50:12 -0800
committerBryan Newbold <bnewbold@robocracy.org>2019-12-12 19:49:50 -0800
commit91662c063b088bb665b86c252ecd27be9d9083be (patch)
tree6613d45fbfc7b63c67adff8b4f6aa1d531eda12b /python/fatcat_tools/importers/common.py
parent0827b67a3f195f151a77ca01708e6c98daf778bf (diff)
downloadfatcat-91662c063b088bb665b86c252ecd27be9d9083be.tar.gz
fatcat-91662c063b088bb665b86c252ecd27be9d9083be.zip
ensure importer description arg isn't clobbered
Diffstat (limited to 'python/fatcat_tools/importers/common.py')
-rw-r--r--python/fatcat_tools/importers/common.py4
1 files changed, 3 insertions, 1 deletions
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index 4a3cd648..13b1e5b8 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -6,6 +6,7 @@ import json
import ftfy
import base64
import sqlite3
+import datetime
import subprocess
import unicodedata
from collections import Counter
@@ -756,10 +757,11 @@ class KafkaJsonPusher(RecordPusher):
print("... got {} kafka messages ({}sec poll interval)".format(
len(batch), self.poll_interval))
if not batch:
- if datetime.datetime.now() - last_push > datetime.timedelta(minutes=5):
+ if datetime.datetime.now() - last_push > datetime.timedelta(seconds=30): #XXX minutes=5
# it has been some time, so flush any current editgroup
self.importer.finish()
last_push = datetime.datetime.now()
+ #print("Flushed any partial import batch: {}".format(self.importer.counts))
continue
# first check errors on entire batch...
for msg in batch: