From dd219464cfc90b9b469fd851b48b08668ff17ba8 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 1 Oct 2021 15:07:20 -0700 Subject: importer common: more verbose logging (with counts) --- python/fatcat_tools/importers/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 6815a155..6ca9a50c 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -732,8 +732,8 @@ class KafkaBs4XmlPusher(RecordPusher): batch = self.consumer.consume( num_messages=self.consume_batch_size, timeout=self.poll_interval) - print("... got {} kafka messages ({}sec poll interval)".format( - len(batch), self.poll_interval)) + print("... got {} kafka messages ({}sec poll interval) {}".format( + len(batch), self.poll_interval, self.importer.counts)) if not batch: if datetime.datetime.now() - last_push > datetime.timedelta(minutes=5): # it has been some time, so flush any current editgroup @@ -796,8 +796,8 @@ class KafkaJsonPusher(RecordPusher): batch = self.consumer.consume( num_messages=self.consume_batch_size, timeout=self.poll_interval) - print("... got {} kafka messages ({}sec poll interval)".format( - len(batch), self.poll_interval)) + print("... got {} kafka messages ({}sec poll interval) {}".format( + len(batch), self.poll_interval, self.importer.counts)) if not batch: if datetime.datetime.now() - last_push > datetime.timedelta(minutes=5): # it has been some time, so flush any current editgroup -- cgit v1.2.3