diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2021-10-01 15:07:20 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2021-10-01 15:07:20 -0700 |
| commit | dd219464cfc90b9b469fd851b48b08668ff17ba8 (patch) | |
| tree | 492e7980794aa61f1c56b39f7fbf0290662f8b66 /python/fatcat_tools/importers | |
| parent | 519c7e77cf3a54b9620adef07fedac9b37a5f9f2 (diff) | |
| download | fatcat-dd219464cfc90b9b469fd851b48b08668ff17ba8.tar.gz fatcat-dd219464cfc90b9b469fd851b48b08668ff17ba8.zip | |
importer common: more verbose logging (with counts)
Diffstat (limited to 'python/fatcat_tools/importers')
| -rw-r--r-- | python/fatcat_tools/importers/common.py | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 6815a155..6ca9a50c 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -732,8 +732,8 @@ class KafkaBs4XmlPusher(RecordPusher): batch = self.consumer.consume( num_messages=self.consume_batch_size, timeout=self.poll_interval) - print("... got {} kafka messages ({}sec poll interval)".format( - len(batch), self.poll_interval)) + print("... got {} kafka messages ({}sec poll interval) {}".format( + len(batch), self.poll_interval, self.importer.counts)) if not batch: if datetime.datetime.now() - last_push > datetime.timedelta(minutes=5): # it has been some time, so flush any current editgroup @@ -796,8 +796,8 @@ class KafkaJsonPusher(RecordPusher): batch = self.consumer.consume( num_messages=self.consume_batch_size, timeout=self.poll_interval) - print("... got {} kafka messages ({}sec poll interval)".format( - len(batch), self.poll_interval)) + print("... got {} kafka messages ({}sec poll interval) {}".format( + len(batch), self.poll_interval, self.importer.counts)) if not batch: if datetime.datetime.now() - last_push > datetime.timedelta(minutes=5): # it has been some time, so flush any current editgroup |