diff options
author | Martin Czygan <martin@archive.org> | 2020-07-06 18:53:24 +0000 |
---|---|---|
committer | Martin Czygan <martin@archive.org> | 2020-07-06 18:53:24 +0000 |
commit | 68cf95bd1d1588c0d3170b4032596756e07ae718 (patch) | |
tree | 3163a803dd6743c84c83a786b5aea7eda3bbca8e /python/fatcat_tools/harvest/harvest_common.py | |
parent | bea909f997bcef51e2624b9eea42c8fbe7115aaa (diff) | |
parent | 8583c6866f2bb89e8dfe5f5e5893048c2fd854e7 (diff) | |
download | fatcat-68cf95bd1d1588c0d3170b4032596756e07ae718.tar.gz fatcat-68cf95bd1d1588c0d3170b4032596756e07ae718.zip |
Merge branch 'bnewbold-lint' into 'master'
lint cleanups
See merge request webgroup/fatcat!62
Diffstat (limited to 'python/fatcat_tools/harvest/harvest_common.py')
-rw-r--r-- | python/fatcat_tools/harvest/harvest_common.py | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/python/fatcat_tools/harvest/harvest_common.py b/python/fatcat_tools/harvest/harvest_common.py index 27ab8b4a..bdae3054 100644 --- a/python/fatcat_tools/harvest/harvest_common.py +++ b/python/fatcat_tools/harvest/harvest_common.py @@ -1,15 +1,13 @@ import sys import json -import time import datetime import requests from requests.adapters import HTTPAdapter # unclear why pylint chokes on this import. Recent 'requests' and 'urllib3' are # in Pipenv.lock, and there are no errors in QA from requests.packages.urllib3.util.retry import Retry # pylint: disable=import-error -from confluent_kafka import Producer, Consumer, TopicPartition, KafkaException, \ - OFFSET_BEGINNING +from confluent_kafka import Producer, Consumer, TopicPartition, KafkaException # Used for parsing ISO date format (YYYY-MM-DD) @@ -130,9 +128,11 @@ class HarvestState: }).encode('utf-8') if kafka_topic: assert(kafka_config) + def fail_fast(err, msg): if err: raise KafkaException(err) + print("Committing status to Kafka: {}".format(kafka_topic), file=sys.stderr) producer_conf = kafka_config.copy() producer_conf.update({ @@ -159,9 +159,11 @@ class HarvestState: return print("Fetching state from kafka topic: {}".format(kafka_topic), file=sys.stderr) + def fail_fast(err, msg): if err: raise KafkaException(err) + conf = kafka_config.copy() conf.update({ 'group.id': 'dummy_init_group', # should never be committed |