summaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/harvest/harvest_common.py
diff options
context:
space:
mode:
authorMartin Czygan <martin@archive.org>2020-07-06 18:53:24 +0000
committerMartin Czygan <martin@archive.org>2020-07-06 18:53:24 +0000
commit68cf95bd1d1588c0d3170b4032596756e07ae718 (patch)
tree3163a803dd6743c84c83a786b5aea7eda3bbca8e /python/fatcat_tools/harvest/harvest_common.py
parentbea909f997bcef51e2624b9eea42c8fbe7115aaa (diff)
parent8583c6866f2bb89e8dfe5f5e5893048c2fd854e7 (diff)
downloadfatcat-68cf95bd1d1588c0d3170b4032596756e07ae718.tar.gz
fatcat-68cf95bd1d1588c0d3170b4032596756e07ae718.zip
Merge branch 'bnewbold-lint' into 'master'
lint cleanups See merge request webgroup/fatcat!62
Diffstat (limited to 'python/fatcat_tools/harvest/harvest_common.py')
-rw-r--r--python/fatcat_tools/harvest/harvest_common.py8
1 files changed, 5 insertions, 3 deletions
diff --git a/python/fatcat_tools/harvest/harvest_common.py b/python/fatcat_tools/harvest/harvest_common.py
index 27ab8b4a..bdae3054 100644
--- a/python/fatcat_tools/harvest/harvest_common.py
+++ b/python/fatcat_tools/harvest/harvest_common.py
@@ -1,15 +1,13 @@
import sys
import json
-import time
import datetime
import requests
from requests.adapters import HTTPAdapter
# unclear why pylint chokes on this import. Recent 'requests' and 'urllib3' are
# in Pipenv.lock, and there are no errors in QA
from requests.packages.urllib3.util.retry import Retry # pylint: disable=import-error
-from confluent_kafka import Producer, Consumer, TopicPartition, KafkaException, \
- OFFSET_BEGINNING
+from confluent_kafka import Producer, Consumer, TopicPartition, KafkaException
# Used for parsing ISO date format (YYYY-MM-DD)
@@ -130,9 +128,11 @@ class HarvestState:
}).encode('utf-8')
if kafka_topic:
assert(kafka_config)
+
def fail_fast(err, msg):
if err:
raise KafkaException(err)
+
print("Committing status to Kafka: {}".format(kafka_topic), file=sys.stderr)
producer_conf = kafka_config.copy()
producer_conf.update({
@@ -159,9 +159,11 @@ class HarvestState:
return
print("Fetching state from kafka topic: {}".format(kafka_topic), file=sys.stderr)
+
def fail_fast(err, msg):
if err:
raise KafkaException(err)
+
conf = kafka_config.copy()
conf.update({
'group.id': 'dummy_init_group', # should never be committed