diff options
author | Bryan Newbold <bnewbold@robocracy.org> | 2018-11-14 23:23:07 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@robocracy.org> | 2018-11-14 23:23:07 -0800 |
commit | ef50d665b31a8cb1b0ddc38671b3334b0fa6fc14 (patch) | |
tree | ea7d4fc799746df9b6afa6715ecc155ca054d796 /python/fatcat_tools/workers/changelog.py | |
parent | a1b63a58410f01f93027f25e8041d0aed6317bbc (diff) | |
download | fatcat-ef50d665b31a8cb1b0ddc38671b3334b0fa6fc14.tar.gz fatcat-ef50d665b31a8cb1b0ddc38671b3334b0fa6fc14.zip |
fix worker code
Diffstat (limited to 'python/fatcat_tools/workers/changelog.py')
-rw-r--r-- | python/fatcat_tools/workers/changelog.py | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index ccfbcf19..d07b5988 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -1,7 +1,6 @@ import json import time -from itertools import islice from fatcat_tools.workers.worker_common import FatcatWorker, most_recent_message from pykafka.common import OffsetType @@ -33,7 +32,7 @@ class FatcatChangelogWorker(FatcatWorker): else: self.offset = 1 - with topic.get_sync_producer() as producer: + with topic.get_producer() as producer: while True: latest = int(self.api.get_changelog(limit=1)[0].index) if latest > self.offset: @@ -83,6 +82,8 @@ class FatcatEntityUpdatesWorker(FatcatWorker): compacted_topic=True, ) + # using a sync producer to try and avoid racey loss of delivery (aka, + # if consumer group updated but produce didn't stick) with release_topic.get_sync_producer() as producer: for msg in consumer: cle = json.loads(msg.value.decode('utf-8')) |