summaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/workers/elastic.py
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-11-15 13:11:52 -0800
committerBryan Newbold <bnewbold@robocracy.org>2018-11-15 13:15:15 -0800
commitbb28a3fc1cc900f2dde31e1dbc492d9661034f41 (patch)
treef037dd3d1bab6cbf08a562dbdd4c09361fe0c030 /python/fatcat_tools/workers/elastic.py
parent9f817c6c70a749f2ac449ab4edfd26c6dd8a7410 (diff)
downloadfatcat-bb28a3fc1cc900f2dde31e1dbc492d9661034f41.tar.gz
fatcat-bb28a3fc1cc900f2dde31e1dbc492d9661034f41.zip
large refactor of python names/paths
- Add __init__.py files for fatcat_tools submodules, and use them in imports - Add a bunch of comments to files. - rename a number of classes and functions to be less verbose
Diffstat (limited to 'python/fatcat_tools/workers/elastic.py')
-rw-r--r--python/fatcat_tools/workers/elastic.py52
1 files changed, 0 insertions, 52 deletions
diff --git a/python/fatcat_tools/workers/elastic.py b/python/fatcat_tools/workers/elastic.py
deleted file mode 100644
index 3a75a1b3..00000000
--- a/python/fatcat_tools/workers/elastic.py
+++ /dev/null
@@ -1,52 +0,0 @@
-
-import json
-import time
-import requests
-from fatcat_tools.transforms import release_elastic_dict
-from fatcat_tools.workers.worker_common import FatcatWorker
-from fatcat_client import ReleaseEntity
-from fatcat_tools.transforms import *
-from pykafka.common import OffsetType
-
-
-class FatcatElasticReleaseWorker(FatcatWorker):
- """
- Consumes from release-updates topic and pushes into (presumably local)
- elasticsearch.
-
- Uses a consumer group to manage offset.
- """
-
- def __init__(self, kafka_hosts, consume_topic, poll_interval=10.0, offset=None,
- elastic_backend="http://localhost:9200", elastic_index="fatcat"):
- super().__init__(kafka_hosts=kafka_hosts,
- consume_topic=consume_topic,
- api_host_url=None)
- self.consumer_group = "elastic-updates"
- self.elastic_backend = elastic_backend
- self.elastic_index = elastic_index
-
- def run(self):
- consume_topic = self.kafka.topics[self.consume_topic]
-
- consumer = consume_topic.get_balanced_consumer(
- consumer_group=self.consumer_group,
- managed=True,
- fetch_message_max_bytes=4000000, # up to ~4MBytes
- auto_commit_enable=True,
- auto_commit_interval_ms=30000, # 30 seconds
- compacted_topic=True,
- )
-
- for msg in consumer:
- json_str = msg.value.decode('utf-8')
- release = entity_from_json(json_str, ReleaseEntity)
- #print(release)
- elastic_endpoint = "{}/{}/release/{}".format(
- self.elastic_backend,
- self.elastic_index,
- release.ident)
- print("Updating document: {}".format(elastic_endpoint))
- resp = requests.post(elastic_endpoint, json=release_elastic_dict(release))
- assert resp.status_code in (200, 201)
- #consumer.commit_offsets()