aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-06-17 18:06:12 -0700
committerBryan Newbold <bnewbold@archive.org>2020-06-17 18:06:12 -0700
commit386cb8335d4d1a66b75301a244f7baed49658588 (patch)
treea837ded7f4579ca7d9adcbd93f711347c7455b86 /python
parent815c2d115bbc2a64595a682bd15b95beac497c82 (diff)
downloadsandcrawler-386cb8335d4d1a66b75301a244f7baed49658588.tar.gz
sandcrawler-386cb8335d4d1a66b75301a244f7baed49658588.zip
tweak kafka topic names and seaweedfs layout
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/persist.py3
-rwxr-xr-xpython/sandcrawler_worker.py19
2 files changed, 12 insertions, 10 deletions
diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py
index 196c4b9..8d421ad 100644
--- a/python/sandcrawler/persist.py
+++ b/python/sandcrawler/persist.py
@@ -452,6 +452,7 @@ class PersistThumbnailWorker(SandcrawlerWorker):
default_bucket=kwargs['s3_bucket'],
)
self.s3_extension = kwargs.get('s3_extension', ".jpg")
+ self.s3_folder = kwargs.get('s3_folder', "pdf")
def process(self, blob, key=None):
"""
@@ -463,7 +464,7 @@ class PersistThumbnailWorker(SandcrawlerWorker):
assert len(blob) >= 50
resp = self.s3.put_blob(
- folder="thumbnail",
+ folder=self.s3_folder,
blob=blob,
sha1hex=key,
extension=self.s3_extension,
diff --git a/python/sandcrawler_worker.py b/python/sandcrawler_worker.py
index 950eb4b..e18d883 100755
--- a/python/sandcrawler_worker.py
+++ b/python/sandcrawler_worker.py
@@ -71,9 +71,9 @@ def run_persist_grobid(args):
pusher.run()
def run_pdf_extract(args):
- consume_topic = "sandcrawler-{}.unextracted-pg".format(args.env)
- text_topic = "sandcrawler-{}.pdftext".format(args.env)
- thumbnail_topic = "sandcrawler-{}.thumbnail-180px-jpeg".format(args.env)
+ consume_topic = "sandcrawler-{}.unextracted".format(args.env)
+ text_topic = "sandcrawler-{}.pdf-text".format(args.env)
+ thumbnail_topic = "sandcrawler-{}.pdf-thumbnail-180px-jpg".format(args.env)
text_sink = KafkaSink(
kafka_hosts=args.kafka_hosts,
produce_topic=text_topic,
@@ -100,7 +100,7 @@ def run_pdf_extract(args):
pusher.run()
def run_persist_pdftext(args):
- consume_topic = "sandcrawler-{}.pdftext".format(args.env)
+ consume_topic = "sandcrawler-{}.pdf-text".format(args.env)
worker = PersistPdfTextWorker(
db_url=args.db_url,
s3_url=args.s3_url,
@@ -114,27 +114,28 @@ def run_persist_pdftext(args):
worker=worker,
kafka_hosts=args.kafka_hosts,
consume_topic=consume_topic,
- group="persist-pdftext",
+ group="persist-pdf-text",
push_batches=True,
batch_size=25,
)
pusher.run()
def run_persist_thumbnail(args):
- consume_topic = "sandcrawler-{}.thumbnail".format(args.env)
+ consume_topic = "sandcrawler-{}.pdf-thumbnail-180px-jpg".format(args.env)
worker = PersistThumbnailWorker(
s3_url=args.s3_url,
s3_bucket=args.s3_bucket,
s3_access_key=args.s3_access_key,
s3_secret_key=args.s3_secret_key,
- # TODO: s3_extension=args.s3_extension,
+ s3_extension=".180px.jpg",
+ s3_folder="pdf",
)
pusher = KafkaJsonPusher(
worker=worker,
kafka_hosts=args.kafka_hosts,
consume_topic=consume_topic,
- group="persist-thumbnail",
- raw_records=True,
+ group="persist-pdf-thumbnail",
+ raw_record=True,
batch_size=25,
)
pusher.run()