From 386cb8335d4d1a66b75301a244f7baed49658588 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 17 Jun 2020 18:06:12 -0700 Subject: tweak kafka topic names and seaweedfs layout --- python/sandcrawler/persist.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'python/sandcrawler/persist.py') diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py index 196c4b9..8d421ad 100644 --- a/python/sandcrawler/persist.py +++ b/python/sandcrawler/persist.py @@ -452,6 +452,7 @@ class PersistThumbnailWorker(SandcrawlerWorker): default_bucket=kwargs['s3_bucket'], ) self.s3_extension = kwargs.get('s3_extension', ".jpg") + self.s3_folder = kwargs.get('s3_folder', "pdf") def process(self, blob, key=None): """ @@ -463,7 +464,7 @@ class PersistThumbnailWorker(SandcrawlerWorker): assert len(blob) >= 50 resp = self.s3.put_blob( - folder="thumbnail", + folder=self.s3_folder, blob=blob, sha1hex=key, extension=self.s3_extension, -- cgit v1.2.3