diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-06-17 18:06:12 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-06-17 18:06:12 -0700 |
| commit | 386cb8335d4d1a66b75301a244f7baed49658588 (patch) | |
| tree | a837ded7f4579ca7d9adcbd93f711347c7455b86 /python/sandcrawler/persist.py | |
| parent | 815c2d115bbc2a64595a682bd15b95beac497c82 (diff) | |
| download | sandcrawler-386cb8335d4d1a66b75301a244f7baed49658588.tar.gz sandcrawler-386cb8335d4d1a66b75301a244f7baed49658588.zip | |
tweak kafka topic names and seaweedfs layout
Diffstat (limited to 'python/sandcrawler/persist.py')
| -rw-r--r-- | python/sandcrawler/persist.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py index 196c4b9..8d421ad 100644 --- a/python/sandcrawler/persist.py +++ b/python/sandcrawler/persist.py @@ -452,6 +452,7 @@ class PersistThumbnailWorker(SandcrawlerWorker): default_bucket=kwargs['s3_bucket'], ) self.s3_extension = kwargs.get('s3_extension', ".jpg") + self.s3_folder = kwargs.get('s3_folder', "pdf") def process(self, blob, key=None): """ @@ -463,7 +464,7 @@ class PersistThumbnailWorker(SandcrawlerWorker): assert len(blob) >= 50 resp = self.s3.put_blob( - folder="thumbnail", + folder=self.s3_folder, blob=blob, sha1hex=key, extension=self.s3_extension, |