diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2021-12-15 13:14:39 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2021-12-15 13:14:41 -0800 |
| commit | dd4a80c2ae16bcc7e6318886e68cea7a1a59da7d (patch) | |
| tree | c43e4a86f3b366f6009be5dd22292f7b32e2a494 | |
| parent | 2cd2ca6f7a0424af3e850e113f2745b1e5555e4c (diff) | |
| download | sandcrawler-dd4a80c2ae16bcc7e6318886e68cea7a1a59da7d.tar.gz sandcrawler-dd4a80c2ae16bcc7e6318886e68cea7a1a59da7d.zip | |
fileset ingest: configure IA session from env
Note that this doesn't currently work for `upload()`, and as a
work-around I created `~/.config/ia.ini` manually on the worker VM.
| -rw-r--r-- | python/sandcrawler/fileset_strategies.py | 7 |
1 files changed, 6 insertions, 1 deletions
diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py index 6e4ad10..29b7edb 100644 --- a/python/sandcrawler/fileset_strategies.py +++ b/python/sandcrawler/fileset_strategies.py @@ -40,7 +40,12 @@ class ArchiveorgFilesetStrategy(FilesetIngestStrategy): except FileExistsError: pass - self.ia_session = internetarchive.get_session() + self.ia_session = internetarchive.get_session(config={ + 's3': { + 'access': os.environ.get("IA_ACCESS_KEY"), + 'secret': os.environ.get("IA_SECRET_KEY"), + }, + }) def check_existing(self, item: FilesetPlatformItem) -> Optional[ArchiveStrategyResult]: """ |