aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2021-12-15 13:14:39 -0800
committerBryan Newbold <bnewbold@archive.org>2021-12-15 13:14:41 -0800
commitdd4a80c2ae16bcc7e6318886e68cea7a1a59da7d (patch)
treec43e4a86f3b366f6009be5dd22292f7b32e2a494 /python
parent2cd2ca6f7a0424af3e850e113f2745b1e5555e4c (diff)
downloadsandcrawler-dd4a80c2ae16bcc7e6318886e68cea7a1a59da7d.tar.gz
sandcrawler-dd4a80c2ae16bcc7e6318886e68cea7a1a59da7d.zip
fileset ingest: configure IA session from env
Note that this doesn't currently work for `upload()`, and as a work-around I created `~/.config/ia.ini` manually on the worker VM.
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/fileset_strategies.py7
1 files changed, 6 insertions, 1 deletions
diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py
index 6e4ad10..29b7edb 100644
--- a/python/sandcrawler/fileset_strategies.py
+++ b/python/sandcrawler/fileset_strategies.py
@@ -40,7 +40,12 @@ class ArchiveorgFilesetStrategy(FilesetIngestStrategy):
except FileExistsError:
pass
- self.ia_session = internetarchive.get_session()
+ self.ia_session = internetarchive.get_session(config={
+ 's3': {
+ 'access': os.environ.get("IA_ACCESS_KEY"),
+ 'secret': os.environ.get("IA_SECRET_KEY"),
+ },
+ })
def check_existing(self, item: FilesetPlatformItem) -> Optional[ArchiveStrategyResult]:
"""