From dd4a80c2ae16bcc7e6318886e68cea7a1a59da7d Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 15 Dec 2021 13:14:39 -0800 Subject: fileset ingest: configure IA session from env Note that this doesn't currently work for `upload()`, and as a work-around I created `~/.config/ia.ini` manually on the worker VM. --- python/sandcrawler/fileset_strategies.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py index 6e4ad10..29b7edb 100644 --- a/python/sandcrawler/fileset_strategies.py +++ b/python/sandcrawler/fileset_strategies.py @@ -40,7 +40,12 @@ class ArchiveorgFilesetStrategy(FilesetIngestStrategy): except FileExistsError: pass - self.ia_session = internetarchive.get_session() + self.ia_session = internetarchive.get_session(config={ + 's3': { + 'access': os.environ.get("IA_ACCESS_KEY"), + 'secret': os.environ.get("IA_SECRET_KEY"), + }, + }) def check_existing(self, item: FilesetPlatformItem) -> Optional[ArchiveStrategyResult]: """ -- cgit v1.2.3