From 100e5348c4203dd346993a185ba3749105e91541 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 15 Dec 2021 13:15:26 -0800 Subject: fileset ingest: create tmp subdirectories if needed --- python/sandcrawler/fileset_strategies.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'python') diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py index 29b7edb..b0131f4 100644 --- a/python/sandcrawler/fileset_strategies.py +++ b/python/sandcrawler/fileset_strategies.py @@ -117,8 +117,13 @@ class ArchiveorgFilesetStrategy(FilesetIngestStrategy): local_path = local_dir + "/" + m.path assert m.platform_url + if not os.path.exists(os.path.dirname(local_path)): + os.mkdir(os.path.dirname(local_path)) if not os.path.exists(local_path): print(f" downloading {m.path}", file=sys.stderr) + # create any sub-directories for this path, if necessary + if not os.path.exists(os.path.dirname(local_path)): + os.mkdir(os.path.dirname(local_path)) with self.ia_session.get( m.platform_url, stream=True, allow_redirects=True ) as r: -- cgit v1.2.3