aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/fileset_strategies.py5
1 files changed, 5 insertions, 0 deletions
diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py
index 29b7edb..b0131f4 100644
--- a/python/sandcrawler/fileset_strategies.py
+++ b/python/sandcrawler/fileset_strategies.py
@@ -117,8 +117,13 @@ class ArchiveorgFilesetStrategy(FilesetIngestStrategy):
local_path = local_dir + "/" + m.path
assert m.platform_url
+ if not os.path.exists(os.path.dirname(local_path)):
+ os.mkdir(os.path.dirname(local_path))
if not os.path.exists(local_path):
print(f" downloading {m.path}", file=sys.stderr)
+ # create any sub-directories for this path, if necessary
+ if not os.path.exists(os.path.dirname(local_path)):
+ os.mkdir(os.path.dirname(local_path))
with self.ia_session.get(
m.platform_url, stream=True, allow_redirects=True
) as r: