aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-08-05 12:56:22 -0700
committerBryan Newbold <bnewbold@archive.org>2020-08-05 12:56:22 -0700
commit576b52831d9f17adaee9839db20b4145ba141d96 (patch)
treedc5c44a2d6017d2ee023488f49855a7934b1afa6
parent5505aa792d9eef6b4a5205249f2db2676d06074e (diff)
downloadsandcrawler-576b52831d9f17adaee9839db20b4145ba141d96.tar.gz
sandcrawler-576b52831d9f17adaee9839db20b4145ba141d96.zip
SPN2: ensure not fetching outlinks
-rw-r--r--python/sandcrawler/ia.py1
1 files changed, 1 insertions, 0 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 1b51dc9..63abb45 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -813,6 +813,7 @@ class SavePageNowClient:
'if_not_archived_within': '1d',
'force_get': force_get,
'skip_first_archive': 1,
+ 'outlinks_availability': 0,
},
)
if resp.status_code == 429: