From ae531a3314742deb1bdd2560ffbcaa2d1f8d829b Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 5 Aug 2020 13:06:08 -0700 Subject: spn2: skip js behavior (experiment) Hoping this will increase crawling throughput with little-to-no impact on fidelity. --- python/sandcrawler/ia.py | 1 + 1 file changed, 1 insertion(+) (limited to 'python/sandcrawler/ia.py') diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index 63abb45..150de53 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -814,6 +814,7 @@ class SavePageNowClient: 'force_get': force_get, 'skip_first_archive': 1, 'outlinks_availability': 0, + 'js_behavior_timeout': 0, }, ) if resp.status_code == 429: -- cgit v1.2.3