aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-08-11 18:41:00 -0700
committerBryan Newbold <bnewbold@archive.org>2020-08-11 18:41:05 -0700
commit37ecbf4f3bd09aea1fd4a9d7bfb90b4b54202ec2 (patch)
tree7fef947fcd882cdb1ed7776dabcafe351278391d
parent92bf9bc28ac0eacab2e06fa3b25b52f0882804c2 (diff)
downloadsandcrawler-37ecbf4f3bd09aea1fd4a9d7bfb90b4b54202ec2.tar.gz
sandcrawler-37ecbf4f3bd09aea1fd4a9d7bfb90b4b54202ec2.zip
Revert "ingest: reduce CDX retry_sleep to 3.0 sec (after SPN)"
This reverts commit 92bf9bc28ac0eacab2e06fa3b25b52f0882804c2. In practice, in prod, this resulted in much larger spn2-cdx-lookup-failure error rates.
-rw-r--r--python/sandcrawler/ia.py2
1 files changed, 1 insertions, 1 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 87e03a4..7b623bc 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -947,7 +947,7 @@ class SavePageNowClient:
url=spn_result.terminal_url,
datetime=spn_result.terminal_dt,
filter_status_code=filter_status_code,
- retry_sleep=3.0,
+ retry_sleep=10.0,
)
except KeyError as ke:
print("CDX KeyError: {}".format(ke), file=sys.stderr)