From 92bf9bc28ac0eacab2e06fa3b25b52f0882804c2 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 11 Aug 2020 17:40:56 -0700 Subject: ingest: reduce CDX retry_sleep to 3.0 sec (after SPN) As we are moving towards just retrying entire ingest requests, we should probably just make this zero. But until then we should give SPN CDX a small chance to sync before giving up. This change expected to improve overall throughput. --- python/sandcrawler/ia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index 7b623bc..87e03a4 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -947,7 +947,7 @@ class SavePageNowClient: url=spn_result.terminal_url, datetime=spn_result.terminal_dt, filter_status_code=filter_status_code, - retry_sleep=10.0, + retry_sleep=3.0, ) except KeyError as ke: print("CDX KeyError: {}".format(ke), file=sys.stderr) -- cgit v1.2.3