aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-08-11 17:40:56 -0700
committerBryan Newbold <bnewbold@archive.org>2020-08-11 17:40:59 -0700
commit92bf9bc28ac0eacab2e06fa3b25b52f0882804c2 (patch)
tree6db86c3a90c76a7027ea787375dcad131e5470da
parent644e412c38c8897e171e3aa1244f1aa6955d8e65 (diff)
downloadsandcrawler-92bf9bc28ac0eacab2e06fa3b25b52f0882804c2.tar.gz
sandcrawler-92bf9bc28ac0eacab2e06fa3b25b52f0882804c2.zip
ingest: reduce CDX retry_sleep to 3.0 sec (after SPN)
As we are moving towards just retrying entire ingest requests, we should probably just make this zero. But until then we should give SPN CDX a small chance to sync before giving up. This change expected to improve overall throughput.
-rw-r--r--python/sandcrawler/ia.py2
1 files changed, 1 insertions, 1 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 7b623bc..87e03a4 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -947,7 +947,7 @@ class SavePageNowClient:
url=spn_result.terminal_url,
datetime=spn_result.terminal_dt,
filter_status_code=filter_status_code,
- retry_sleep=10.0,
+ retry_sleep=3.0,
)
except KeyError as ke:
print("CDX KeyError: {}".format(ke), file=sys.stderr)