aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-01-10 16:06:26 -0800
committerBryan Newbold <bnewbold@archive.org>2020-01-10 16:06:26 -0800
commit49f5027eadfa76d1cd0072347bd887b436e0cbed (patch)
treee1cf043bf48f37bd7459fd87792e1dbfcb77c3b4 /python
parentbe114d6a8e7bd51f7b4336cc1c5529ec2cc00f67 (diff)
downloadsandcrawler-49f5027eadfa76d1cd0072347bd887b436e0cbed.tar.gz
sandcrawler-49f5027eadfa76d1cd0072347bd887b436e0cbed.zip
add sleep-and-retry workaround for CDX after SPNv2
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/ia.py10
1 files changed, 9 insertions, 1 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index db03a5d..27bbc34 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -131,7 +131,7 @@ class CdxApiClient:
rows.append(row)
return rows
- def fetch(self, url, datetime, filter_status_code=None):
+ def fetch(self, url, datetime, filter_status_code=None, retry_sleep=None):
"""
Fetches a single CDX row by url/datetime. Raises a KeyError if not
found, because we expect to be looking up a specific full record.
@@ -150,9 +150,17 @@ class CdxApiClient:
params['filter'] = "statuscode:{}".format(filter_status_code)
resp = self._query_api(params)
if not resp:
+ if retry_sleep:
+ print("CDX fetch failed; will sleep {}sec and try again".format(retry_sleep))
+ time.sleep(retry_sleep)
+ return self.fetch(url, datetime, filter_status_code=filter_status_code, retry_sleep=None)
raise KeyError("CDX url/datetime not found: {} {}".format(url, datetime))
row = resp[0]
if not (row.url == url and row.datetime == datetime):
+ if retry_sleep:
+ print("CDX fetch failed; will sleep {}sec and try again".format(retry_sleep))
+ time.sleep(retry_sleep)
+ return self.fetch(url, datetime, filter_status_code=filter_status_code, retry_sleep=None)
raise KeyError("Didn't get exact CDX url/datetime match. url:{} dt:{} got:{}".format(url, datetime, row))
if filter_status_code:
assert row.status_code == filter_status_code