aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/ia.py5
1 files changed, 4 insertions, 1 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 52ff4d5..39227ca 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -68,7 +68,10 @@ class CdxApiClient:
sha1hex=b32_hex(cdx[5]),
)
if follow_redirects and cdx['http_status'] in (301, 302):
- resp = requests.get(self.wayback_endpoint + cdx['datetime'] + "id_/" + cdx['url'])
+ try:
+ resp = requests.get(self.wayback_endpoint + cdx['datetime'] + "id_/" + cdx['url'])
+ except requests.exceptions.TooManyRedirects:
+ raise CdxApiError("redirect loop (wayback fetch)")
next_url = '/'.join(resp.url.split('/')[5:])
if next_url == url:
raise CdxApiError("redirect loop (by url)")