aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-03-05 00:40:00 -0800
committerBryan Newbold <bnewbold@archive.org>2020-03-05 00:40:00 -0800
commit31f2545eb8af78cde9e4e4178489a8630aca0d09 (patch)
tree92f63af1bb45c9a18881524e693995bc003d77a9 /python
parentb182e84b5f3e1ce5732bed657dc0d12fd3257537 (diff)
downloadsandcrawler-31f2545eb8af78cde9e4e4178489a8630aca0d09.tar.gz
sandcrawler-31f2545eb8af78cde9e4e4178489a8630aca0d09.zip
ia: catch wayback ChunkedEncodingError
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/ia.py3
1 files changed, 3 insertions, 0 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 267ecdc..99e92be 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -467,8 +467,11 @@ class WaybackClient:
)
except requests.exceptions.TooManyRedirects:
raise WaybackError("redirect loop (wayback replay fetch)")
+ except requests.exceptions.ChunkedEncodingError:
+ raise WaybackError("ChunkedEncodingError (wayback replay fetch)")
except UnicodeDecodeError:
raise WaybackError("UnicodeDecodeError in replay request (can mean nasty redirect URL): {}".format(url))
+
try:
resp.raise_for_status()
except Exception as e: