aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-01-15 23:20:37 -0800
committerBryan Newbold <bnewbold@archive.org>2020-01-15 23:20:37 -0800
commit246f8033ba189bdf3ddbf64b2cb851d86ec43b75 (patch)
tree4e1ee5ff2f057289c22ca9d3321e60f05b0b8925
parent34ba59a446a43ecf32dbfded5e330cf4178ba355 (diff)
downloadsandcrawler-246f8033ba189bdf3ddbf64b2cb851d86ec43b75.tar.gz
sandcrawler-246f8033ba189bdf3ddbf64b2cb851d86ec43b75.zip
handle UnicodeDecodeError in the other GET instance
-rw-r--r--python/sandcrawler/ia.py2
1 files changed, 2 insertions, 0 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 9fdb52e..02258df 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -448,6 +448,8 @@ class WaybackClient:
)
except requests.exceptions.TooManyRedirects:
raise WaybackError("redirect loop (wayback replay fetch)")
+ except UnicodeDecodeError:
+ raise WaybackError("UnicodeDecodeError in replay request (can mean nasty redirect URL): {}".format(url))
try:
resp.raise_for_status()
except Exception as e: