aboutsummaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-03-30 13:40:02 -0700
committerBryan Newbold <bnewbold@archive.org>2020-03-30 13:40:02 -0700
commit5866adfc116cc779e869867ccebfe7adc70b4967 (patch)
tree3061c63dbd633fc4fce9d8d160fd6da4c88db670 /scripts
parent80e92e6a114ddb2514dddf0b1a81038a18a2c93d (diff)
downloadfatcat-covid19-5866adfc116cc779e869867ccebfe7adc70b4967.tar.gz
fatcat-covid19-5866adfc116cc779e869867ccebfe7adc70b4967.zip
deliver_file2disk: fewer retries, TooManyRedirects
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/deliver_file2disk.py4
1 files changed, 3 insertions, 1 deletions
diff --git a/scripts/deliver_file2disk.py b/scripts/deliver_file2disk.py
index 9ec234a..f708b34 100755
--- a/scripts/deliver_file2disk.py
+++ b/scripts/deliver_file2disk.py
@@ -51,7 +51,7 @@ def gen_file_metadata(blob):
mimetype=mimetype,
)
-def requests_retry_session(retries=10, backoff_factor=3,
+def requests_retry_session(retries=2, backoff_factor=3,
status_forcelist=(500, 502, 504), session=None):
"""
From: https://www.peterbe.com/plog/best-practice-with-retries-with-requests
@@ -143,6 +143,8 @@ class DeliverFatcatDisk:
resp = self.session.get(url)
except requests.exceptions.RetryError:
return ('wayback-error', None)
+ except requests.exceptions.TooManyRedirects:
+ return ('too-many-redirects', None)
if resp.status_code != 200:
return ('fetch:{}'.format(resp.status_code), None)
else: