diff options
author | Bryan Newbold <bnewbold@archive.org> | 2020-03-30 13:40:02 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2020-03-30 13:40:02 -0700 |
commit | 5866adfc116cc779e869867ccebfe7adc70b4967 (patch) | |
tree | 3061c63dbd633fc4fce9d8d160fd6da4c88db670 | |
parent | 80e92e6a114ddb2514dddf0b1a81038a18a2c93d (diff) | |
download | fatcat-covid19-5866adfc116cc779e869867ccebfe7adc70b4967.tar.gz fatcat-covid19-5866adfc116cc779e869867ccebfe7adc70b4967.zip |
deliver_file2disk: fewer retries, TooManyRedirects
-rwxr-xr-x | scripts/deliver_file2disk.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/scripts/deliver_file2disk.py b/scripts/deliver_file2disk.py index 9ec234a..f708b34 100755 --- a/scripts/deliver_file2disk.py +++ b/scripts/deliver_file2disk.py @@ -51,7 +51,7 @@ def gen_file_metadata(blob): mimetype=mimetype, ) -def requests_retry_session(retries=10, backoff_factor=3, +def requests_retry_session(retries=2, backoff_factor=3, status_forcelist=(500, 502, 504), session=None): """ From: https://www.peterbe.com/plog/best-practice-with-retries-with-requests @@ -143,6 +143,8 @@ class DeliverFatcatDisk: resp = self.session.get(url) except requests.exceptions.RetryError: return ('wayback-error', None) + except requests.exceptions.TooManyRedirects: + return ('too-many-redirects', None) if resp.status_code != 200: return ('fetch:{}'.format(resp.status_code), None) else: |