From 5866adfc116cc779e869867ccebfe7adc70b4967 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 30 Mar 2020 13:40:02 -0700 Subject: deliver_file2disk: fewer retries, TooManyRedirects --- scripts/deliver_file2disk.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/deliver_file2disk.py b/scripts/deliver_file2disk.py index 9ec234a..f708b34 100755 --- a/scripts/deliver_file2disk.py +++ b/scripts/deliver_file2disk.py @@ -51,7 +51,7 @@ def gen_file_metadata(blob): mimetype=mimetype, ) -def requests_retry_session(retries=10, backoff_factor=3, +def requests_retry_session(retries=2, backoff_factor=3, status_forcelist=(500, 502, 504), session=None): """ From: https://www.peterbe.com/plog/best-practice-with-retries-with-requests @@ -143,6 +143,8 @@ class DeliverFatcatDisk: resp = self.session.get(url) except requests.exceptions.RetryError: return ('wayback-error', None) + except requests.exceptions.TooManyRedirects: + return ('too-many-redirects', None) if resp.status_code != 200: return ('fetch:{}'.format(resp.status_code), None) else: -- cgit v1.2.3