From 5cc02ee97b39d72c88dd2e0755abf6f0d3a4b050 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Tue, 13 Jul 2021 12:34:41 -0700
Subject: crawl: SPN self-redirect hack

---
 python/sandcrawler/ia.py | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'python')

diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 9b2635b..166cf20 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -1025,6 +1025,15 @@ class SavePageNowClient:
                     filter_status_code=filter_status_code,
                     retry_sleep=9.0,
                 )
+                # sometimes there are fuzzy http/https self-redirects with the
+                # same SURT; try to work around that
+                if cdx_row.status_code >= 300 and cdx_row.status_code < 400:
+                    cdx_row = wayback_client.cdx_client.fetch(
+                        url=spn_result.terminal_url,
+                        datetime=spn_result.terminal_dt,
+                        filter_status_code=200,
+                        retry_sleep=9.0,
+                    )
             except KeyError as ke:
                 print("  CDX KeyError: {}".format(ke), file=sys.stderr)
                 return ResourceResult(
-- 
cgit v1.2.3