diff options
| -rw-r--r-- | python/sandcrawler/ia.py | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index 99a7f36..d334d24 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -395,6 +395,7 @@ class WaybackClient: self.replay_headers = { "User-Agent": "Mozilla/5.0 sandcrawler.WaybackClient", } + self.http_session = requests_retry_session() def fetch_petabox( self, csize: int, offset: int, warc_path: str, resolve_revisit: bool = True @@ -603,7 +604,7 @@ class WaybackClient: assert datetime.isdigit() try: - resp = requests.get( + resp = self.http_session.get( self.wayback_endpoint + datetime + "id_/" + url, allow_redirects=False, headers=self.replay_headers, @@ -670,7 +671,7 @@ class WaybackClient: assert datetime.isdigit() try: - resp = requests.get( + resp = self.http_session.get( self.wayback_endpoint + datetime + "id_/" + url, allow_redirects=False, headers=self.replay_headers, |