From 891299fd461b17c60fb48364cd5dca08c0711c32 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 3 Nov 2021 19:21:54 -0700 Subject: IA (wayback): actually use an HTTP session for replay fetches I am embarassed this wasn't actually the case already! Looks like I had even instantiated a session but wasn't using it. Hopefully this change, which adds extra retries and better backoff behavior, will improve sandcrawler ingest throughput. --- python/sandcrawler/ia.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'python') diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index 99a7f36..d334d24 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -395,6 +395,7 @@ class WaybackClient: self.replay_headers = { "User-Agent": "Mozilla/5.0 sandcrawler.WaybackClient", } + self.http_session = requests_retry_session() def fetch_petabox( self, csize: int, offset: int, warc_path: str, resolve_revisit: bool = True @@ -603,7 +604,7 @@ class WaybackClient: assert datetime.isdigit() try: - resp = requests.get( + resp = self.http_session.get( self.wayback_endpoint + datetime + "id_/" + url, allow_redirects=False, headers=self.replay_headers, @@ -670,7 +671,7 @@ class WaybackClient: assert datetime.isdigit() try: - resp = requests.get( + resp = self.http_session.get( self.wayback_endpoint + datetime + "id_/" + url, allow_redirects=False, headers=self.replay_headers, -- cgit v1.2.3