diff options
| -rw-r--r-- | python/sandcrawler/ia.py | 45 |
1 files changed, 40 insertions, 5 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index ac0fef8..74cd978 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -212,6 +212,7 @@ class WaybackClient: self.warc_uri_prefix = kwargs.get('warc_uri_prefix', 'https://archive.org/serve/') self.rstore = None self.max_redirects = 25 + self.wayback_endpoint = "https://web.archive.org/web/" def fetch_petabox(self, c_size, offset, warc_path): """ @@ -292,9 +293,20 @@ class WaybackClient: Intended for use with SPN2 requests, where request body has not ended up in petabox yet. - TODO: is this actually necessary? + TODO: is this really necessary? """ - raise NotImplementedError + try: + resp = requests.get(self.wayback_endpoint + datetime + "id_/" + url) + except requests.exceptions.TooManyRedirects: + raise WaybackError("redirect loop (wayback replay fetch)") + try: + resp.raise_for_status() + except Exception as e: + raise WaybackError(str(e)) + # TODO: some verification here? + #print(resp.url, file=sys.stderr) + #print(resp.content) + return resp.content def lookup_resource(self, start_url, best_mimetype=None): """ @@ -473,6 +485,16 @@ class SavePageNowClient: if not final_json: raise SavePageNowError("SPN2 timed out (polling count exceeded)") + if final_json.get('original_job_id'): + resp = self.v2_session.get("{}/status/{}".format(self.v2endpoint, final_json['original_job_id'])) + try: + resp.raise_for_status() + except: + raise SavePageNowError(resp.content) + final_json = resp.json() + + #print(final_json, file=sys.stderr) + if final_json['status'] == "success": return SavePageNowResult( True, @@ -515,10 +537,23 @@ class SavePageNowClient: cdx=None, ) - # fetch CDX and body - cdx_row = wayback_client.cdx_client.fetch(spn_result.terminal_url, spn_result.terminal_dt) + # sleep one second to let CDX API update (XXX) + time.sleep(1.0) + + # fetch CDX, then body + cdx_row = wayback_client.cdx_client.fetch( + spn_result.terminal_url, + spn_result.terminal_dt, + filter_status_code=200, + ) + print(spn_result, file=sys.stderr) + print(cdx_row, file=sys.stderr) assert cdx_row.status_code == 200 - body = wayback_client.fetch_petabox_body(cdx_row.warc_csize, cdx_row.warc_offset, cdx_row.warc_path) + body = wayback_client.fetch_replay_body(cdx_row.url, cdx_row.datetime) + file_meta = gen_file_metadata(body) + if cdx_row.sha1hex != file_meta['sha1hex']: + print(file_meta, file=sys.stderr) + raise WaybackError("replay fetch resulted in non-matching SHA1 (vs. CDX API)") # not a full CDX yet cdx_partial = cdx_partial_from_row(cdx_row) |