From 31d1a6a713d177990609767d508209ced19ca396 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 2 Nov 2021 18:14:59 -0700 Subject: fmt (black): fatcat_tools/ --- python/fatcat_tools/importers/wayback_static.py | 166 ++++++++++++++---------- 1 file changed, 94 insertions(+), 72 deletions(-) (limited to 'python/fatcat_tools/importers/wayback_static.py') diff --git a/python/fatcat_tools/importers/wayback_static.py b/python/fatcat_tools/importers/wayback_static.py index 196f86ff..22fefad3 100755 --- a/python/fatcat_tools/importers/wayback_static.py +++ b/python/fatcat_tools/importers/wayback_static.py @@ -33,22 +33,23 @@ REQ_SESSION = requests.Session() def parse_wbm_url(url): """Takes a wayback machine URL, and returns a tuple: - (timestamp, datetime, original_url) + (timestamp, datetime, original_url) """ - chunks = url.split('/') + chunks = url.split("/") assert len(chunks) >= 6 - assert chunks[2] == 'web.archive.org' - assert chunks[3] == 'web' - return (chunks[4], - parse_wbm_timestamp(chunks[4]), - '/'.join(chunks[5:])) + assert chunks[2] == "web.archive.org" + assert chunks[3] == "web" + return (chunks[4], parse_wbm_timestamp(chunks[4]), "/".join(chunks[5:])) + def test_parse_wbm_url(): u = "http://web.archive.org/web/20010712114837/http://www.dlib.org/dlib/june01/reich/06reich.html" assert parse_wbm_url(u) == ( "20010712114837", datetime.datetime(2001, 7, 12, 11, 48, 37), - "http://www.dlib.org/dlib/june01/reich/06reich.html") + "http://www.dlib.org/dlib/june01/reich/06reich.html", + ) + def parse_wbm_timestamp(timestamp): """ @@ -56,7 +57,7 @@ def parse_wbm_timestamp(timestamp): python datetime object (UTC) """ # strip any "im_" or "id_" suffix - if timestamp.endswith('_'): + if timestamp.endswith("_"): timestamp = timestamp[:-3] # inflexible; require the full second-precision timestamp assert len(timestamp) == 14 @@ -66,11 +67,13 @@ def parse_wbm_timestamp(timestamp): day=int(timestamp[6:8]), hour=int(timestamp[8:10]), minute=int(timestamp[10:12]), - second=int(timestamp[12:14])) + second=int(timestamp[12:14]), + ) + def test_parse_wbm_timestamp(): - assert parse_wbm_timestamp("20010712114837") == \ - datetime.datetime(2001, 7, 12, 11, 48, 37) + assert parse_wbm_timestamp("20010712114837") == datetime.datetime(2001, 7, 12, 11, 48, 37) + def fetch_wbm(url): resp = REQ_SESSION.get(url) @@ -78,31 +81,35 @@ def fetch_wbm(url): assert resp.content return resp.content + def lookup_cdx(embed_url, verify_hashes=True, cdx_output=None): sys.stderr.write(embed_url + "\n") - assert embed_url.startswith('/web/') - embed_url = embed_url.split('/') + assert embed_url.startswith("/web/") + embed_url = embed_url.split("/") timestamp = embed_url[2] - if timestamp.endswith('_'): + if timestamp.endswith("_"): timestamp = timestamp[:-3] - url = '/'.join(embed_url[3:]) - #print((timestamp, url)) - resp = REQ_SESSION.get(CDX_API_BASE, params=dict( - url=url, - closest=timestamp, - sort="closest", - resolveRevisits="true", - matchType="exact", - limit=1, - )) + url = "/".join(embed_url[3:]) + # print((timestamp, url)) + resp = REQ_SESSION.get( + CDX_API_BASE, + params=dict( + url=url, + closest=timestamp, + sort="closest", + resolveRevisits="true", + matchType="exact", + limit=1, + ), + ) resp.raise_for_status() - #print(resp.url) + # print(resp.url) if resp.content: - hit = resp.content.decode('utf-8').split('\n')[0] + hit = resp.content.decode("utf-8").split("\n")[0] if cdx_output: cdx_output.write(hit + "\n") - cdx = hit.split(' ') - cdx = [x if (x and x != '-') else None for x in cdx] + cdx = hit.split(" ") + cdx = [x if (x and x != "-") else None for x in cdx] webcapture_cdx = WebcaptureCdxLine( surt=cdx[0], timestamp=parse_wbm_timestamp(cdx[1]).isoformat() + "Z", @@ -113,9 +120,9 @@ def lookup_cdx(embed_url, verify_hashes=True, cdx_output=None): sha256=None, ) if verify_hashes: - resp = REQ_SESSION.get(GWB_URL_BASE + "/{}id_/{}".format( - cdx[1], # raw timestamp - webcapture_cdx.url)) + resp = REQ_SESSION.get( + GWB_URL_BASE + "/{}id_/{}".format(cdx[1], webcapture_cdx.url) # raw timestamp + ) resp.raise_for_status() assert webcapture_cdx.sha1 == hashlib.sha1(resp.content).digest().hex() webcapture_cdx.sha256 = hashlib.sha256(resp.content).digest().hex() @@ -124,47 +131,50 @@ def lookup_cdx(embed_url, verify_hashes=True, cdx_output=None): else: return None + def wayback_url_to_relative(url): """ Wayback URLs can be relative or absolute in rewritten documents. This function converts any form of rewritten URL to a relative (to web.archive.org) one, or returns None if it isn't a rewritten URL at all. """ - if url.startswith('https://web.archive.org/'): + if url.startswith("https://web.archive.org/"): url = url[23:] - elif url.startswith('http://web.archive.org/'): + elif url.startswith("http://web.archive.org/"): url = url[22:] - if url.startswith('/web/'): + if url.startswith("/web/"): return url else: return None + def extract_embeds(soup): embeds = set() # - for tag in soup.find_all('link', href=True): - if tag['rel'] not in ('stylesheet',): + for tag in soup.find_all("link", href=True): + if tag["rel"] not in ("stylesheet",): continue - url = wayback_url_to_relative(tag['href']) + url = wayback_url_to_relative(tag["href"]) if url: embeds.add(url) # - for tag in soup.find_all('img', src=True): - url = wayback_url_to_relative(tag['src']) + for tag in soup.find_all("img", src=True): + url = wayback_url_to_relative(tag["src"]) if url: embeds.add(url) #