diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/sandcrawler/html_ingest.py | 193 |
1 files changed, 193 insertions, 0 deletions
diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py new file mode 100644 index 0000000..0e1439c --- /dev/null +++ b/python/sandcrawler/html_ingest.py @@ -0,0 +1,193 @@ + +import sys +import json +import datetime +import argparse +from typing import List, Optional, Any + +import trafilatura +import pydantic +from selectolax.parser import HTMLParser + +from sandcrawler.ia import WaybackClient, CdxApiClient, ResourceResult +from sandcrawler.misc import gen_file_metadata +from sandcrawler.html_metadata import BiblioMetadata, html_extract_resources, html_extract_biblio, load_adblock_rules + + +def html_extract_fulltext_teixml(doc: bytes) -> dict: + tei_xml = trafilatura.extract(doc, + tei_output=True, + include_comments=False, + include_formatting=True, + ) + if tei_xml: + return dict(status="success", tei_xml=tei_xml) + else: + return dict(status="empty-xml") + +class WebResource(pydantic.BaseModel): + surt: str + timestamp: datetime.datetime + url: str + sha1hex: str + mimetype: str + status_code: int + size: Optional[int] + sha256hex: Optional[str] + resource_type: Optional[str] + + +def quick_fetch_html_resources(resources: List[dict], cdx_client: CdxApiClient, when: Optional[datetime.datetime]) -> List[WebResource]: + """ + This is the lazy version that just does a CDX lookup for each resource. + + Takes a list instead of single record because we may want to circuit break + on failure, and may introduce concurrency internal to this function. + """ + + full = [] + for resource in resources: + cdx_row = cdx_client.lookup_best(resource['url']) + if not cdx_row: + raise Exception("CDX lookup failed") + if cdx_row.url != resource['url']: + pass + #raise Exception( + # f"CDX lookup URL mismatch: {cdx_row.url} != {resource['url']}") + full.append(WebResource( + surt=cdx_row.surt, + timestamp=cdx_row.datetime, + url=cdx_row.url, + sha1hex=cdx_row.sha1hex, + mimetype=cdx_row.mimetype, + status_code=cdx_row.status_code, + size=None, + sha256hex=None, + resource_type=resource['type'], + )) + + return full + + +def fetch_html_resources(resources: List[dict], wayback_client: WaybackClient, when: Optional[datetime.datetime]) -> List[WebResource]: + """ + This is the full version which fetches each resource from wayback/petabox + and calculates additional hashes. + + Could make this concurrent in the future, eg: https://realpython.com/python-concurrency/#threading-version + """ + + full = [] + for resource in resources: + wayback_resp = wayback_client.lookup_resource(resource['url']) + if not wayback_resp: + raise Exception("wayback lookup failed") + assert wayback_resp.status == 'success' + if wayback_resp.cdx.url != resource['url']: + pass + #raise Exception( + # f"CDX lookup URL mismatch: {cdx_row.url} != {resource['url']}") + file_meta = gen_file_metadata(wayback_resp.body) + assert file_meta['sha1hex'] == wayback_resp.cdx.sha1hex + full.append(WebResource( + surt=wayback_resp.cdx.surt, + timestamp=wayback_resp.cdx.datetime, + url=wayback_resp.cdx.url, + sha1hex=file_meta['sha1hex'], + mimetype=file_meta['mimetype'], + status_code=wayback_resp.cdx.status_code, + size=file_meta['size_bytes'], + sha256hex=file_meta['sha256hex'], + resource_type=resource['type'], + )) + + return full + + +def run_single(url: str, timestamp: Optional[str] = None, quick_mode: bool = False) -> None: + + adblock = load_adblock_rules() + wayback_client = WaybackClient() + + html_resource = wayback_client.lookup_resource(url, "text/html") + if html_resource.status != "success": + print(json.dumps(html_resource, indent=2)) + return + + file_meta = gen_file_metadata(html_resource.body) + # XXX: + assert file_meta['mimetype'] == "text/html" + + html_doc = HTMLParser(html_resource.body) + html_meta = html_extract_biblio(html_doc) + html_fulltext = html_extract_fulltext_teixml(html_resource.body) + raw_resources = html_extract_resources(html_resource.terminal_url, html_doc, adblock) + + # XXX: + when = None + + full_resources: List[WebResource] = [] + if quick_mode: + full_resources = quick_fetch_html_resources(raw_resources, wayback_client.cdx_client, when) + else: + full_resources = fetch_html_resources(raw_resources, wayback_client, when) + + output = dict( + status="success", + #html_resource=html_resource, + file_meta=file_meta, + html_fulltext=html_fulltext, + # XXX: + html_meta=html_meta and html_meta.dict(exclude_none=True, exclude={'release_date'}), + resources=[r.dict(exclude_none=True, exclude={'timestamp'}) for r in full_resources], + ) + + print(json.dumps(output, indent=2)) + + +def main() -> None: + """ + Run this command like: + + python -m sandcrawler.html_ingest + """ + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + subparsers = parser.add_subparsers() + + sub = subparsers.add_parser( + "single", help="tries to ingest a single URL, dumps result to stdout" + ) + sub.set_defaults(func="run_single") + sub.add_argument( + "url", + help="URL to fetch", + type=str, + ) + sub.add_argument( + "--timestamp", + help="timestamp for which to fetch document from wayback", + type=str, + ) + sub.add_argument( + "--quick-mode", + help="don't fetch resources, only do CDX lookup", + action="store_true", + ) + + args = parser.parse_args() + if not args.__dict__.get("func"): + print("tell me what to do! (try --help)") + sys.exit(-1) + + if args.func == "run_single": + run_single(args.url, args.timestamp, args.quick_mode) + else: + #func = getattr(wp, args.func) + #func() + raise NotImplementedError() + +if __name__ == "__main__": + main() |