From 6a701f966b8bc760bf904c0569562b0159e13559 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Sun, 8 Nov 2020 21:52:47 -0800
Subject: basic DOAJ ingest request conversion script

---
 python/scripts/doaj2ingestrequest.py | 139 +++++++++++++++++++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100755 python/scripts/doaj2ingestrequest.py

(limited to 'python/scripts')

diff --git a/python/scripts/doaj2ingestrequest.py b/python/scripts/doaj2ingestrequest.py
new file mode 100755
index 0000000..f1bae8c
--- /dev/null
+++ b/python/scripts/doaj2ingestrequest.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+"""
+Transform an DOAJ article dump (JSON) into ingest requests.
+
+TODO: should we also attempt PDF ingest for HTML links? They seem to often be
+landing pages. Or could have some pipeline that notices, eg, `citation_pdf_url`
+in the HTML headers and adds an ingest request on that basis. Or even just run
+the re-ingest in-process and publish a second result.
+"""
+
+import sys
+import json
+import argparse
+import urlcanon
+from typing import Optional, List
+
+DOMAIN_BLOCKLIST = [
+    # large OA publishers (we get via DOI)
+
+    # large repos and aggregators (we crawl directly)
+    "://arxiv.org/",
+    "://europepmc.org/",
+    "ncbi.nlm.nih.gov/",
+    #"semanticscholar.org/",
+    "://doi.org/",
+    "zenodo.org/",
+    "figshare.com/",
+    "://archive.org/",
+    ".archive.org/",
+
+    # large publishers/platforms; may remove in the future
+    #"://link.springer.com/",
+    #"://dergipark.gov.tr/",
+    #"frontiersin.org/",
+    #"scielo",
+]
+
+# these default to PDF; note that we also do pdf ingests for HTML pages
+CONTENT_TYPE_MAP = {
+    "abstract": [],
+    "doc": [],
+    "": ["pdf"],
+
+    "doi": ["pdf"],
+    "url": ["pdf"],
+    "fulltext": ["pdf"],
+    "anySimpleType": ["pdf"],
+
+    "application/pdf": ["pdf"],
+    "html": ["html", "pdf"],
+    "text/html": ["html", "pdf"],
+    "xml": ["xml"],
+}
+
+def canon(s: str) -> str:
+    parsed = urlcanon.parse_url(s)
+    return str(urlcanon.whatwg(parsed))
+
+def transform(obj: dict) -> List[dict]:
+    """
+    Transforms from a single DOAJ object to zero or more ingest requests.
+    Returns a list of dicts.
+    """
+
+    doaj_id = obj['id'].lower()
+    assert doaj_id
+
+    bibjson = obj['bibjson']
+    if not bibjson['link']:
+        return []
+
+    requests = []
+
+    doi: Optional[str] = None
+    for ident in (bibjson['identifier'] or []):
+        if ident['type'].lower() == "doi" and ident['id'].startswith('10.'):
+            doi = ident['id'].lower()
+
+    for link in (bibjson['link'] or []):
+        if link.get('type') != "fulltext" or not link.get('url'):
+            continue
+        ingest_types = CONTENT_TYPE_MAP.get((link.get('content_type') or '').lower())
+        if not ingest_types:
+            continue
+        skip = False
+        for domain in DOMAIN_BLOCKLIST:
+            if domain in link['url'].lower():
+                skip = True
+        if skip:
+            continue
+        try:
+            base_url = canon(link['url'])
+        except UnicodeEncodeError:
+            continue
+
+        for ingest_type in ingest_types:
+            request = {
+                'base_url': base_url,
+                'ingest_type': ingest_type,
+                'link_source': 'doaj',
+                'link_source_id': doaj_id,
+                'ingest_request_source': 'doaj',
+                'release_stage': 'published',
+                'rel': 'publisher',
+                'ext_ids': {
+                    'doi': doi,
+                    'doaj': doaj_id,
+                },
+                'edit_extra': {},
+            }
+            requests.append(request)
+
+    return requests
+
+def run(args) -> None:
+    for l in args.json_file:
+        if not l.strip():
+            continue
+        row = json.loads(l)
+
+        requests = transform(row) or []
+        for r in requests:
+            print("{}".format(json.dumps(r, sort_keys=True)))
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('json_file',
+        help="DOAJ article dump file to use",
+        type=argparse.FileType('r'))
+    subparsers = parser.add_subparsers()
+
+    args = parser.parse_args()
+
+    run(args)
+
+if __name__ == '__main__':
+    main()
-- 
cgit v1.2.3