reformat python code with black

1 files changed, 158 insertions, 109 deletions

diff --git a/fatcat_scholar/transform.py b/fatcat_scholar/transform.py
index 953ebff..b5a0223 100644
--- a/fatcat_scholar/transform.py
+++ b/fatcat_scholar/transform.py
@@ -1,4 +1,3 @@
-
 import os
 import io
 import sys
@@ -10,83 +9,89 @@ import internetarchive
 
 from fatcat_scholar.api_entities import *
 from fatcat_scholar.djvu import djvu_extract_leaf_texts
-from fatcat_scholar.sandcrawler import SandcrawlerPostgrestClient, SandcrawlerMinioClient
+from fatcat_scholar.sandcrawler import (
+    SandcrawlerPostgrestClient,
+    SandcrawlerMinioClient,
+)
 from fatcat_scholar.issue_db import IssueDB, SimIssueRow
 from fatcat_scholar.schema import *
 from fatcat_scholar.grobid2json import teixml2json
 
 
 def es_fulltext_from_sim(sim: Dict[str, Any]) -> Optional[ScholarFulltext]:
-    if not sim['page_texts']:
+    if not sim["page_texts"]:
         return None
-    first_page = sim['page_texts'][0]['page_num']
-    issue_item = sim['issue_item']
+    first_page = sim["page_texts"][0]["page_num"]
+    issue_item = sim["issue_item"]
     return ScholarFulltext(
-        lang_code=None, # TODO: pub/issue metadata? or langdetect?
-        body="\n".join([p['raw_text'] for p in sim['page_texts']]),
-        #acknowledgement=None,
-        #annex=None,
-        release_ident=sim.get('release_ident'),
-        #file_ident=None,
-        #file_sha1=None,
-        #file_mimetype=None,
+        lang_code=None,  # TODO: pub/issue metadata? or langdetect?
+        body="\n".join([p["raw_text"] for p in sim["page_texts"]]),
+        # acknowledgement=None,
+        # annex=None,
+        release_ident=sim.get("release_ident"),
+        # file_ident=None,
+        # file_sha1=None,
+        # file_mimetype=None,
         thumbnail_url=f"https://archive.org/serve/{issue_item}/__ia_thumb.jpg",
         access_url=f"https://archive.org/details/{issue_item}/page/{first_page}",
         access_type=AccessType.ia_sim,
     )
 
+
 def es_sim_from_sim(sim: Dict[str, Any]) -> ScholarSim:
     first_page = None
-    if sim['page_texts']:
-        first_page = sim['page_texts'][0]['page_num']
+    if sim["page_texts"]:
+        first_page = sim["page_texts"][0]["page_num"]
     return ScholarSim(
-        issue_item=sim['issue_item'],
-        pub_collection=sim['pub_item_metadata']['metadata']['identifier'],
-        sim_pubid=sim['issue_item_metadata']['metadata']['sim_pubid'],
+        issue_item=sim["issue_item"],
+        pub_collection=sim["pub_item_metadata"]["metadata"]["identifier"],
+        sim_pubid=sim["issue_item_metadata"]["metadata"]["sim_pubid"],
         first_page=first_page,
     )
 
+
 SIM_RELEASE_TYPE_MAP = {
-    'Scholarly Journals': 'article-journal',
+    "Scholarly Journals": "article-journal",
     # TODO:
 }
 SIM_LANG_MAP = {
-    'English': 'en',
+    "English": "en",
     # TODO:
 }
 SIM_COUNTRY_MAP = {
-    'Netherlands': 'nl',
+    "Netherlands": "nl",
     # TODO:
 }
 
+
 def es_biblio_from_sim(sim: Dict[str, Any]) -> ScholarBiblio:
 
-    issue_meta = sim['issue_item_metadata']['metadata']
-    pub_meta = sim['pub_item_metadata']['metadata']
+    issue_meta = sim["issue_item_metadata"]["metadata"]
+    pub_meta = sim["pub_item_metadata"]["metadata"]
 
     first_page = None
-    if sim['page_texts']:
-        first_page = sim['page_texts'][0]['page_num']
-    container_name = sim['pub_item_metadata']['metadata']['title']
+    if sim["page_texts"]:
+        first_page = sim["page_texts"][0]["page_num"]
+    container_name = sim["pub_item_metadata"]["metadata"]["title"]
     last_word = container_name.split()[-1]
-    if len(last_word) == 9 and last_word[4] == '-':
+    if len(last_word) == 9 and last_word[4] == "-":
         container_name = container_name[:-10]
 
     issns = []
-    raw_issn = issue_meta.get('issn')
+    raw_issn = issue_meta.get("issn")
     if raw_issn and len(raw_issn) == 9:
         issns.append(raw_issn)
 
-    volume = issue_meta.get('volume')
+    volume = issue_meta.get("volume")
     volume_int = None
     if volume and volume.isdigit():
         volume_int = int(volume)
-    issue = issue_meta.get('issue')
+    issue = issue_meta.get("issue")
     issue_int = None
     if issue and issue.isdigit():
         issue_int = int(issue)
 
-    date = issue_meta.get('date')
+    date = issue_meta.get("date")
     release_year = None
     if date and len(date) > 4 and date[:4].isdigit():
         release_year = int(date[:4])
@@ -96,52 +101,52 @@ def es_biblio_from_sim(sim: Dict[str, Any]) -> ScholarBiblio:
         release_date = date
 
     return ScholarBiblio(
-        #release_ident=release.ident,
+        # release_ident=release.ident,
         title=None,
-        #subtitle=None,
-        #original_title=release.original_title,
+        # subtitle=None,
+        # original_title=release.original_title,
         release_date=release_date,
         release_year=release_year,
-        release_type=SIM_RELEASE_TYPE_MAP.get(pub_meta.get('pub_type')),
-        release_stage="published", # as a default
-        #withdrawn_status=release.withdrawn_status,
-        lang_code=SIM_LANG_MAP.get(pub_meta.get('language')),
-        country_code=SIM_COUNTRY_MAP.get(pub_meta.get('country')),
+        release_type=SIM_RELEASE_TYPE_MAP.get(pub_meta.get("pub_type")),
+        release_stage="published",  # as a default
+        # withdrawn_status=release.withdrawn_status,
+        lang_code=SIM_LANG_MAP.get(pub_meta.get("language")),
+        country_code=SIM_COUNTRY_MAP.get(pub_meta.get("country")),
         volume=volume,
         volume_int=volume_int,
         issue=issue,
         issue_int=issue_int,
-        pages=sim.get('pages'),
+        pages=sim.get("pages"),
         first_page=first_page,
         first_page_int=None,
-        #number=None,
-
+        # number=None,
         # no external identifiers
-
-        #license_slug=release.license_slug,
-        publisher=issue_meta.get('publisher'),
+        # license_slug=release.license_slug,
+        publisher=issue_meta.get("publisher"),
         container_name=container_name,
-        container_original_name=None, # TODO pass-through
-        container_ident=None, # TODO: pass-through
-        container_type=None, # TODO
-        container_issnl=None, # TODO: pass-through
+        container_original_name=None,  # TODO pass-through
+        container_ident=None,  # TODO: pass-through
+        container_type=None,  # TODO
+        container_issnl=None,  # TODO: pass-through
         issns=issns,
-
         # no contrib/affiliation info
         contrib_names=[],
         affiliations=[],
     )
 
-def _add_file_release_meta(fulltext: ScholarFulltext, re: ReleaseEntity, fe: FileEntity) -> ScholarFulltext:
+
+def _add_file_release_meta(
+    fulltext: ScholarFulltext, re: ReleaseEntity, fe: FileEntity
+) -> ScholarFulltext:
     best_url = None
     best_url_type = None
     for url in fe.urls:
         best_url = url.url
         best_url_type = AccessType.web
-        if '//archive.org/' in url.url:
+        if "//archive.org/" in url.url:
             best_url_type = AccessType.ia_file
             break
-        elif '//web.archive.org/' in url.url:
+        elif "//web.archive.org/" in url.url:
             best_url_type = AccessType.wayback
             break
         if url.rel == "repository":
@@ -157,30 +162,36 @@ def _add_file_release_meta(fulltext: ScholarFulltext, re: ReleaseEntity, fe: Fil
     return fulltext
 
 
-def es_fulltext_from_grobid(tei_xml: str, re: ReleaseEntity, fe: FileEntity) -> Optional[ScholarFulltext]:
+def es_fulltext_from_grobid(
+    tei_xml: str, re: ReleaseEntity, fe: FileEntity
+) -> Optional[ScholarFulltext]:
     obj = teixml2json(tei_xml)
-    if not obj.get('body'):
+    if not obj.get("body"):
         return None
     ret = ScholarFulltext(
-        lang_code=obj.get('lang'),
-        body=obj.get('body'),
-        acknowledgement=obj.get('acknowledgement'),
-        annex=obj.get('annex'),
-        thumbnail_url=None, # TODO: sandcrawler thumbnails
+        lang_code=obj.get("lang"),
+        body=obj.get("body"),
+        acknowledgement=obj.get("acknowledgement"),
+        annex=obj.get("annex"),
+        thumbnail_url=None,  # TODO: sandcrawler thumbnails
     )
     return _add_file_release_meta(ret, re, fe)
 
-def es_fulltext_from_pdftotext(pdftotext: Any, re: ReleaseEntity, fe: FileEntity) -> Optional[ScholarFulltext]:
+
+def es_fulltext_from_pdftotext(
+    pdftotext: Any, re: ReleaseEntity, fe: FileEntity
+) -> Optional[ScholarFulltext]:
 
     ret = ScholarFulltext(
         lang_code=re.language,
-        body=pdftotext['raw_text'],
+        body=pdftotext["raw_text"],
         acknowledgement=None,
         annex=None,
-        thumbnail_url=None, # TODO: sandcrawler thumbnails
+        thumbnail_url=None,  # TODO: sandcrawler thumbnails
     )
     return _add_file_release_meta(ret, re, fe)
 
+
 def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
 
     tags: List[str] = []
@@ -203,7 +214,9 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
         work_ident = heavy.releases[0].work_id
         key = f"work_{work_ident}"
         assert heavy.biblio_release_ident
-        primary_release = [r for r in heavy.releases if r.ident == heavy.biblio_release_ident][0]
+        primary_release = [
+            r for r in heavy.releases if r.ident == heavy.biblio_release_ident
+        ][0]
         biblio = es_biblio_from_release(primary_release)
 
         # TODO: abstracts from releases also; abstracts_dict; abstracts from GROBID parse
@@ -212,19 +225,44 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
         raise NotImplementedError(f"doc_type: {heavy.doc_type}")
 
     if heavy.grobid_fulltext:
-        fulltext_release = [r for r in heavy.releases if r.ident == heavy.grobid_fulltext['release_ident']][0]
-        fulltext_file = [f for f in fulltext_release.files if f.ident == heavy.grobid_fulltext['file_ident']][0]
-        fulltext = es_fulltext_from_grobid(heavy.grobid_fulltext['tei_xml'], fulltext_release, fulltext_file)
+        fulltext_release = [
+            r
+            for r in heavy.releases
+            if r.ident == heavy.grobid_fulltext["release_ident"]
+        ][0]
+        fulltext_file = [
+            f
+            for f in fulltext_release.files
+            if f.ident == heavy.grobid_fulltext["file_ident"]
+        ][0]
+        fulltext = es_fulltext_from_grobid(
+            heavy.grobid_fulltext["tei_xml"], fulltext_release, fulltext_file
+        )
 
         # hack to pull through thumbnail from local pdftotext
-        if fulltext and fulltext.file_sha1 and not fulltext.thumbnail_url and heavy.pdftotext_fulltext:
+        if (
+            fulltext
+            and fulltext.file_sha1
+            and not fulltext.thumbnail_url
+            and heavy.pdftotext_fulltext
+        ):
             # https://covid19.fatcat.wiki/fulltext_web/thumbnail/c9/c9e87f843b3cf7dc47881fa3d3ccb4693d7d9521.png
             fulltext.thumbnail_url = f"https://covid19.fatcat.wiki/fulltext_web/thumbnail/{fulltext.file_sha1[:2]}/{fulltext.file_sha1}.png"
 
     if not fulltext and heavy.pdftotext_fulltext:
-        fulltext_release = [r for r in heavy.releases if r.ident == heavy.pdftotext_fulltext['release_ident']][0]
-        fulltext_file = [f for f in fulltext_release.files if f.ident == heavy.pdftotext_fulltext['file_ident']][0]
-        fulltext = es_fulltext_from_pdftotext(heavy.pdftotext_fulltext, fulltext_release, fulltext_file)
+        fulltext_release = [
+            r
+            for r in heavy.releases
+            if r.ident == heavy.pdftotext_fulltext["release_ident"]
+        ][0]
+        fulltext_file = [
+            f
+            for f in fulltext_release.files
+            if f.ident == heavy.pdftotext_fulltext["file_ident"]
+        ][0]
+        fulltext = es_fulltext_from_pdftotext(
+            heavy.pdftotext_fulltext, fulltext_release, fulltext_file
+        )
 
     # TODO: additional access list
     access_dict = dict()
@@ -246,41 +284,41 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
 
     # tags
     if biblio.license_slug and biblio.license_slug.lower().startswith("cc-"):
-        tags.append('oa')
+        tags.append("oa")
     if primary_release and primary_release.container:
         container = primary_release.container
         if container.extra:
-            if container.extra.get('doaj'):
-                tags.append('doaj')
-                tags.append('oa')
-            if container.extra.get('road'):
-                tags.append('road')
-                tags.append('oa')
-            if container.extra.get('szczepanski'):
-                tags.append('szczepanski')
-                tags.append('oa')
-            if container.extra.get('ia', {}).get('longtail_oa'):
-                tags.append('longtail')
-                tags.append('oa')
-            if container.extra.get('sherpa_romeo', {}).get('color') == 'white':
-                tags.append('oa')
-            if container.extra.get('default_license', '').lower().startswith('cc-'):
-                tags.append('oa')
-            if container.extra.get('platform'):
+            if container.extra.get("doaj"):
+                tags.append("doaj")
+                tags.append("oa")
+            if container.extra.get("road"):
+                tags.append("road")
+                tags.append("oa")
+            if container.extra.get("szczepanski"):
+                tags.append("szczepanski")
+                tags.append("oa")
+            if container.extra.get("ia", {}).get("longtail_oa"):
+                tags.append("longtail")
+                tags.append("oa")
+            if container.extra.get("sherpa_romeo", {}).get("color") == "white":
+                tags.append("oa")
+            if container.extra.get("default_license", "").lower().startswith("cc-"):
+                tags.append("oa")
+            if container.extra.get("platform"):
                 # scielo, ojs, wordpress, etc
-                tags.append(container.extra['platform'].lower())
-    if biblio.doi_prefix == '10.2307':
-        tags.append('jstor')
+                tags.append(container.extra["platform"].lower())
+    if biblio.doi_prefix == "10.2307":
+        tags.append("jstor")
 
     # biorxiv/medrxiv hacks
     if not biblio.container_name and biblio.release_stage != "published":
         for _, acc in access_dict.items():
             if "://www.medrxiv.org/" in acc.access_url:
-                biblio.container_name = 'medRxiv'
+                biblio.container_name = "medRxiv"
                 if biblio.release_stage == None:
                     biblio.release_stage = "submitted"
             elif "://www.biorxiv.org/" in acc.access_url:
-                biblio.container_name = 'bioRxiv'
+                biblio.container_name = "bioRxiv"
                 if biblio.release_stage == None:
                     biblio.release_stage = "submitted"
     tags = list(set(tags))
@@ -291,7 +329,6 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
         doc_index_ts=datetime.datetime.utcnow(),
         work_ident=work_ident,
         tags=tags,
-
         biblio=biblio,
         fulltext=fulltext,
         ia_sim=ia_sim,
@@ -300,23 +337,28 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
         access=list(access_dict.values()),
     )
 
+
 def run_transform(infile):
     for line in infile:
         obj = json.loads(line)
 
         heavy = IntermediateBundle(
-            doc_type=DocType(obj['doc_type']),
-            releases=[entity_from_json(json.dumps(re), ReleaseEntity) for re in obj['releases']],
-            biblio_release_ident=obj.get('biblio_release_ident'),
-            grobid_fulltext=obj.get('grobid_fulltext'),
-            pdftotext_fulltext=obj.get('pdftotext_fulltext'),
-            sim_fulltext=obj.get('sim_fulltext'),
+            doc_type=DocType(obj["doc_type"]),
+            releases=[
+                entity_from_json(json.dumps(re), ReleaseEntity)
+                for re in obj["releases"]
+            ],
+            biblio_release_ident=obj.get("biblio_release_ident"),
+            grobid_fulltext=obj.get("grobid_fulltext"),
+            pdftotext_fulltext=obj.get("pdftotext_fulltext"),
+            sim_fulltext=obj.get("sim_fulltext"),
         )
         es_doc = transform_heavy(heavy)
         if not es_doc:
             continue
         print(es_doc.json())
 
+
 def main():
     """
     Run this command like:
@@ -325,25 +367,32 @@ def main():
     """
 
     parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
     subparsers = parser.add_subparsers()
 
-    sub = subparsers.add_parser('run_transform',
-        help="iterates through 'heavy' intermediate")
-    sub.set_defaults(func='run_transform')
-    sub.add_argument("json_file",
+    sub = subparsers.add_parser(
+        "run_transform", help="iterates through 'heavy' intermediate"
+    )
+    sub.set_defaults(func="run_transform")
+    sub.add_argument(
+        "json_file",
         help="intermediate globs as JSON-lines",
-        nargs='?', default=sys.stdin, type=argparse.FileType('r'))
+        nargs="?",
+        default=sys.stdin,
+        type=argparse.FileType("r"),
+    )
 
     args = parser.parse_args()
     if not args.__dict__.get("func"):
         print("tell me what to do! (try --help)")
         sys.exit(-1)
 
-    if args.func == 'run_transform':
+    if args.func == "run_transform":
         run_transform(infile=args.json_file)
     else:
         raise NotImplementedError(args.func)
 
-if __name__=="__main__":
+
+if __name__ == "__main__":
     main()