issn => journal_metadata in several places

1 files changed, 89 insertions, 0 deletions

diff --git a/python/fatcat_tools/importers/journal_metadata.py b/python/fatcat_tools/importers/journal_metadata.py
new file mode 100644
index 00000000..859662ae
--- /dev/null
+++ b/python/fatcat_tools/importers/journal_metadata.py
@@ -0,0 +1,89 @@
+
+import sys
+import json
+import itertools
+import fatcat_client
+from .common import FatcatImporter
+
+
+def or_none(s):
+    if s is None:
+        return None
+    if len(s) == 0:
+        return None
+    return s
+
+def truthy(s):
+    if s is None:
+        return None
+    s = s.lower()
+
+    if s in ('true', 't', 'yes', 'y', '1'):
+        return True
+    elif s in ('false', 'f', 'no', 'n', '0'):
+        return False
+    else:
+        return None
+
+class JournalMetadataImporter(FatcatImporter):
+    """
+    Imports journal metadata ("containers") by ISSN, currently from a custom
+    (data munged) .csv file format
+
+    CSV format (generated from git.archive.org/webgroup/oa-journal-analysis):
+
+        ISSN-L,in_doaj,in_road,in_norwegian,in_crossref,title,publisher,url,lang,ISSN-print,ISSN-electronic,doi_count,has_doi,is_oa,is_kept,publisher_size,url_live,url_live_status,url_live_final_status,url_live_final_url,url_live_status_simple,url_live_final_status_simple,url_domain,gwb_pdf_count
+    """
+
+    def __init__(self, api, **kwargs):
+
+        eg_desc = kwargs.get('editgroup_description',
+            "Automated import of container-level metadata, by ISSN. Metadata from Internet Archive munging.")
+        eg_extra = kwargs.get('editgroup_extra', dict())
+        eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.JournalMetadataImporter')
+        super().__init__(api,
+            editgroup_description=eg_desc,
+            editgroup_extra=eg_extra)
+
+    def parse_journal_metadata_row(self, row):
+        """
+        row is a python dict (parsed from CSV).
+        returns a ContainerEntity (or None if invalid or couldn't parse)
+        """
+        title = or_none(row['title'])
+        issnl = or_none(row['ISSN-L'])
+        if title is None or issnl is None:
+            return None
+        extra = dict(
+            in_doaj=truthy(row['in_doaj']),
+            in_road=truthy(row['in_road']),
+            in_norwegian=truthy(row['in_norwegian']),
+            language=or_none(row['lang']),
+            url=or_none(row['url']),
+            ISSNp=or_none(row['ISSN-print']),
+            ISSNe=or_none(row['ISSN-electronic']),
+            is_oa=truthy(row['is_oa']),
+            is_kept=truthy(row['is_kept']),
+        )
+        ce = fatcat_client.ContainerEntity(
+            issnl=issnl,
+            name=title,
+            publisher=or_none(row['publisher']),
+            abbrev=None,
+            coden=None,
+            extra=extra)
+        return ce
+
+    def create_row(self, row, editgroup_id=None):
+        ce = self.parse_journal_metadata_row(row)
+        if ce is not None:
+            self.api.create_container(ce, editgroup_id=editgroup_id)
+            self.counts['insert'] += 1
+
+    def create_batch(self, batch):
+        """Reads and processes in batches (not API-call-per-line)"""
+        objects = [self.parse_journal_metadata_row(l)
+                   for l in batch if (l is not None)]
+        objects = [o for o in objects if (o is not None)]
+        self.api.create_container_batch(objects, autoaccept="true")
+        self.counts['insert'] += len(objects)