JSTOR importer polish

1 files changed, 51 insertions, 18 deletions

diff --git a/python/fatcat_tools/importers/jstor.py b/python/fatcat_tools/importers/jstor.py
index c8a7b20a..c846cbde 100644
--- a/python/fatcat_tools/importers/jstor.py
+++ b/python/fatcat_tools/importers/jstor.py
@@ -7,6 +7,7 @@ from bs4 import BeautifulSoup
 
 import fatcat_client
 from .common import EntityImporter, clean, LANG_MAP_MARC
+from .crossref import CONTAINER_TYPE_MAP
 
 # TODO: more entries?
 JSTOR_CONTRIB_MAP = {
@@ -16,6 +17,14 @@ JSTOR_CONTRIB_MAP = {
     'illustrator': 'illustrator',
 }
 
+JSTOR_TYPE_MAP = {
+    "book-review": "review-book",
+    "editorial": "editorial",
+    "misc": "stub",
+    "news": "article",
+    "research-article": "article-journal",
+}
+
 class JstorImporter(EntityImporter):
     """
     Importer for JSTOR bulk XML metadata (eg, from their Early Journals
@@ -38,6 +47,9 @@ class JstorImporter(EntityImporter):
 
         self.read_issn_map_file(issn_map_file)
 
+    def map_container_type(self, crossref_type):
+        return CONTAINER_TYPE_MAP.get(crossref_type)
+
     def want(self, obj):
         return True
 
@@ -49,13 +61,22 @@ class JstorImporter(EntityImporter):
         extra = dict()
         extra_jstor = dict()
 
+        release_type = JSTOR_TYPE_MAP.get(article['article-type'])
         title = article_meta.find("article-title")
-        if title:
+        if title and title.string:
             title = title.string.strip()
-            if title.endswith('.'):
-                title = title[:-1]
+        elif title and not title.string:
+            title = None
+
+        if not title and release_type.startswith('review') and article_meta.product.source:
+            title = "Review: {}".format(article_meta.product.source.string)
+
+        if not title:
+            return None
+
+        if title.endswith('.'):
+            title = title[:-1]
 
-        release_type = "article-journal"
         if "[Abstract]" in title:
             # TODO: strip the "[Abstract]" bit?
             release_type = "abstract"
@@ -108,28 +129,40 @@ class JstorImporter(EntityImporter):
 
         jstor_id = article_meta.find("article-id", {"pub-id-type": "jstor"})
         if jstor_id:
-            jstor_id = jstor_id.string
+            jstor_id = jstor_id.string.strip()
+        if not jstor_id and doi:
+            assert doi.startswith('10.2307/')
+            jstor_id = doi.replace('10.2307/', '')
+        assert jstor_id and int(jstor_id)
 
         contribs = []
         cgroup = article_meta.find("contrib-group")
         if cgroup:
             for c in cgroup.find_all("contrib"):
                 given = c.find("given-names")
+                if given:
+                    given = clean(given.string)
                 surname = c.find("surname")
-                if given and surname:
-                    name = "{} {}".format(given.string, surname.string)
-                elif surname:
-                    name = surname.string
-                else:
-                    name = None
-                role = JSTOR_CONTRIB_MAP.get(c['contrib-type'])
-                if not role and c['contrib-type']:
+                if surname:
+                    surname = clean(surname.string)
+                raw_name = c.find("string-name")
+                if raw_name:
+                    raw_name = clean(raw_name.string)
+
+                if not raw_name:
+                    if given and surname:
+                        raw_name = "{} {}".format(given, surname)
+                    elif surname:
+                        raw_name = surname
+
+                role = JSTOR_CONTRIB_MAP.get(c.get('contrib-type', 'author'))
+                if not role and c.get('contrib-type'):
                     sys.stderr.write("NOT IN JSTOR_CONTRIB_MAP: {}\n".format(c['contrib-type']))
                 contribs.append(fatcat_client.ReleaseContrib(
-                    role=JSTOR_CONTRIB_MAP.get(c['contrib-type']),
-                    raw_name=clean(name),
-                    given_name=clean(given.string),
-                    surname=clean(surname.string),
+                    role=role,
+                    raw_name=raw_name,
+                    given_name=given,
+                    surname=surname,
                 ))
 
         release_year = None
@@ -164,7 +197,7 @@ class JstorImporter(EntityImporter):
         language = None
         cm = article_meta.find("custom-meta")
         if cm.find("meta-name").string == "lang":
-            language = cm.find("meta-value").string
+            language = cm.find("meta-value").string.split()[0]
             language = LANG_MAP_MARC.get(language)
             if not language:
                 warnings.warn("MISSING MARC LANG: {}".format(cm.find("meta-value").string))