Merge branch 'bnewbold-ingest-improvements' into 'master'

ingest behavior changes; some datacite metadata tweaks

See merge request webgroup/fatcat!78

8 files changed, 120 insertions, 38 deletions

diff --git a/python/fatcat_tools/importers/datacite.py b/python/fatcat_tools/importers/datacite.py
index f93362d6..6c050565 100644
--- a/python/fatcat_tools/importers/datacite.py
+++ b/python/fatcat_tools/importers/datacite.py
@@ -478,35 +478,7 @@ class DataciteImporter(EntityImporter):
                 license_slug = slug
             license_extra.append(lic)
 
-        # Release type. Try to determine the release type from a variety of
-        # types supplied in datacite. The "attributes.types.resourceType" is
-        # uncontrolled (170000+ unique values, from "null", "Dataset" to
-        # "Jupyter Notebook" and "Macroseismic Data Points" or "2 days of IP
-        # flows in 2009") citeproc may be the closest, but not always supplied.
-        # Order lookup roughly by completeness of mapping.
-        for typeType in ('citeproc', 'ris', 'schemaOrg', 'bibtex', 'resourceTypeGeneral'):
-            value = attributes.get('types', {}).get(typeType)
-            release_type = DATACITE_TYPE_MAP.get(typeType, {}).get(value)
-            if release_type is not None:
-                break
-
-        if release_type is None:
-            print("[{}] no mapped type: {}".format(doi, value), file=sys.stderr)
-
-        # release_type exception: Global Biodiversity Information Facility
-        # publishes highly interesting datasets, but titles are mostly the same
-        # ("GBIF Occurrence Download" or "Occurrence Download"); set
-        # release_type to "stub" (CSL/FC).
-        if publisher == 'The Global Biodiversity Information Facility':
-            release_type = 'stub'
-
-        # release_type exception: lots of "Experimental Crystal Structure Determination"
-        if publisher == 'Cambridge Crystallographic Data Centre':
-            release_type = 'entry'
-
-        # Supplement files, e.g. "Additional file 1: ASE constructs in questionnaire."
-        if title.lower().startswith('additional file'):
-            release_type = 'stub'
+        release_type = self.datacite_release_type(doi, attributes)
 
         # Language values are varied ("ger", "es", "English", "ENG", "en-us",
         # "other", ...). Try to crush it with langcodes: "It may sound to you
@@ -693,6 +665,85 @@ class DataciteImporter(EntityImporter):
             license_slug=license_slug,
             version=version,
         )
+        re = self.biblio_hacks(re)
+        return re
+
+    @staticmethod
+    def datacite_release_type(doi, attributes):
+        """
+        Release type. Try to determine the release type from a variety of types
+        supplied in datacite. The "attributes.types.resourceType" is
+        uncontrolled (170000+ unique values, from "null", "Dataset" to "Jupyter
+        Notebook" and "Macroseismic Data Points" or "2 days of IP flows in
+        2009") citeproc may be the closest, but not always supplied.  Order
+        lookup roughly by completeness of mapping.
+        """
+
+        release_type = None
+        if not attributes.get('types'):
+            return None
+        types = attributes['types']
+
+        for typeType in ('citeproc', 'ris', 'schemaOrg', 'bibtex', 'resourceTypeGeneral'):
+            value = types.get(typeType)
+            release_type = DATACITE_TYPE_MAP.get(typeType, {}).get(value)
+            if release_type is not None:
+                break
+
+        # special case: figshare "collections" which group other entities
+        if doi.startswith('10.6084/') or doi.startswith('10.25384'):
+            if types.get('resourceType') == "Collection":
+                release_type = "stub"
+
+        if release_type is None:
+            print("[{}] no mapped type: {}".format(doi, types), file=sys.stderr)
+
+        return release_type
+
+    @staticmethod
+    def biblio_hacks(re):
+        """
+        This function handles known special cases. For example,
+        publisher-specific or platform-specific workarounds.
+        """
+
+        # only runs on datacite entities with a DOI
+        assert re.ext_ids.doi
+
+        # release_type exception: Global Biodiversity Information Facility
+        # publishes highly interesting datasets, but titles are mostly the same
+        # ("GBIF Occurrence Download" or "Occurrence Download"); set
+        # release_type to "stub" (CSL/FC).
+        if re.title == 'GBIF Occurrence Download' and re.ext_ids.doi.startswith('10.15468/dl.'):
+            re.release_type = 'stub'
+
+        # release_type exception: lots of "Experimental Crystal Structure Determination"
+        # publisher: "Cambridge Crystallographic Data Centre"
+        if re.ext_ids.doi.startswith('10.5517/'):
+            re.release_type = 'entry'
+
+        # Supplement files, e.g. "Additional file 1: ASE constructs in questionnaire."
+        if re.title.lower().startswith('additional file') and re.release_type in ('article', 'article-journal'):
+            re.release_type = 'component'
+
+        # figshare
+        if re.ext_ids.doi.startswith('10.6084/') or re.ext_ids.doi.startswith('10.25384'):
+            # set version if DOI ends with versioned suffix
+            doi_suffix = re.ext_ids.doi.split('.')[-1]
+            if doi_suffix and doi_suffix.startswith('v') and doi_suffix[1:].isdigit():
+                re.version = doi_suffix
+            # "Figure 123 from " -> component
+            # "Table S1. ;Figure S1;Figure S2. ;Figure S3. ;Figure S4. from Use of organic exudates from two polar diatoms by bacterial isolates from the Arctic ocean"
+            if " from " in re.title and re.release_type not in ('stub', 'graphic'):
+                if re.title.startswith("Figure "):
+                    re.release_type = "component"
+                elif re.title.startswith("Table "):
+                    re.release_type = "component"
+
+        # figshare.com
+        if re.ext_ids.doi.startswith('10.6084/m9.figshare.') and re.extra.get('container_name') is None:
+            re.extra['container_name'] = "figshare.com"
+
         return re
 
     def try_update(self, re):
diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py
index d5891ad1..65a8fcd8 100644
--- a/python/fatcat_tools/workers/changelog.py
+++ b/python/fatcat_tools/workers/changelog.py
@@ -89,7 +89,7 @@ class EntityUpdatesWorker(FatcatWorker):
         self.ingest_file_request_topic = ingest_file_request_topic
         self.poll_interval = poll_interval
         self.consumer_group = "entity-updates"
-        self.ingest_oa_only = True
+        self.ingest_oa_only = False
         self.ingest_pdf_doi_prefix_blocklist = [
             # gbif.org: many DOIs, not PDF fulltext
             "10.15468/",
@@ -101,12 +101,20 @@ class EntityUpdatesWorker(FatcatWorker):
             "10.3932/",
             # ccdc.cam.ac.uk: crystal structures
             "10.5517/",
+            # researchgate: mostly blocks our crawler
+            "10.13140/",
+            # springerlink: mostly blocks crawler
+            "10.1007/",
+            # nature group: mostly blocks crawler
+            "10.1038/",
+            # SAGE: mostly blocks crawler
+            "10.1177/",
+            # IOP: mostly blocks crawler
+            "10.1088/",
         ]
         self.live_pdf_ingest_doi_prefix_acceptlist = [
             # biorxiv and medrxiv
             "10.1101/",
-            # researchgate
-            "10.13140/",
             # the lancet (often hybrid OA)
             "10.1016/s0140-6736",
             "10.1016/s2213-2600",
@@ -151,6 +159,7 @@ class EntityUpdatesWorker(FatcatWorker):
         link_source = ingest_request.get('ingest_request')
         ingest_type = ingest_request.get('ingest_type')
         doi = ingest_request.get('ext_ids', {}).get('doi')
+        es = release_to_elasticsearch(release)
 
         is_document = release.release_type in (
             'article',
@@ -167,6 +176,7 @@ class EntityUpdatesWorker(FatcatWorker):
             'paper-conference',
             'patent',
             'peer_review',
+            'post',
             'report',
             'retraction',
             'review',
@@ -191,7 +201,7 @@ class EntityUpdatesWorker(FatcatWorker):
                     in_acceptlist = True
 
         if self.ingest_oa_only and link_source not in ('arxiv', 'pmc'):
-            es = release_to_elasticsearch(release)
+
             # most datacite documents are in IRs and should be crawled
             is_datacite_doc = False
             if release.extra and ('datacite' in release.extra) and is_document:
@@ -199,6 +209,12 @@ class EntityUpdatesWorker(FatcatWorker):
             if not (es['is_oa'] or in_acceptlist or is_datacite_doc):
                 return False
 
+        # big publishers *generally* have accurate OA metadata, use
+        # preservation networks, and block our crawlers. So unless OA, or
+        # explicitly on accept list, or not preserved, skip crawling
+        if es['publisher_type'] == 'big5' and es['is_preserved'] and not (es['is_oa'] or in_acceptlist):
+            return False
+
         # if ingest_type is pdf but release_type is almost certainly not a PDF,
         # skip it. This is mostly a datacite thing.
         if ingest_type == "pdf" and is_not_pdf:
@@ -209,6 +225,20 @@ class EntityUpdatesWorker(FatcatWorker):
                 if doi.startswith(prefix):
                     return False
 
+        # figshare
+        if doi and doi.startswith('10.6084/') or doi.startswith('10.25384/'):
+            # don't crawl "most recent version" (aka "group") DOIs
+            if not release.version:
+                return False
+
+        # zenodo
+        if doi and doi.startswith('10.5281/'):
+            # if this is a "grouping" DOI of multiple "version" DOIs, do not crawl (will crawl the versioned DOIs)
+            if release.extra and release.extra.get('relations'):
+                for rel in release.extra['relations']:
+                    if (rel.get('relationType') == 'HasVersion' and rel.get('relatedIdentifier', '').startswith('10.5281/')):
+                        return False
+
         return True
 
     def run(self):
diff --git a/python/tests/files/datacite/datacite_result_16.json b/python/tests/files/datacite/datacite_result_16.json
index ef26c47c..d0b933d1 100644
--- a/python/tests/files/datacite/datacite_result_16.json
+++ b/python/tests/files/datacite/datacite_result_16.json
@@ -13,6 +13,7 @@
     "doi": "10.6084/m9.figshare.1282478"
   },
   "extra": {
+    "container_name": "figshare.com",
     "datacite": {
       "license": [
         {
diff --git a/python/tests/files/datacite/datacite_result_25.json b/python/tests/files/datacite/datacite_result_25.json
index 6a29e8de..3101c9d8 100644
--- a/python/tests/files/datacite/datacite_result_25.json
+++ b/python/tests/files/datacite/datacite_result_25.json
@@ -18,7 +18,7 @@
   "refs": [],
   "release_date": "2017-08-24",
   "release_stage": "published",
-  "release_type": "stub",
+  "release_type": "component",
   "release_year": 2017,
   "subtitle": "DEF",
   "title": "Additional file 123: ABC"
diff --git a/python/tests/files/datacite/datacite_result_26.json b/python/tests/files/datacite/datacite_result_26.json
index f6e589ef..35f8700a 100644
--- a/python/tests/files/datacite/datacite_result_26.json
+++ b/python/tests/files/datacite/datacite_result_26.json
@@ -27,7 +27,7 @@
   "refs": [],
   "release_date": "2017-08-24",
   "release_stage": "published",
-  "release_type": "stub",
+  "release_type": "component",
   "release_year": 2017,
   "subtitle": "DEF",
   "title": "Additional file 123: ABC"
diff --git a/python/tests/files/datacite/datacite_result_27.json b/python/tests/files/datacite/datacite_result_27.json
index e934fb41..80fde389 100644
--- a/python/tests/files/datacite/datacite_result_27.json
+++ b/python/tests/files/datacite/datacite_result_27.json
@@ -27,7 +27,7 @@
   "refs": [],
   "release_date": "2017-08-24",
   "release_stage": "published",
-  "release_type": "stub",
+  "release_type": "component",
   "release_year": 2017,
   "subtitle": "DEF",
   "title": "Additional file 123: ABC"
diff --git a/python/tests/files/datacite/datacite_result_28.json b/python/tests/files/datacite/datacite_result_28.json
index bcb1caaf..9eb6511a 100644
--- a/python/tests/files/datacite/datacite_result_28.json
+++ b/python/tests/files/datacite/datacite_result_28.json
@@ -27,7 +27,7 @@
   "refs": [],
   "release_date": "2017-08-24",
   "release_stage": "published",
-  "release_type": "stub",
+  "release_type": "component",
   "release_year": 2017,
   "subtitle": "DEF",
   "title": "Additional file 123: ABC"
diff --git a/python/tests/files/datacite/datacite_result_29.json b/python/tests/files/datacite/datacite_result_29.json
index bcb1caaf..9eb6511a 100644
--- a/python/tests/files/datacite/datacite_result_29.json
+++ b/python/tests/files/datacite/datacite_result_29.json
@@ -27,7 +27,7 @@
   "refs": [],
   "release_date": "2017-08-24",
   "release_stage": "published",
-  "release_type": "stub",
+  "release_type": "component",
   "release_year": 2017,
   "subtitle": "DEF",
   "title": "Additional file 123: ABC"