diff options
Diffstat (limited to 'python/fatcat_tools')
| -rw-r--r-- | python/fatcat_tools/importers/datacite.py | 34 | ||||
| -rw-r--r-- | python/fatcat_tools/workers/changelog.py | 2 |
2 files changed, 36 insertions, 0 deletions
diff --git a/python/fatcat_tools/importers/datacite.py b/python/fatcat_tools/importers/datacite.py index b310f8bc..1d098aca 100644 --- a/python/fatcat_tools/importers/datacite.py +++ b/python/fatcat_tools/importers/datacite.py @@ -511,6 +511,8 @@ class DataciteImporter(EntityImporter): ): relations.append(rel) + # TODO: could use many of these relations to do release/work grouping + if relations: extra_datacite["relations"] = relations @@ -646,6 +648,38 @@ class DataciteImporter(EntityImporter): ): re.extra["container_name"] = "figshare.com" + # Columbia Institutional Repository includes full bibliographic + # metadata, which results in incorrect container_id matches. But this + # DOI prefix also publishes actual journals! + if ( + re.ext_ids.doi.startswith("10.7916/") + and "-" in re.ext_ids.doi + and re.publisher == "Columbia University" + and re.extra + and re.extra.get("datacite") + ): + for relation in re.extra["datacite"].get("relations", []): + if relation.get("relationType") == "IsVariantFormOf": + re.container_id = None + if re.release_stage in ("published", None): + re.release_stage = "submitted" + + # several institutional and other repositories (including "RWTH" and + # "DESY") also results in incorrect container_id matches. + # This probably doesn't filter out enough, but is a start. + IR_DOI_PREFIXES = [ + "10.15495/epub_ubt_", + "10.18154/rwth-20", + "10.3204/pubdb-", + "10.3204/phppubdb-", + "10.26204/kluedo/", + ] + for prefix in IR_DOI_PREFIXES and re.extra and re.extra.get("datacite"): + if re.ext_ids.doi.startswith(prefix): + for relation in re.extra["datacite"].get("relations", []): + if relation.get("relationType") == "IsVariantFormOf": + re.container_id = None + return re def try_update(self, re: ReleaseEntity) -> bool: diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index 1af47d4b..436e0e00 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -131,6 +131,8 @@ class EntityUpdatesWorker(FatcatWorker): "10.1088/", # JSTOR: mostly blocks crawler "10.2307/", + # arxiv: duplicates with arxiv identifiers (temporary) + "10.48550/", ] self.live_pdf_ingest_doi_prefix_acceptlist = [ # biorxiv and medrxiv |