From fa11747574f086e99459914f93d24bad7a8eacce Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 17 Jun 2021 16:26:50 -0700 Subject: more consistent and defensive lower-casing of DOIs After noticing more upper/lower ambiguity in production. In particular, we have some old ingest requests in sandcrawler DB, which get re-submitted/re-tried, which have capitalized DOIs in the link source id field. --- python/fatcat_tools/importers/ingest.py | 4 ++++ python/fatcat_tools/importers/jalc.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'python/fatcat_tools/importers') diff --git a/python/fatcat_tools/importers/ingest.py b/python/fatcat_tools/importers/ingest.py index 483932ad..ae3e147a 100644 --- a/python/fatcat_tools/importers/ingest.py +++ b/python/fatcat_tools/importers/ingest.py @@ -133,6 +133,8 @@ class IngestFileResultImporter(EntityImporter): extid = request['ext_ids'].get(extid_type) if not extid: continue + if extid_type == 'doi': + extid = extid.lower() try: release = self.api.lookup_release(**{extid_type: extid}) except fatcat_openapi_client.rest.ApiException as err: @@ -217,6 +219,8 @@ class IngestFileResultImporter(EntityImporter): if request.get('link_source') and request.get('link_source_id'): edit_extra['link_source'] = request['link_source'] edit_extra['link_source_id'] = request['link_source_id'] + if edit_extra['link_source'] == 'doi': + edit_extra['link_source_id'] = edit_extra['link_source_id'].lower() return edit_extra diff --git a/python/fatcat_tools/importers/jalc.py b/python/fatcat_tools/importers/jalc.py index 9bf2621c..12f5450f 100644 --- a/python/fatcat_tools/importers/jalc.py +++ b/python/fatcat_tools/importers/jalc.py @@ -5,6 +5,7 @@ import datetime from bs4 import BeautifulSoup import fatcat_openapi_client +from fatcat_tools.normal import clean_doi from .common import EntityImporter, clean, is_cjk, DATE_FMT @@ -171,7 +172,7 @@ class JalcImporter(EntityImporter): doi = None if record.doi: - doi = record.doi.string.lower().strip() + doi = clean_doi(record.doi.string.strip().lower()) if doi.startswith('http://dx.doi.org/'): doi = doi.replace('http://dx.doi.org/', '') elif doi.startswith('https://dx.doi.org/'): -- cgit v1.2.3