aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2021-06-17 16:26:50 -0700
committerBryan Newbold <bnewbold@robocracy.org>2021-06-23 17:51:15 -0700
commitfa11747574f086e99459914f93d24bad7a8eacce (patch)
tree9e05825c26de60042142dd846cee31d85661a6b4 /python
parentcf2c3432d3c28d2ba0a92c62fabda4ca434afe46 (diff)
downloadfatcat-fa11747574f086e99459914f93d24bad7a8eacce.tar.gz
fatcat-fa11747574f086e99459914f93d24bad7a8eacce.zip
more consistent and defensive lower-casing of DOIs
After noticing more upper/lower ambiguity in production. In particular, we have some old ingest requests in sandcrawler DB, which get re-submitted/re-tried, which have capitalized DOIs in the link source id field.
Diffstat (limited to 'python')
-rw-r--r--python/fatcat_tools/importers/ingest.py4
-rw-r--r--python/fatcat_tools/importers/jalc.py3
-rw-r--r--python/fatcat_tools/transforms/ingest.py4
3 files changed, 8 insertions, 3 deletions
diff --git a/python/fatcat_tools/importers/ingest.py b/python/fatcat_tools/importers/ingest.py
index 483932ad..ae3e147a 100644
--- a/python/fatcat_tools/importers/ingest.py
+++ b/python/fatcat_tools/importers/ingest.py
@@ -133,6 +133,8 @@ class IngestFileResultImporter(EntityImporter):
extid = request['ext_ids'].get(extid_type)
if not extid:
continue
+ if extid_type == 'doi':
+ extid = extid.lower()
try:
release = self.api.lookup_release(**{extid_type: extid})
except fatcat_openapi_client.rest.ApiException as err:
@@ -217,6 +219,8 @@ class IngestFileResultImporter(EntityImporter):
if request.get('link_source') and request.get('link_source_id'):
edit_extra['link_source'] = request['link_source']
edit_extra['link_source_id'] = request['link_source_id']
+ if edit_extra['link_source'] == 'doi':
+ edit_extra['link_source_id'] = edit_extra['link_source_id'].lower()
return edit_extra
diff --git a/python/fatcat_tools/importers/jalc.py b/python/fatcat_tools/importers/jalc.py
index 9bf2621c..12f5450f 100644
--- a/python/fatcat_tools/importers/jalc.py
+++ b/python/fatcat_tools/importers/jalc.py
@@ -5,6 +5,7 @@ import datetime
from bs4 import BeautifulSoup
import fatcat_openapi_client
+from fatcat_tools.normal import clean_doi
from .common import EntityImporter, clean, is_cjk, DATE_FMT
@@ -171,7 +172,7 @@ class JalcImporter(EntityImporter):
doi = None
if record.doi:
- doi = record.doi.string.lower().strip()
+ doi = clean_doi(record.doi.string.strip().lower())
if doi.startswith('http://dx.doi.org/'):
doi = doi.replace('http://dx.doi.org/', '')
elif doi.startswith('https://dx.doi.org/'):
diff --git a/python/fatcat_tools/transforms/ingest.py b/python/fatcat_tools/transforms/ingest.py
index 9aaeaa84..42927b2a 100644
--- a/python/fatcat_tools/transforms/ingest.py
+++ b/python/fatcat_tools/transforms/ingest.py
@@ -50,9 +50,9 @@ def release_ingest_request(release, ingest_request_source='fatcat', ingest_type=
link_source = "pmc"
link_source_id = release.ext_ids.pmcid
elif release.ext_ids.doi:
- url = "https://doi.org/{}".format(release.ext_ids.doi)
+ url = "https://doi.org/{}".format(release.ext_ids.doi.lower())
link_source = "doi"
- link_source_id = release.ext_ids.doi
+ link_source_id = release.ext_ids.doi.lower()
if not url:
return None