diff options
Diffstat (limited to 'python/fatcat_tools/importers')
| -rw-r--r-- | python/fatcat_tools/importers/__init__.py | 5 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/pubmed.py | 512 |
2 files changed, 515 insertions, 2 deletions
diff --git a/python/fatcat_tools/importers/__init__.py b/python/fatcat_tools/importers/__init__.py index 8ec219f8..6f8849d6 100644 --- a/python/fatcat_tools/importers/__init__.py +++ b/python/fatcat_tools/importers/__init__.py @@ -12,11 +12,12 @@ To run an import you combine two classes; one each of: """ -from .common import EntityImporter, JsonLinePusher, LinePusher, CsvPusher, SqlitePusher, Bs4XmlFilePusher, KafkaJsonPusher, make_kafka_consumer, clean, is_cjk -from .crossref import CrossrefImporter, CROSSREF_TYPE_MAP +from .common import EntityImporter, JsonLinePusher, LinePusher, CsvPusher, SqlitePusher, Bs4XmlFilePusher, KafkaJsonPusher, make_kafka_consumer, clean, is_cjk, LANG_MAP_MARC +from .crossref import CrossrefImporter, CROSSREF_TYPE_MAP, lookup_license_slug from .jalc import JalcImporter from .jstor import JstorImporter from .arxiv import ArxivRawImporter +from .pubmed import PubmedImporter from .grobid_metadata import GrobidMetadataImporter from .journal_metadata import JournalMetadataImporter from .matched import MatchedImporter diff --git a/python/fatcat_tools/importers/pubmed.py b/python/fatcat_tools/importers/pubmed.py new file mode 100644 index 00000000..1feb41cd --- /dev/null +++ b/python/fatcat_tools/importers/pubmed.py @@ -0,0 +1,512 @@ + +import sys +import json +import sqlite3 +import datetime +import warnings +from bs4 import BeautifulSoup +from bs4.element import NavigableString + +import fatcat_client +from .common import EntityImporter, clean, LANG_MAP_MARC + +# from: https://www.ncbi.nlm.nih.gov/books/NBK3827/table/pubmedhelp.T.publication_types/?report=objectonly +PUBMED_RELEASE_TYPE_MAP = { + #Adaptive Clinical Trial + "Address": "speech", + "Autobiography": "book", + #Bibliography + "Biography": "book", + #Case Reports + "Classical Article": "article-journal", + #Clinical Conference + #Clinical Study + #Clinical Trial + #Clinical Trial, Phase I + #Clinical Trial, Phase II + #Clinical Trial, Phase III + #Clinical Trial, Phase IV + #Clinical Trial Protocol + #Clinical Trial, Veterinary + #Collected Works + #Comparative Study + #Congress + #Consensus Development Conference + #Consensus Development Conference, NIH + #Controlled Clinical Trial + "Dataset": "dataset", + #Dictionary + #Directory + #Duplicate Publication + "Editorial": "editorial", + #English Abstract # doesn't indicate that this is abstract-only + #Equivalence Trial + #Evaluation Studies + #Expression of Concern + #Festschrift + #Government Document + #Guideline + "Historical Article": "article-journal", + #Interactive Tutorial + "Interview": "interview", + "Introductory Journal Article": "article-journal", + "Journal Article": "article-journal", + "Lecture": "speech", + "Legal Case": "legal_case", + "Legislation": "legislation", + "Letter": "letter", + #Meta-Analysis + #Multicenter Study + #News + "Newspaper Article": "article-newspaper", + #Observational Study + #Observational Study, Veterinary + #Overall + #Patient Education Handout + #Periodical Index + #Personal Narrative + #Portrait + #Practice Guideline + #Pragmatic Clinical Trial + #Publication Components + #Publication Formats + #Publication Type Category + #Randomized Controlled Trial + #Research Support, American Recovery and Reinvestment Act + #Research Support, N.I.H., Extramural + #Research Support, N.I.H., Intramural + #Research Support, Non-U.S. Gov't Research Support, U.S. Gov't, Non-P.H.S. + #Research Support, U.S. Gov't, P.H.S. + #Review # in the "literature review" sense, not "product review" + #Scientific Integrity Review + #Study Characteristics + #Support of Research + #Systematic Review + "Technical Report": "report", + #Twin Study + #Validation Studies + #Video-Audio Media + #Webcasts +} + +MONTH_ABBR_MAP = { + "Jan": 1, "01": 1, + "Feb": 2, "02": 2, + "Mar": 3, "03": 3, + "Apr": 4, "04": 4, + "May": 5, "05": 5, + "Jun": 6, "06": 6, + "Jul": 7, "07": 7, + "Aug": 8, "08": 8, + "Sep": 9, "09": 9, + "Oct": 10, "10": 10, + "Nov": 11, "11": 11, + "Dec": 12, "12": 12, +} + + +class PubmedImporter(EntityImporter): + """ + Importer for PubMed/MEDLINE XML metadata. + + TODO: MEDLINE doesn't include PMC/OA license; could include in importer? + TODO: clean (ftfy) title, original title, etc + XXX: withdrawn + XXX: full author names + """ + + def __init__(self): + pass + + def __init__(self, api, issn_map_file, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Automated import of PubMed/MEDLINE XML metadata") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.PubmedImporter') + super().__init__(api, + issn_map_file=issn_map_file, + editgroup_description=eg_desc, + editgroup_extra=eg_extra, + **kwargs) + + extid_map_file = kwargs.get('extid_map_file') + self.extid_map_db = None + if extid_map_file: + db_uri = "file:{}?mode=ro".format(extid_map_file) + print("Using external ID map: {}".format(db_uri)) + self.extid_map_db = sqlite3.connect(db_uri, uri=True) + else: + print("Not using external ID map") + + self.create_containers = kwargs.get('create_containers') + self.read_issn_map_file(issn_map_file) + + def lookup_ext_ids(self, pmid): + if self.extid_map_db is None: + return dict(doi=None, core_id=None, pmid=None, pmcid=None, + wikidata_qid=None, arxiv_id=None, jstor_id=None) + row = self.extid_map_db.execute("SELECT core, doi, pmcid, wikidata FROM ids WHERE pmid=? LIMIT 1", + [pmid]).fetchone() + if row is None: + return dict(doi=None, core_id=None, pmid=None, pmcid=None, + wikidata_qid=None, arxiv_id=None, jstor_id=None) + row = [str(cell or '') or None for cell in row] + return dict( + core_id=row[0], + doi=row[1], + pmcid=row[2], + wikidata_qid=row[3], + # TODO: + arxiv_id=None, + jstor_id=None, + ) + + def want(self, obj): + return True + + def parse_record(self, a): + + medline = a.MedlineCitation + # PubmedData isn't required by DTD, but seems to always be present + pubmed = a.PubmedData + extra = dict() + extra_pubmed = dict() + + identifiers = pubmed.ArticleIdList + pmid = medline.PMID.string.strip() + doi = identifiers.find("ArticleId", IdType="doi") + if doi: + doi = doi.string.lower() + + pmcid = identifiers.find("ArticleId", IdType="pmc") + if pmcid: + pmcid = pmcid.string + + release_type = None + pub_types = [] + for pub_type in medline.Article.PublicationTypeList.find_all("PublicationType"): + pub_types.append(pub_type.string) + if pub_type.string in PUBMED_RELEASE_TYPE_MAP: + release_type = PUBMED_RELEASE_TYPE_MAP[pub_type.string] + break + if pub_types: + extra_pubmed['pub_types'] = pub_types + if medline.Article.PublicationTypeList.find(string="Retraction of Publication"): + release_type = "retraction" + retraction_of = medline.find("CommentsCorrections", RefType="RetractionOf") + if retraction_of: + extra_pubmed['retraction_of_raw'] = retraction_of.RefSource.string + extra_pubmed['retraction_of_pmid'] = retraction_of.PMID.string + + # everything in medline is published + release_stage = "published" + if medline.Article.PublicationTypeList.find(string="Corrected and Republished Article"): + release_stage = "updated" + if medline.Article.PublicationTypeList.find(string="Retraction of Publication"): + release_stage = "retraction" + if medline.Article.PublicationTypeList.find(string="Retracted Publication"): + withdrawn_status = "retracted" + + pages = medline.find('MedlinePgn') + if pages: + pages = pages.string + + title = medline.Article.ArticleTitle.string # always present + if title: + if title.endswith('.'): + title = title[:-1] + # this hides some "special" titles, but the vast majority are + # translations; translations don't always include the original_title + if title.startswith('[') and title.endswith(']'): + title = title[1:-1] + else: + # TODO: will filter out later + title = None + + original_title = medline.Article.find("VernacularTitle", recurse=False) + if original_title: + original_title = original_title.string or None + if original_title and original_title.endswith('.'): + original_title = original_title[:-1] + + # TODO: happening in alpha order, not handling multi-language well. + # also need to convert lang codes: https://www.nlm.nih.gov/bsd/language_table.html + language = medline.Article.Language + if language: + language = language.string + # TODO: map to two-letter + if language in ("und", "un"): + # "undetermined" + language = None + else: + language = LANG_MAP_MARC.get(language) + if not language: + warnings.warn("MISSING MARC LANG: {}".format(medline.Article.Language.string)) + + ### Journal/Issue Metadata + # MedlineJournalInfo is always present + issnl = None + container_id = None + container_name = None + container_extra = dict() + mji = medline.MedlineJournalInfo + if mji.find("Country"): + container_extra['country_name'] = mji.Country.string + if mji.find("ISSNLinking"): + issnl = mji.ISSNLinking.string + + journal = medline.Article.Journal + issnp = journal.find("ISSN", IssnType="Print") + if issnp: + container_extra['issnp'] = issnp.string + if not issnl: + issnll = self.issn2issnl(issnp) + + if issnl: + container_id = self.lookup_issnl(issnl) + + pub_date = journal.PubDate + release_date = None + release_year = None + if pub_date.Year: + release_year = int(pub_date.Year.string) + if pub_date.find("Day") and pub_date.find("Month"): + release_date = datetime.date( + release_year, + MONTH_ABBR_MAP[pub_date.Month.string], + int(pub_date.Day.string)) + release_date = release_date.isoformat() + elif pub_date.find("MedlineDate") and False: #XXX more/better date parsing? + release_year = int(pub_date.MedlineDate.string.split()[0][:4]) + + if journal.find("Title"): + container_name = journal.Title.string + + if (container_id is None and self.create_containers and (issnl is not None) + and container_name): + # name, type, publisher, issnl + # extra: issnp, issne, original_name, languages, country + ce = fatcat_client.ContainerEntity( + name=container_name, + container_type='journal', + #XXX: publisher not included? + issnl=issnl, + extra=(container_extra or None)) + ce_edit = self.create_container(ce) + container_id = ce_edit.ident + + ji = journal.JournalIssue + volume = None + if ji.find("Volume"): + volume = ji.Volume.string + issue = None + if ji.find("Issue"): + issue = ji.Issue.string + + ### Abstracts + # "All abstracts are in English" + abstracts = [] + first_abstract = medline.find("AbstractText") + if first_abstract and first_abstract.get('NlmCategory'): + joined = "\n".join([m.get_text() for m in medline.find_all("AbstractText")]) + abstracts.append(fatcat_client.ReleaseAbstract( + content=joined, + mimetype="text/plain", + lang="en", + )) + else: + for abstract in medline.find_all("AbstractText"): + abstracts.append(fatcat_client.ReleaseAbstract( + content=abstract.get_text().strip(), + mimetype="text/plain", + lang="en", + )) + if abstract.find('math'): + abstracts.append(fatcat_client.ReleaseAbstract( + # strip the <AbstractText> tags + content=str(abstract)[14:-15], + mimetype="application/mathml+xml", + lang="en", + )) + if not abstracts: + abstracts = None + + ### Contribs + contribs = [] + if medline.AuthorList: + for author in medline.AuthorList.find_all("Author"): + given_name = None + surname = None + raw_name = None + if author.ForeName: + given_name = author.ForeName.string + if author.LastName: + surname = author.LastName.string + if given_name and surname: + raw_name = "{} {}".format(given_name, surname) + elif surname: + raw_name = surname + contrib_extra = dict() + orcid = author.find("Identifier", Source="ORCID") + if orcid: + # needs re-formatting from, eg, "0000000179841889" + orcid = orcid.string + if orcid.startswith("http://orcid.org/"): + orcid = orcid.replace("http://orcid.org/", "") + elif orcid.startswith("https://orcid.org/"): + orcid = orcid.replace("https://orcid.org/", "") + elif not '-' in orcid: + orcid = "{}-{}-{}-{}".format( + orcid[0:4], + orcid[4:8], + orcid[8:12], + orcid[12:16], + ) + # XXX: do lookup by ORCID + #contrib_extra['orcid'] = orcid + affiliation = author.find("Affiliation") + raw_affiliation = None + if affiliation: + raw_affiliation = affiliation.string + if author.find("EqualContrib"): + # TODO: schema for this? + contrib_extra['equal_contrib'] = True + contribs.append(fatcat_client.ReleaseContrib( + raw_name=raw_name, + given_name=given_name, + surname=surname, + role="author", + raw_affiliation=raw_affiliation, + extra=contrib_extra, + )) + + if medline.AuthorList['CompleteYN'] == 'N': + contribs.append(fatcat_client.ReleaseContrib(raw_name="et al.")) + if not contribs: + contribs = None + + ### References + refs = [] + if pubmed.ReferenceList: + for ref in pubmed.ReferenceList.find_all('Reference'): + ref_obj = dict() + ref_extra = dict() + ref_pmid = ref.find("ArticleId", IdType="pubmed") + if ref_pmid: + ref_extra['pmid'] = ref_pmid.string + # TODO: do reference lookups here based on PMID/DOI + ref_raw = ref.Citation + if ref_raw: + ref_extra['unstructured'] = ref_raw.string + if ref_extra: + ref_obj['extra'] = ref_extra + refs.append(fatcat_client.ReleaseRef( + extra=ref_obj.get('extra'), + )) + if not refs: + refs = None + + # extra: + # withdrawn_date + # translation_of + # subtitle + # aliases + # container_name + # group-title + # pubmed: retraction refs + if extra_pubmed: + extra['pubmed'] = extra_pubmed + if not extra: + extra = None + + re = fatcat_client.ReleaseEntity( + work_id=None, + title=clean(title), + original_title=clean(original_title), + release_type=release_type, + release_stage=release_stage, + release_date=release_date, + release_year=release_year, + ext_ids=fatcat_client.ReleaseExtIds( + doi=doi, + pmid=pmid, + pmcid=pmcid, + #isbn13 # never in Article + ), + volume=volume, + issue=issue, + pages=pages, + #publisher # not included? + language=language, + #license_slug # not in MEDLINE + abstracts=abstracts, + contribs=contribs, + refs=refs, + container_id=container_id, + extra=extra, + ) + return re + + def try_update(self, re): + + # first, lookup existing by PMID (which must be defined) + existing = None + try: + existing = self.api.lookup_release(pmid=re.ext_ids.pmid) + except fatcat_client.rest.ApiException as err: + if err.status != 404: + raise err + + # then try DOI lookup if there is one + if not existing and re.ext_ids.doi: + try: + existing = self.api.lookup_release(doi=re.ext_ids.doi) + except fatcat_client.rest.ApiException as err: + if err.status != 404: + raise err + if existing and existing.ext_ids.pmid and existing.ext_ids.pmid != re.ext_ids.pmid: + warnings.warn("PMID/DOI mismatch: release {}, pmid {} != {}".format( + existing.ident, existing.ext_ids.pmid, re.ext_ids.pmid)) + self.counts['exists-pmid-doi-mismatch'] += 1 + return False + + if existing and existing.ext_ids.pmid and existing.refs: + # TODO: any other reasons to do an update? + # don't update if it already has PMID + self.counts['exists'] += 1 + return False + elif existing: + # but do update if only DOI was set + existing.ext_ids.doi = existing.ext_ids.doi or re.ext_ids.doi + existing.ext_ids.pmid = existing.ext_ids.pmid or re.ext_ids.pmid + existing.ext_ids.pmcid = existing.ext_ids.pmcid or re.ext_ids.pmcid + existing.refs = existing.refs or re.refs + existing.extra['pubmed'] = re.extra['pubmed'] + self.api.update_release(self.get_editgroup_id(), existing.ident, existing) + self.counts['update'] += 1 + return False + + return True + + def insert_batch(self, batch): + self.api.create_release_auto_batch(fatcat_client.ReleaseAutoBatch( + editgroup=fatcat_client.Editgroup( + description=self.editgroup_description, + extra=self.editgroup_extra), + entity_list=batch)) + + def parse_file(self, handle): + + # 1. open with beautiful soup + soup = BeautifulSoup(handle, "xml") + + # 2. iterate over articles, call parse_article on each + for article in soup.find_all("PubmedArticle"): + resp = self.parse_article(article) + print(json.dumps(resp)) + #sys.exit(-1) + +if __name__=='__main__': + parser = PubMedParser() + parser.parse_file(open(sys.argv[1])) |