diff options
| -rw-r--r-- | python/fatcat_tools/importers/__init__.py | 5 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/pubmed.py (renamed from python/parse_pubmed_xml.py) | 290 | ||||
| -rw-r--r-- | python/tests/import_pubmed.py | 80 |
3 files changed, 298 insertions, 77 deletions
diff --git a/python/fatcat_tools/importers/__init__.py b/python/fatcat_tools/importers/__init__.py index 8ec219f8..6f8849d6 100644 --- a/python/fatcat_tools/importers/__init__.py +++ b/python/fatcat_tools/importers/__init__.py @@ -12,11 +12,12 @@ To run an import you combine two classes; one each of: """ -from .common import EntityImporter, JsonLinePusher, LinePusher, CsvPusher, SqlitePusher, Bs4XmlFilePusher, KafkaJsonPusher, make_kafka_consumer, clean, is_cjk -from .crossref import CrossrefImporter, CROSSREF_TYPE_MAP +from .common import EntityImporter, JsonLinePusher, LinePusher, CsvPusher, SqlitePusher, Bs4XmlFilePusher, KafkaJsonPusher, make_kafka_consumer, clean, is_cjk, LANG_MAP_MARC +from .crossref import CrossrefImporter, CROSSREF_TYPE_MAP, lookup_license_slug from .jalc import JalcImporter from .jstor import JstorImporter from .arxiv import ArxivRawImporter +from .pubmed import PubmedImporter from .grobid_metadata import GrobidMetadataImporter from .journal_metadata import JournalMetadataImporter from .matched import MatchedImporter diff --git a/python/parse_pubmed_xml.py b/python/fatcat_tools/importers/pubmed.py index 413333cc..1feb41cd 100644 --- a/python/parse_pubmed_xml.py +++ b/python/fatcat_tools/importers/pubmed.py @@ -1,10 +1,15 @@ import sys import json +import sqlite3 import datetime +import warnings from bs4 import BeautifulSoup from bs4.element import NavigableString +import fatcat_client +from .common import EntityImporter, clean, LANG_MAP_MARC + # from: https://www.ncbi.nlm.nih.gov/books/NBK3827/table/pubmedhelp.T.publication_types/?report=objectonly PUBMED_RELEASE_TYPE_MAP = { #Adaptive Clinical Trial @@ -99,29 +104,68 @@ MONTH_ABBR_MAP = { "Dec": 12, "12": 12, } -class PubMedParser(): - """ - Converts PubMed/MEDLINE XML into in release entity (which can dump as JSON) +class PubmedImporter(EntityImporter): + """ + Importer for PubMed/MEDLINE XML metadata. + TODO: MEDLINE doesn't include PMC/OA license; could include in importer? TODO: clean (ftfy) title, original title, etc + XXX: withdrawn + XXX: full author names """ def __init__(self): pass - def parse_file(self, handle): - - # 1. open with beautiful soup - soup = BeautifulSoup(handle, "xml") + def __init__(self, api, issn_map_file, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Automated import of PubMed/MEDLINE XML metadata") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.PubmedImporter') + super().__init__(api, + issn_map_file=issn_map_file, + editgroup_description=eg_desc, + editgroup_extra=eg_extra, + **kwargs) + + extid_map_file = kwargs.get('extid_map_file') + self.extid_map_db = None + if extid_map_file: + db_uri = "file:{}?mode=ro".format(extid_map_file) + print("Using external ID map: {}".format(db_uri)) + self.extid_map_db = sqlite3.connect(db_uri, uri=True) + else: + print("Not using external ID map") + + self.create_containers = kwargs.get('create_containers') + self.read_issn_map_file(issn_map_file) + + def lookup_ext_ids(self, pmid): + if self.extid_map_db is None: + return dict(doi=None, core_id=None, pmid=None, pmcid=None, + wikidata_qid=None, arxiv_id=None, jstor_id=None) + row = self.extid_map_db.execute("SELECT core, doi, pmcid, wikidata FROM ids WHERE pmid=? LIMIT 1", + [pmid]).fetchone() + if row is None: + return dict(doi=None, core_id=None, pmid=None, pmcid=None, + wikidata_qid=None, arxiv_id=None, jstor_id=None) + row = [str(cell or '') or None for cell in row] + return dict( + core_id=row[0], + doi=row[1], + pmcid=row[2], + wikidata_qid=row[3], + # TODO: + arxiv_id=None, + jstor_id=None, + ) - # 2. iterate over articles, call parse_article on each - for article in soup.find_all("PubmedArticle"): - resp = self.parse_article(article) - print(json.dumps(resp)) - #sys.exit(-1) + def want(self, obj): + return True - def parse_article(self, a): + def parse_record(self, a): medline = a.MedlineCitation # PubmedData isn't required by DTD, but seems to always be present @@ -130,6 +174,7 @@ class PubMedParser(): extra_pubmed = dict() identifiers = pubmed.ArticleIdList + pmid = medline.PMID.string.strip() doi = identifiers.find("ArticleId", IdType="doi") if doi: doi = doi.string.lower() @@ -139,10 +184,14 @@ class PubMedParser(): pmcid = pmcid.string release_type = None + pub_types = [] for pub_type in medline.Article.PublicationTypeList.find_all("PublicationType"): + pub_types.append(pub_type.string) if pub_type.string in PUBMED_RELEASE_TYPE_MAP: release_type = PUBMED_RELEASE_TYPE_MAP[pub_type.string] - break + break + if pub_types: + extra_pubmed['pub_types'] = pub_types if medline.Article.PublicationTypeList.find(string="Retraction of Publication"): release_type = "retraction" retraction_of = medline.find("CommentsCorrections", RefType="RetractionOf") @@ -151,11 +200,13 @@ class PubMedParser(): extra_pubmed['retraction_of_pmid'] = retraction_of.PMID.string # everything in medline is published - release_status = "published" + release_stage = "published" if medline.Article.PublicationTypeList.find(string="Corrected and Republished Article"): - release_status = "updated" + release_stage = "updated" + if medline.Article.PublicationTypeList.find(string="Retraction of Publication"): + release_stage = "retraction" if medline.Article.PublicationTypeList.find(string="Retracted Publication"): - release_status = "retracted" + withdrawn_status = "retracted" pages = medline.find('MedlinePgn') if pages: @@ -188,27 +239,37 @@ class PubMedParser(): if language in ("und", "un"): # "undetermined" language = None + else: + language = LANG_MAP_MARC.get(language) + if not language: + warnings.warn("MISSING MARC LANG: {}".format(medline.Article.Language.string)) ### Journal/Issue Metadata # MedlineJournalInfo is always present - container = dict() + issnl = None + container_id = None + container_name = None container_extra = dict() mji = medline.MedlineJournalInfo if mji.find("Country"): container_extra['country_name'] = mji.Country.string if mji.find("ISSNLinking"): - container['issnl'] = mji.ISSNLinking.string + issnl = mji.ISSNLinking.string journal = medline.Article.Journal issnp = journal.find("ISSN", IssnType="Print") if issnp: container_extra['issnp'] = issnp.string + if not issnl: + issnll = self.issn2issnl(issnp) + + if issnl: + container_id = self.lookup_issnl(issnl) pub_date = journal.PubDate release_date = None - if pub_date.find("MedlineDate"): - release_year = int(pub_date.MedlineDate.string.split()[0][:4]) - else: + release_year = None + if pub_date.Year: release_year = int(pub_date.Year.string) if pub_date.find("Day") and pub_date.find("Month"): release_date = datetime.date( @@ -216,6 +277,24 @@ class PubMedParser(): MONTH_ABBR_MAP[pub_date.Month.string], int(pub_date.Day.string)) release_date = release_date.isoformat() + elif pub_date.find("MedlineDate") and False: #XXX more/better date parsing? + release_year = int(pub_date.MedlineDate.string.split()[0][:4]) + + if journal.find("Title"): + container_name = journal.Title.string + + if (container_id is None and self.create_containers and (issnl is not None) + and container_name): + # name, type, publisher, issnl + # extra: issnp, issne, original_name, languages, country + ce = fatcat_client.ContainerEntity( + name=container_name, + container_type='journal', + #XXX: publisher not included? + issnl=issnl, + extra=(container_extra or None)) + ce_edit = self.create_container(ce) + container_id = ce_edit.ident ji = journal.JournalIssue volume = None @@ -224,13 +303,6 @@ class PubMedParser(): issue = None if ji.find("Issue"): issue = ji.Issue.string - if journal.find("Title"): - container['name'] = journal.Title.string - - if extra_pubmed: - extra['pubmed'] = extra_pubmed - if not extra: - extra = None ### Abstracts # "All abstracts are in English" @@ -238,20 +310,20 @@ class PubMedParser(): first_abstract = medline.find("AbstractText") if first_abstract and first_abstract.get('NlmCategory'): joined = "\n".join([m.get_text() for m in medline.find_all("AbstractText")]) - abstracts.append(dict( + abstracts.append(fatcat_client.ReleaseAbstract( content=joined, mimetype="text/plain", lang="en", )) else: for abstract in medline.find_all("AbstractText"): - abstracts.append(dict( + abstracts.append(fatcat_client.ReleaseAbstract( content=abstract.get_text().strip(), mimetype="text/plain", lang="en", )) if abstract.find('math'): - abstracts.append(dict( + abstracts.append(fatcat_client.ReleaseAbstract( # strip the <AbstractText> tags content=str(abstract)[14:-15], mimetype="application/mathml+xml", @@ -264,13 +336,17 @@ class PubMedParser(): contribs = [] if medline.AuthorList: for author in medline.AuthorList.find_all("Author"): - contrib = dict( - role="author", - ) + given_name = None + surname = None + raw_name = None if author.ForeName: - contrib['raw_name'] = "{} {}".format(author.ForeName.string, author.LastName.string) - elif author.LastName: - contrib['raw_name'] = author.LastName.string + given_name = author.ForeName.string + if author.LastName: + surname = author.LastName.string + if given_name and surname: + raw_name = "{} {}".format(given_name, surname) + elif surname: + raw_name = surname contrib_extra = dict() orcid = author.find("Identifier", Source="ORCID") if orcid: @@ -287,19 +363,26 @@ class PubMedParser(): orcid[8:12], orcid[12:16], ) - contrib_extra['orcid'] = orcid + # XXX: do lookup by ORCID + #contrib_extra['orcid'] = orcid affiliation = author.find("Affiliation") + raw_affiliation = None if affiliation: - contrib['raw_affiliation'] = affiliation.string + raw_affiliation = affiliation.string if author.find("EqualContrib"): # TODO: schema for this? contrib_extra['equal_contrib'] = True - if contrib_extra: - contrib['extra'] = contrib_extra - contribs.append(contrib) + contribs.append(fatcat_client.ReleaseContrib( + raw_name=raw_name, + given_name=given_name, + surname=surname, + role="author", + raw_affiliation=raw_affiliation, + extra=contrib_extra, + )) if medline.AuthorList['CompleteYN'] == 'N': - contribs.append(dict(raw_name="et al.")) + contribs.append(fatcat_client.ReleaseContrib(raw_name="et al.")) if not contribs: contribs = None @@ -312,61 +395,118 @@ class PubMedParser(): ref_pmid = ref.find("ArticleId", IdType="pubmed") if ref_pmid: ref_extra['pmid'] = ref_pmid.string + # TODO: do reference lookups here based on PMID/DOI ref_raw = ref.Citation if ref_raw: - ref_extra['raw'] = ref_raw.string + ref_extra['unstructured'] = ref_raw.string if ref_extra: ref_obj['extra'] = ref_extra - refs.append(ref_obj) + refs.append(fatcat_client.ReleaseRef( + extra=ref_obj.get('extra'), + )) if not refs: refs = None - re = dict( + # extra: + # withdrawn_date + # translation_of + # subtitle + # aliases + # container_name + # group-title + # pubmed: retraction refs + if extra_pubmed: + extra['pubmed'] = extra_pubmed + if not extra: + extra = None + + re = fatcat_client.ReleaseEntity( work_id=None, - title=title, - original_title=original_title, + title=clean(title), + original_title=clean(original_title), release_type=release_type, - release_status=release_status, + release_stage=release_stage, release_date=release_date, release_year=release_year, - doi=doi, - pmid=int(medline.PMID.string), # always present - pmcid=pmcid, - #isbn13 # never in Article + ext_ids=fatcat_client.ReleaseExtIds( + doi=doi, + pmid=pmid, + pmcid=pmcid, + #isbn13 # never in Article + ), volume=volume, issue=issue, pages=pages, #publisher # not included? language=language, #license_slug # not in MEDLINE - - # content, mimetype, lang abstracts=abstracts, - - # raw_name, role, raw_affiliation, extra contribs=contribs, - - # key, year, container_name, title, locator - # extra: volume, authors, issue, publisher, identifiers refs=refs, - - # name, type, publisher, issnl - # extra: issnp, issne, original_name, languages, country - container=container, - - # extra: - # withdrawn_date - # translation_of - # subtitle - # aliases - # container_name - # group-title - # pubmed: retraction refs + container_id=container_id, extra=extra, ) - return re + def try_update(self, re): + + # first, lookup existing by PMID (which must be defined) + existing = None + try: + existing = self.api.lookup_release(pmid=re.ext_ids.pmid) + except fatcat_client.rest.ApiException as err: + if err.status != 404: + raise err + + # then try DOI lookup if there is one + if not existing and re.ext_ids.doi: + try: + existing = self.api.lookup_release(doi=re.ext_ids.doi) + except fatcat_client.rest.ApiException as err: + if err.status != 404: + raise err + if existing and existing.ext_ids.pmid and existing.ext_ids.pmid != re.ext_ids.pmid: + warnings.warn("PMID/DOI mismatch: release {}, pmid {} != {}".format( + existing.ident, existing.ext_ids.pmid, re.ext_ids.pmid)) + self.counts['exists-pmid-doi-mismatch'] += 1 + return False + + if existing and existing.ext_ids.pmid and existing.refs: + # TODO: any other reasons to do an update? + # don't update if it already has PMID + self.counts['exists'] += 1 + return False + elif existing: + # but do update if only DOI was set + existing.ext_ids.doi = existing.ext_ids.doi or re.ext_ids.doi + existing.ext_ids.pmid = existing.ext_ids.pmid or re.ext_ids.pmid + existing.ext_ids.pmcid = existing.ext_ids.pmcid or re.ext_ids.pmcid + existing.refs = existing.refs or re.refs + existing.extra['pubmed'] = re.extra['pubmed'] + self.api.update_release(self.get_editgroup_id(), existing.ident, existing) + self.counts['update'] += 1 + return False + + return True + + def insert_batch(self, batch): + self.api.create_release_auto_batch(fatcat_client.ReleaseAutoBatch( + editgroup=fatcat_client.Editgroup( + description=self.editgroup_description, + extra=self.editgroup_extra), + entity_list=batch)) + + def parse_file(self, handle): + + # 1. open with beautiful soup + soup = BeautifulSoup(handle, "xml") + + # 2. iterate over articles, call parse_article on each + for article in soup.find_all("PubmedArticle"): + resp = self.parse_article(article) + print(json.dumps(resp)) + #sys.exit(-1) + if __name__=='__main__': parser = PubMedParser() parser.parse_file(open(sys.argv[1])) diff --git a/python/tests/import_pubmed.py b/python/tests/import_pubmed.py new file mode 100644 index 00000000..eacc3815 --- /dev/null +++ b/python/tests/import_pubmed.py @@ -0,0 +1,80 @@ + +import json, gzip +import pytest +from fatcat_tools.importers import PubmedImporter, Bs4XmlFilePusher +from fixtures import api +from bs4 import BeautifulSoup + + +@pytest.fixture(scope="function") +def pubmed_importer(api): + with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file: + yield PubmedImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', bezerk_mode=True) + +@pytest.fixture(scope="function") +def pubmed_importer_existing(api): + with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file: + yield PubmedImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', bezerk_mode=False) + +def test_pubmed_importer(pubmed_importer): + last_index = pubmed_importer.api.get_changelog(limit=1)[0].index + with open('tests/files/pubmedsample_2019.xml', 'r') as f: + pubmed_importer.bezerk_mode = True + counts = Bs4XmlFilePusher(pubmed_importer, f, "PubmedArticle").run() + assert counts['insert'] == 1 + assert counts['exists'] == 0 + assert counts['skip'] == 0 + + # fetch most recent editgroup + change = pubmed_importer.api.get_changelog_entry(index=last_index+1) + eg = change.editgroup + assert eg.description + assert "pubmed" in eg.description.lower() + assert eg.extra['git_rev'] + assert "fatcat_tools.PubmedImporter" in eg.extra['agent'] + + last_index = pubmed_importer.api.get_changelog(limit=1)[0].index + with open('tests/files/pubmedsample_2019.xml', 'r') as f: + pubmed_importer.bezerk_mode = False + pubmed_importer.reset() + counts = Bs4XmlFilePusher(pubmed_importer, f, "PubmedArticle").run() + assert counts['insert'] == 0 + assert counts['exists'] == 1 + assert counts['skip'] == 0 + assert last_index == pubmed_importer.api.get_changelog(limit=1)[0].index + +def test_pubmed_xml_parse(pubmed_importer): + with open('tests/files/pubmedsample_2019.xml', 'r') as f: + soup = BeautifulSoup(f, "xml") + r1 = pubmed_importer.parse_record(soup.find_all("PubmedArticle")[0]) + r2 = pubmed_importer.parse_record(soup.find_all("PubmedArticle")[-1]) + + assert r1.title == "Hospital debt management and cost reimbursement" + assert r1.subtitle == None + assert r1.original_title == None + assert r1.publisher == None + assert r1.release_type == "article-journal" + assert r1.release_stage == "published" + assert r1.license_slug == None + assert r1.ext_ids.doi == None + assert r1.ext_ids.pmid == "973217" + assert r1.language == "en" + assert r1.volume == "3" + assert r1.issue == "1" + assert r1.pages == "69-81" + assert r1.release_date == None # not "1976-12-03", which is medline ingest date + assert r1.release_year == 1976 + # matched by ISSN, so shouldn't be in there? + #assert extra['container_name'] == "Abstracts of the Papers Communicated to the Royal Society of London" + assert len(r1.contribs) == 1 + + assert r1.contribs[0].raw_name == "F R Blume" + assert r1.contribs[0].given_name == "F R" + assert r1.contribs[0].surname == "Blume" + + print(r1.extra) + # TODO: assert r1.extra['pubmed']['mesh_topics'] == ['Accounting', 'Economics, Hospital', 'Hospital Administration'] + assert r1.extra['pubmed']['pub_types'] == ['Journal Article'] + assert not r1.refs + + # XXX: r2 tests |