diff options
Diffstat (limited to 'python/fatcat_tools/importers/pubmed.py')
| -rw-r--r-- | python/fatcat_tools/importers/pubmed.py | 355 |
1 files changed, 197 insertions, 158 deletions
diff --git a/python/fatcat_tools/importers/pubmed.py b/python/fatcat_tools/importers/pubmed.py index 00ad54d0..cfdafcf7 100644 --- a/python/fatcat_tools/importers/pubmed.py +++ b/python/fatcat_tools/importers/pubmed.py @@ -1,4 +1,3 @@ - import datetime import json import sys @@ -13,42 +12,42 @@ from .common import LANG_MAP_MARC, EntityImporter, clean # from: https://www.ncbi.nlm.nih.gov/books/NBK3827/table/pubmedhelp.T.publication_types/?report=objectonly PUBMED_RELEASE_TYPE_MAP = { - #Adaptive Clinical Trial + # Adaptive Clinical Trial "Address": "speech", "Autobiography": "book", - #Bibliography + # Bibliography "Biography": "book", - #Case Reports + # Case Reports "Classical Article": "article-journal", - #Clinical Conference - #Clinical Study - #Clinical Trial - #Clinical Trial, Phase I - #Clinical Trial, Phase II - #Clinical Trial, Phase III - #Clinical Trial, Phase IV - #Clinical Trial Protocol - #Clinical Trial, Veterinary - #Collected Works - #Comparative Study - #Congress - #Consensus Development Conference - #Consensus Development Conference, NIH - #Controlled Clinical Trial + # Clinical Conference + # Clinical Study + # Clinical Trial + # Clinical Trial, Phase I + # Clinical Trial, Phase II + # Clinical Trial, Phase III + # Clinical Trial, Phase IV + # Clinical Trial Protocol + # Clinical Trial, Veterinary + # Collected Works + # Comparative Study + # Congress + # Consensus Development Conference + # Consensus Development Conference, NIH + # Controlled Clinical Trial "Dataset": "dataset", - #Dictionary - #Directory - #Duplicate Publication + # Dictionary + # Directory + # Duplicate Publication "Editorial": "editorial", - #English Abstract # doesn't indicate that this is abstract-only - #Equivalence Trial - #Evaluation Studies - #Expression of Concern - #Festschrift - #Government Document - #Guideline + # English Abstract # doesn't indicate that this is abstract-only + # Equivalence Trial + # Evaluation Studies + # Expression of Concern + # Festschrift + # Government Document + # Guideline "Historical Article": "article-journal", - #Interactive Tutorial + # Interactive Tutorial "Interview": "interview", "Introductory Journal Article": "article-journal", "Journal Article": "article-journal", @@ -56,53 +55,65 @@ PUBMED_RELEASE_TYPE_MAP = { "Legal Case": "legal_case", "Legislation": "legislation", "Letter": "letter", - #Meta-Analysis - #Multicenter Study - #News + # Meta-Analysis + # Multicenter Study + # News "Newspaper Article": "article-newspaper", - #Observational Study - #Observational Study, Veterinary - #Overall - #Patient Education Handout - #Periodical Index - #Personal Narrative - #Portrait - #Practice Guideline - #Pragmatic Clinical Trial - #Publication Components - #Publication Formats - #Publication Type Category - #Randomized Controlled Trial - #Research Support, American Recovery and Reinvestment Act - #Research Support, N.I.H., Extramural - #Research Support, N.I.H., Intramural - #Research Support, Non-U.S. Gov't Research Support, U.S. Gov't, Non-P.H.S. - #Research Support, U.S. Gov't, P.H.S. - #Review # in the "literature review" sense, not "product review" - #Scientific Integrity Review - #Study Characteristics - #Support of Research - #Systematic Review + # Observational Study + # Observational Study, Veterinary + # Overall + # Patient Education Handout + # Periodical Index + # Personal Narrative + # Portrait + # Practice Guideline + # Pragmatic Clinical Trial + # Publication Components + # Publication Formats + # Publication Type Category + # Randomized Controlled Trial + # Research Support, American Recovery and Reinvestment Act + # Research Support, N.I.H., Extramural + # Research Support, N.I.H., Intramural + # Research Support, Non-U.S. Gov't Research Support, U.S. Gov't, Non-P.H.S. + # Research Support, U.S. Gov't, P.H.S. + # Review # in the "literature review" sense, not "product review" + # Scientific Integrity Review + # Study Characteristics + # Support of Research + # Systematic Review "Technical Report": "report", - #Twin Study - #Validation Studies - #Video-Audio Media - #Webcasts + # Twin Study + # Validation Studies + # Video-Audio Media + # Webcasts } MONTH_ABBR_MAP = { - "Jan": 1, "01": 1, - "Feb": 2, "02": 2, - "Mar": 3, "03": 3, - "Apr": 4, "04": 4, - "May": 5, "05": 5, - "Jun": 6, "06": 6, - "Jul": 7, "07": 7, - "Aug": 8, "08": 8, - "Sep": 9, "09": 9, - "Oct": 10, "10": 10, - "Nov": 11, "11": 11, - "Dec": 12, "12": 12, + "Jan": 1, + "01": 1, + "Feb": 2, + "02": 2, + "Mar": 3, + "03": 3, + "Apr": 4, + "04": 4, + "May": 5, + "05": 5, + "Jun": 6, + "06": 6, + "Jul": 7, + "07": 7, + "Aug": 8, + "08": 8, + "Sep": 9, + "09": 9, + "Oct": 10, + "10": 10, + "Nov": 11, + "11": 11, + "Dec": 12, + "12": 12, } # From: https://www.ncbi.nlm.nih.gov/books/NBK7249/ @@ -295,11 +306,10 @@ COUNTRY_NAME_MAP = { "United Kingdom": "gb", "United States": "us", "Uruguay": "uy", - # Additions from running over large files "Bosnia and Herzegovina": "ba", - #"International" - "China (Republic : 1949- )": "tw", # pretty sure this is tw not cn + # "International" + "China (Republic : 1949- )": "tw", # pretty sure this is tw not cn "Russia (Federation)": "ru", "Scotland": "gb", "England": "gb", @@ -320,18 +330,21 @@ class PubmedImporter(EntityImporter): def __init__(self, api, issn_map_file, lookup_refs=True, **kwargs): - eg_desc = kwargs.get('editgroup_description', - "Automated import of PubMed/MEDLINE XML metadata") - eg_extra = kwargs.get('editgroup_extra', dict()) - eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.PubmedImporter') - super().__init__(api, + eg_desc = kwargs.get( + "editgroup_description", "Automated import of PubMed/MEDLINE XML metadata" + ) + eg_extra = kwargs.get("editgroup_extra", dict()) + eg_extra["agent"] = eg_extra.get("agent", "fatcat_tools.PubmedImporter") + super().__init__( + api, issn_map_file=issn_map_file, editgroup_description=eg_desc, editgroup_extra=eg_extra, - **kwargs) + **kwargs + ) self.lookup_refs = lookup_refs - self.create_containers = kwargs.get('create_containers', True) + self.create_containers = kwargs.get("create_containers", True) self.read_issn_map_file(issn_map_file) def want(self, obj): @@ -365,15 +378,15 @@ class PubmedImporter(EntityImporter): release_type = PUBMED_RELEASE_TYPE_MAP[pub_type.string] break if pub_types: - extra_pubmed['pub_types'] = pub_types + extra_pubmed["pub_types"] = pub_types if medline.Article.PublicationTypeList.find(string="Retraction of Publication"): release_type = "retraction" retraction_of = medline.find("CommentsCorrections", RefType="RetractionOf") if retraction_of: if retraction_of.RefSource: - extra_pubmed['retraction_of_raw'] = retraction_of.RefSource.string + extra_pubmed["retraction_of_raw"] = retraction_of.RefSource.string if retraction_of.PMID: - extra_pubmed['retraction_of_pmid'] = retraction_of.PMID.string + extra_pubmed["retraction_of_pmid"] = retraction_of.PMID.string # everything in medline is published release_stage = "published" @@ -388,18 +401,18 @@ class PubmedImporter(EntityImporter): elif medline.find("CommentsCorrections", RefType="ExpressionOfConcernIn"): withdrawn_status = "concern" - pages = medline.find('MedlinePgn') + pages = medline.find("MedlinePgn") if pages: pages = pages.string - title = medline.Article.ArticleTitle.get_text() # always present + title = medline.Article.ArticleTitle.get_text() # always present if title: - title = title.replace('\n', ' ') - if title.endswith('.'): + title = title.replace("\n", " ") + if title.endswith("."): title = title[:-1] # this hides some "special" titles, but the vast majority are # translations; translations don't always include the original_title - if title.startswith('[') and title.endswith(']'): + if title.startswith("[") and title.endswith("]"): title = title[1:-1] else: # will filter out later @@ -408,8 +421,8 @@ class PubmedImporter(EntityImporter): original_title = medline.Article.find("VernacularTitle", recurse=False) if original_title: original_title = original_title.get_text() or None - original_title = original_title.replace('\n', ' ') - if original_title and original_title.endswith('.'): + original_title = original_title.replace("\n", " ") + if original_title and original_title.endswith("."): original_title = original_title[:-1] if original_title and not title: @@ -428,7 +441,9 @@ class PubmedImporter(EntityImporter): else: language = LANG_MAP_MARC.get(language) if not language and not (medline.Article.Language.get_text() in LANG_MAP_MARC): - warnings.warn("MISSING MARC LANG: {}".format(medline.Article.Language.string)) + warnings.warn( + "MISSING MARC LANG: {}".format(medline.Article.Language.string) + ) ### Journal/Issue Metadata # MedlineJournalInfo is always present @@ -441,9 +456,9 @@ class PubmedImporter(EntityImporter): country_name = mji.Country.string.strip() country_code = COUNTRY_NAME_MAP.get(country_name) if country_code: - container_extra['country'] = country_code + container_extra["country"] = country_code elif country_name: - container_extra['country_name'] = country_name + container_extra["country_name"] = country_name if mji.find("ISSNLinking"): issnl = mji.ISSNLinking.string @@ -462,7 +477,7 @@ class PubmedImporter(EntityImporter): if issnl: container_id = self.lookup_issnl(issnl) - pub_date = medline.Article.find('ArticleDate') + pub_date = medline.Article.find("ArticleDate") if not pub_date: pub_date = journal.PubDate if not pub_date: @@ -476,7 +491,8 @@ class PubmedImporter(EntityImporter): release_date = datetime.date( release_year, MONTH_ABBR_MAP[pub_date.Month.string], - int(pub_date.Day.string)) + int(pub_date.Day.string), + ) release_date = release_date.isoformat() except ValueError as ve: print("bad date, skipping: {}".format(ve), file=sys.stderr) @@ -486,25 +502,35 @@ class PubmedImporter(EntityImporter): if len(medline_date) >= 4 and medline_date[:4].isdigit(): release_year = int(medline_date[:4]) if release_year < 1300 or release_year > 2040: - print("bad medline year, skipping: {}".format(release_year), file=sys.stderr) + print( + "bad medline year, skipping: {}".format(release_year), file=sys.stderr + ) release_year = None else: - print("unparsable medline date, skipping: {}".format(medline_date), file=sys.stderr) + print( + "unparsable medline date, skipping: {}".format(medline_date), + file=sys.stderr, + ) if journal.find("Title"): container_name = journal.Title.get_text() - if (container_id is None and self.create_containers and (issnl is not None) - and container_name): + if ( + container_id is None + and self.create_containers + and (issnl is not None) + and container_name + ): # name, type, publisher, issnl # extra: original_name, languages, country ce = fatcat_openapi_client.ContainerEntity( name=container_name, - container_type='journal', - #NOTE: publisher not included + container_type="journal", + # NOTE: publisher not included issnl=issnl, issnp=issnp, - extra=(container_extra or None)) + extra=(container_extra or None), + ) ce_edit = self.create_container(ce) container_id = ce_edit.ident self._issnl_id_map[issnl] = container_id @@ -521,8 +547,10 @@ class PubmedImporter(EntityImporter): # "All abstracts are in English" abstracts = [] primary_abstract = medline.find("Abstract") - if primary_abstract and primary_abstract.AbstractText.get('NlmCategory'): - joined = "\n".join([m.get_text() for m in primary_abstract.find_all("AbstractText")]) + if primary_abstract and primary_abstract.AbstractText.get("NlmCategory"): + joined = "\n".join( + [m.get_text() for m in primary_abstract.find_all("AbstractText")] + ) abst = fatcat_openapi_client.ReleaseAbstract( content=joined, mimetype="text/plain", @@ -539,7 +567,7 @@ class PubmedImporter(EntityImporter): ) if abst.content: abstracts.append(abst) - if abstract.find('math'): + if abstract.find("math"): abst = fatcat_openapi_client.ReleaseAbstract( # strip the <AbstractText> tags content=str(abstract)[14:-15], @@ -551,8 +579,8 @@ class PubmedImporter(EntityImporter): other_abstracts = medline.find_all("OtherAbstract") for other in other_abstracts: lang = "en" - if other.get('Language'): - lang = LANG_MAP_MARC.get(other['Language']) + if other.get("Language"): + lang = LANG_MAP_MARC.get(other["Language"]) abst = fatcat_openapi_client.ReleaseAbstract( content=other.AbstractText.get_text().strip(), mimetype="text/plain", @@ -572,15 +600,15 @@ class PubmedImporter(EntityImporter): surname = None raw_name = None if author.ForeName: - given_name = author.ForeName.get_text().replace('\n', ' ') + given_name = author.ForeName.get_text().replace("\n", " ") if author.LastName: - surname = author.LastName.get_text().replace('\n', ' ') + surname = author.LastName.get_text().replace("\n", " ") if given_name and surname: raw_name = "{} {}".format(given_name, surname) elif surname: raw_name = surname if not raw_name and author.CollectiveName and author.CollectiveName.get_text(): - raw_name = author.CollectiveName.get_text().replace('\n', ' ') + raw_name = author.CollectiveName.get_text().replace("\n", " ") contrib_extra = dict() orcid = author.find("Identifier", Source="ORCID") if orcid: @@ -590,7 +618,7 @@ class PubmedImporter(EntityImporter): orcid = orcid.replace("http://orcid.org/", "") elif orcid.startswith("https://orcid.org/"): orcid = orcid.replace("https://orcid.org/", "") - elif '-' not in orcid: + elif "-" not in orcid: orcid = "{}-{}-{}-{}".format( orcid[0:4], orcid[4:8], @@ -598,27 +626,31 @@ class PubmedImporter(EntityImporter): orcid[12:16], ) creator_id = self.lookup_orcid(orcid) - contrib_extra['orcid'] = orcid + contrib_extra["orcid"] = orcid affiliations = author.find_all("Affiliation") raw_affiliation = None if affiliations: - raw_affiliation = affiliations[0].get_text().replace('\n', ' ') + raw_affiliation = affiliations[0].get_text().replace("\n", " ") if len(affiliations) > 1: - contrib_extra['more_affiliations'] = [ra.get_text().replace('\n', ' ') for ra in affiliations[1:]] + contrib_extra["more_affiliations"] = [ + ra.get_text().replace("\n", " ") for ra in affiliations[1:] + ] if author.find("EqualContrib"): # TODO: schema for this? - contrib_extra['equal'] = True - contribs.append(fatcat_openapi_client.ReleaseContrib( - raw_name=raw_name, - given_name=given_name, - surname=surname, - role="author", - raw_affiliation=raw_affiliation, - creator_id=creator_id, - extra=contrib_extra, - )) - - if medline.AuthorList['CompleteYN'] == 'N': + contrib_extra["equal"] = True + contribs.append( + fatcat_openapi_client.ReleaseContrib( + raw_name=raw_name, + given_name=given_name, + surname=surname, + role="author", + raw_affiliation=raw_affiliation, + creator_id=creator_id, + extra=contrib_extra, + ) + ) + + if medline.AuthorList["CompleteYN"] == "N": contribs.append(fatcat_openapi_client.ReleaseContrib(raw_name="et al.")) for i, contrib in enumerate(contribs): @@ -633,7 +665,7 @@ class PubmedImporter(EntityImporter): # note that Reference always exists within a ReferenceList, but # that there may be multiple ReferenceList (eg, sometimes one per # Reference) - for ref in pubmed.find_all('Reference'): + for ref in pubmed.find_all("Reference"): ref_extra = dict() ref_doi = ref.find("ArticleId", IdType="doi") if ref_doi: @@ -643,22 +675,24 @@ class PubmedImporter(EntityImporter): ref_pmid = clean_pmid(ref_pmid.string) ref_release_id = None if ref_doi: - ref_extra['doi'] = ref_doi + ref_extra["doi"] = ref_doi if self.lookup_refs: ref_release_id = self.lookup_doi(ref_doi) if ref_pmid: - ref_extra['pmid'] = ref_pmid + ref_extra["pmid"] = ref_pmid if self.lookup_refs: ref_release_id = self.lookup_pmid(ref_pmid) ref_raw = ref.Citation if ref_raw: - ref_extra['unstructured'] = ref_raw.get_text() + ref_extra["unstructured"] = ref_raw.get_text() if not ref_extra: ref_extra = None - refs.append(fatcat_openapi_client.ReleaseRef( - target_release_id=ref_release_id, - extra=ref_extra, - )) + refs.append( + fatcat_openapi_client.ReleaseRef( + target_release_id=ref_release_id, + extra=ref_extra, + ) + ) if not refs: refs = None @@ -669,7 +703,7 @@ class PubmedImporter(EntityImporter): # group-title # pubmed: retraction refs if extra_pubmed: - extra['pubmed'] = extra_pubmed + extra["pubmed"] = extra_pubmed if not extra: extra = None @@ -690,14 +724,14 @@ class PubmedImporter(EntityImporter): doi=doi, pmid=pmid, pmcid=pmcid, - #isbn13 # never in Article + # isbn13 # never in Article ), volume=volume, issue=issue, pages=pages, - #publisher # not included? + # publisher # not included? language=language, - #license_slug # not in MEDLINE + # license_slug # not in MEDLINE abstracts=abstracts, contribs=contribs, refs=refs, @@ -725,21 +759,22 @@ class PubmedImporter(EntityImporter): raise err if existing and existing.ext_ids.pmid and existing.ext_ids.pmid != re.ext_ids.pmid: warn_str = "PMID/DOI mismatch: release {}, pmid {} != {}".format( - existing.ident, existing.ext_ids.pmid, re.ext_ids.pmid) + existing.ident, existing.ext_ids.pmid, re.ext_ids.pmid + ) warnings.warn(warn_str) - self.counts['warn-pmid-doi-mismatch'] += 1 + self.counts["warn-pmid-doi-mismatch"] += 1 # don't clobber DOI, but do group together re.ext_ids.doi = None re.work_id = existing.work_id if existing and not self.do_updates: - self.counts['exists'] += 1 + self.counts["exists"] += 1 return False if existing and existing.ext_ids.pmid and (existing.refs or not re.refs): # TODO: any other reasons to do an update? # don't update if it already has PMID - self.counts['exists'] += 1 + self.counts["exists"] += 1 return False elif existing: # but do update if only DOI was set @@ -750,12 +785,12 @@ class PubmedImporter(EntityImporter): existing.container_id = existing.container_id or re.container_id existing.refs = existing.refs or re.refs existing.abstracts = existing.abstracts or re.abstracts - existing.extra['pubmed'] = re.extra['pubmed'] + existing.extra["pubmed"] = re.extra["pubmed"] # fix stub titles if existing.title in [ - "OUP accepted manuscript", - ]: + "OUP accepted manuscript", + ]: existing.title = re.title existing.original_title = existing.original_title or re.original_title @@ -770,8 +805,8 @@ class PubmedImporter(EntityImporter): existing.language = existing.language or re.language # update subtitle in-place first - if not existing.subtitle and existing.extra.get('subtitle'): - subtitle = existing.extra.pop('subtitle') + if not existing.subtitle and existing.extra.get("subtitle"): + subtitle = existing.extra.pop("subtitle") if type(subtitle) == list: subtitle = subtitle[0] if subtitle: @@ -781,13 +816,13 @@ class PubmedImporter(EntityImporter): try: self.api.update_release(self.get_editgroup_id(), existing.ident, existing) - self.counts['update'] += 1 + self.counts["update"] += 1 except fatcat_openapi_client.rest.ApiException as err: # there is a code path where we try to update the same release # twice in a row; if that happens, just skip # NOTE: API behavior might change in the future? if "release_edit_editgroup_id_ident_id_key" in err.body: - self.counts['skip-update-conflict'] += 1 + self.counts["skip-update-conflict"] += 1 return False else: raise err @@ -797,11 +832,14 @@ class PubmedImporter(EntityImporter): return True def insert_batch(self, batch): - self.api.create_release_auto_batch(fatcat_openapi_client.ReleaseAutoBatch( - editgroup=fatcat_openapi_client.Editgroup( - description=self.editgroup_description, - extra=self.editgroup_extra), - entity_list=batch)) + self.api.create_release_auto_batch( + fatcat_openapi_client.ReleaseAutoBatch( + editgroup=fatcat_openapi_client.Editgroup( + description=self.editgroup_description, extra=self.editgroup_extra + ), + entity_list=batch, + ) + ) def parse_file(self, handle): @@ -812,8 +850,9 @@ class PubmedImporter(EntityImporter): for article in soup.find_all("PubmedArticle"): resp = self.parse_record(article) print(json.dumps(resp)) - #sys.exit(-1) + # sys.exit(-1) + -if __name__=='__main__': +if __name__ == "__main__": parser = PubmedImporter(None, None) parser.parse_file(open(sys.argv[1])) |