diff options
| -rwxr-xr-x | python/fatcat_import.py | 30 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/__init__.py | 1 | ||||
| -rw-r--r-- | python/fatcat_tools/importers/datacite.py | 458 | ||||
| -rw-r--r-- | python/tests/import_datacite.py | 25 |
4 files changed, 514 insertions, 0 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py index 8d82dab3..d7651792 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -166,6 +166,17 @@ def run_cdl_dash_dat(args): print("fileset id: {}".format(fs.ident)) print("link: https://fatcat.wiki/fileset/{}".format(fs.ident)) +def run_datacite(args): + dci = DataciteImporter(args.api, + args.issn_map_file, + edit_batch_size=args.batch_size, + bezerk_mode=args.bezerk_mode) + if args.kafka_mode: + KafkaJsonPusher(fci, args.kafka_hosts, args.kafka_env, "api-datacite", + "fatcat-import", consume_batch_size=args.batch_size).run() + else: + JsonLinePusher(dci, args.json_file).run() + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -439,6 +450,25 @@ def main(): type=str, help="use existing editgroup (instead of creating a new one)") + sub_datacite = subparsers.add_parser('datacite', + help="import datacite.org metadata") + sub_datacite.add_argument('json_file', + help="File with jsonlines from datacite.org v2 API to import from", + default=sys.stdin, type=argparse.FileType('r')) + sub_datacite.add_argument('issn_map_file', + help="ISSN to ISSN-L mapping file", + default=None, type=argparse.FileType('r')) + sub_datacite.add_argument('--kafka-mode', + action='store_true', + help="consume from kafka topic (not stdin)") + sub_datacite.add_argument('--bezerk-mode', + action='store_true', + help="don't lookup existing DOIs, just insert (clobbers; only for fast bootstrap)") + sub_datacite.set_defaults( + func=run_datacite, + auth_var="FATCAT_API_AUTH_TOKEN", + ) + args = parser.parse_args() if not args.__dict__.get("func"): print("tell me what to do!") diff --git a/python/fatcat_tools/importers/__init__.py b/python/fatcat_tools/importers/__init__.py index bb9c5b17..d936605f 100644 --- a/python/fatcat_tools/importers/__init__.py +++ b/python/fatcat_tools/importers/__init__.py @@ -14,6 +14,7 @@ To run an import you combine two classes; one each of: from .common import EntityImporter, JsonLinePusher, LinePusher, CsvPusher, SqlitePusher, Bs4XmlFilePusher, Bs4XmlLargeFilePusher, Bs4XmlLinesPusher, Bs4XmlFileListPusher, KafkaJsonPusher, make_kafka_consumer, clean, is_cjk, LANG_MAP_MARC from .crossref import CrossrefImporter, CROSSREF_TYPE_MAP, lookup_license_slug +from .datacite import DataciteImporter from .jalc import JalcImporter from .jstor import JstorImporter from .arxiv import ArxivRawImporter diff --git a/python/fatcat_tools/importers/datacite.py b/python/fatcat_tools/importers/datacite.py new file mode 100644 index 00000000..faa8e2be --- /dev/null +++ b/python/fatcat_tools/importers/datacite.py @@ -0,0 +1,458 @@ +""" +WIP: Importer for datacite.org data. + +Example doc at: https://gist.github.com/miku/5610a2d64e3fee82d16f5d3f3a295fc8 +""" + +from .common import EntityImporter +import dateparser +import datetime +import fatcat_openapi_client +import json +import sys + +# https://guide.fatcat.wiki/entity_container.html#container_type-vocabulary +CONTAINER_TYPE_MAP = { + 'Journal': 'journal', + 'Series': 'journal', + 'Book Series': 'book-series', +} + +# TODO(martin): merge this with other maps, maybe. +LICENSE_SLUG_MAP = { + "//creativecommons.org/licenses/by/2.0": "CC-BY", + "//creativecommons.org/licenses/by/2.0/uk/legalcode": "CC-BY", + "//creativecommons.org/licenses/by/3.0": "CC-BY", + "//creativecommons.org/licenses/by/3.0/us": "CC-BY", + "//creativecommons.org/licenses/by/4.0": "CC-BY", + "//creativecommons.org/licenses/by/4.0/deed.de": "CC-BY", + "//creativecommons.org/licenses/by/4.0/deed.en_US": "CC-BY", + "//creativecommons.org/licenses/by/4.0/legalcode": "CC-BY", + "//creativecommons.org/licenses/by-nc/2.0": "CC-BY-NC", + "//creativecommons.org/licenses/by-nc/3.0": "CC-BY-NC", + "//creativecommons.org/licenses/by-nc/4.0": "CC-BY-NC", + "//creativecommons.org/licenses/by-nc/4.0/legalcode": "CC-BY-NC", + "//creativecommons.org/licenses/by-nc-nd/3.0": "CC-BY-NC-ND", + "//creativecommons.org/licenses/by-nc-nd/3.0/gr": "CC-BY-NC-ND", + "//creativecommons.org/licenses/by-nc-nd/4.0": "CC-BY-NC-ND", + "//creativecommons.org/licenses/by-nc-nd/4.0": "CC-BY-ND", + "//creativecommons.org/licenses/by-nc-nd/4.0/legalcode": "CC-BY-ND", + "//creativecommons.org/licenses/by-nc-sa/4.0": "CC-BY-NC-SA", + "//creativecommons.org/licenses/by-nc-sa/4.0": "CC-BY-SA", + "//creativecommons.org/licenses/by-nd/4.0": "CC-BY-ND", + "//creativecommons.org/licenses/by-sa/3.0/de": "CC-BY-SA", + "//creativecommons.org/licenses/by-sa/3.0/gr": "CC-BY-SA", + "//creativecommons.org/licenses/by-sa/4.0": "CC-BY-SA", + "//creativecommons.org/licenses/by-sa/4.0/legalcode": "CC-BY-SA", + "//creativecommons.org/licenses/CC-BY/4.0": "CC-BY", + "//creativecommons.org/licenses/publicdomain/zero/1.0": "CC-0", + "//creativecommons.org/publicdomain/zero/1.0": "CC-0", + "//creativecommons.org/publicdomain/zero/1.0": "CC-0", + "//creativecommons.org/publicdomain/zero/1.0/legalcode": "CC-0", + "//opensource.org/licenses/MIT": "MIT", + "//www.elsevier.com/open-access/userlicense/1.0": "ELSEVIER-USER-1.0", + "//www.gnu.org/licenses/gpl-3.0.en.html": "GPLv3", + "//www.gnu.org/licenses/old-licenses/gpl-2.0.en.html": "GPLv2", + "//www.karger.com/Services/SiteLicenses": "KARGER", + "//www.opensource.org/licenses/Apache-2.0": "Apache-2.0", + "//www.opensource.org/licenses/BSD-3-Clause": "BSD-3-Clause", + "//www.opensource.org/licenses/EUPL-1.1": "EUPL-1.1", # redirects to EUPL-1.2 + "//www.opensource.org/licenses/MIT": "MIT", + # "http://royalsocietypublishing.org/licence": "", # OA and "normal", https://royalsociety.org/journals/authors/licence-to-publish/ + # "http://rsc.li/journals-terms-of-use": "RSC", + # "http://www.fu-berlin.de/sites/refubium/rechtliches/Nutzungsbedingungen": "", # 53 UrhG. + # "http://www.nrcresearchpress.com/page/about/CorporateTextAndDataMining": "", + # "http://www.springer.com/tdm": "", + # "https://cds.unistra.fr/vizier-org/licences_vizier.html": "", # Maybe try to "SPN" those: https://web.archive.org/web/*/https://cds.unistra.fr/vizier-org/licences_vizier.html + # "https://link.aps.org/licenses/aps-default-accepted-manuscript-license": "", + # "https://oparu.uni-ulm.de/xmlui/license_opod_v1": "", + # "https://publikationen.bibliothek.kit.edu/kitopen-lizenz": "", + # "https://rightsstatements.org/page/InC/1.0?language=en": "", + # "https://services.ceda.ac.uk/cedasite/register/info": "", + # "https://wdc.dlr.de/ndmc/userfiles/file/NDMC-Data_Sharing_Principles.pdf": "", # 404 + # "https://www.cambridge.org/core/terms": "", + # "https://www.elsevier.com/tdm/userlicense/1.0", + # "info:eu-repo/semantics/closedAccess": "", # https://wiki.surfnet.nl/display/standards/info-eu-repo/#info-eu-repo-AccessRights + # "info:eu-repo/semantics/embargoedAccess": "", + # "info:eu-repo/semantics/openAccess": "", +} + +class DataciteImporter(EntityImporter): + """ + Importer for datacite records. TODO(martin): Do we need issn_map_file? + """ + + def __init__(self, api, issn_map_file, **kwargs): + + eg_desc = kwargs.get('editgroup_description', + "Automated import of Datacite DOI metadata, harvested from REST API") + eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.DataciteImporter') + super().__init__(api, + issn_map_file=issn_map_file, + editgroup_description=eg_desc, + editgroup_extra=eg_extra, + **kwargs) + + self.create_containers = kwargs.get('create_containers', True) + self.read_issn_map_file(issn_map_file) + + def parse_record(self, obj): + """ + TODO(martin): Map datacite to RE. + + WIP, notes: + + * Many subjects, should they end up in extra? + * attributes.creators and attributes.contributors + + $ jq '.attributes.creators[]?.nameType?' datacite.500k | sort | uniq -c | sort -nr + 3963663 "Personal" + 289795 null + 8892 "Organizational" + + Shall we use issued, available? + + { + "date": "2011-11-18", + "dateType": "Accepted" + }, + { + "date": "2011-11-18", + "dateType": "Available" + }, + { + "date": "2011-11-07", + "dateType": "Copyrighted" + }, + { + "date": "2011-11-18", + "dateType": "Issued" + }, + { + "date": "2011-11-07", + "dateType": "Issued" + } + + TODO(martin): Quick analysis of dates and stages. + """ + + if 'attributes' not in obj: + return None + + attributes = obj['attributes'] + + # > Contributors + # + # "attributes.creators[].contributorType": [ + # "author" + # ], + # "attributes.creators[].nameIdentifiers[].nameIdentifierScheme": [ + # "LCNA", + # "GND", + # "email", + # "NAF", + # "OSF", + # "RRID", + # "ORCID", + # "SCOPUS", + # "NRCPID", + # "schema.org", + # "GRID", + # "MGDS", + # "VIAF", + # "JACoW-ID" + # ], + # + # "https://orcid.org/0000-0002-9902-738X", + # "http://jacow.org/JACoW-00001280", + # "Wiebe_Peter", + # "https://osf.io/https://osf.io/kjfuy/", + # "http://www.viaf.org176549220", + # "2239", + # "Jeffries_Martin", + # "https://orcid.org/0000-0002-1493-6630", + # "0000-0002-6233-612X", + # + # "creators": [ + # { + # "name": "Bögli, Hans", + # "nameType": "Personal", + # "givenName": "Hans", + # "familyName": "Bögli", + # "affiliation": [] + # } + # ], + + contribs = [] + + for i, c in enumerate(attributes['creators']): + if not c.get('nameType') == 'Personal': + continue + creator_id = None + for nid in c.get('nameIdentifiers', []): + if not nid.get('nameIdentifierScheme').lower() == "orcid": + continue + orcid = nid.get('nameIdentifier', '').replace('https://orcid.org/', '') + if not orcid: + continue + creator_id = self.lookup_orcid(orcid) + # If creator_id is None, should we create creators? + contribs.append(fatcat_openapi_client.ReleaseContrib( + creator_id=creator_id, + index=i, + raw_name=c.get('name'), + given_name=c.get('givenName'), + surname=c.get('familyName'), + )) + + # > Title + # + # "attributes.titles[].titleType": [ + # "AlternativeTitle", + # "Other", + # "Subtitle", + # null, + # "TranslatedTitle" + # ], + title, subtitle = None, None + + for entry in attributes.get('titles', []): + if not title and 'titleType' not in entry: + title = entry.get('title').strip() + if entry.get('titleType') == 'Subtitle': + subtitle = entry.get('title').strip() + + # > Dates + # + # "attributes.dates[].dateType": [ + # "Accepted", + # "Available" + # "Collected", + # "Copyrighted", + # "Created", + # "Issued", + # "Submitted", + # "Updated", + # "Valid", + # ], + # + # Different documents have different dates defined. Choose the topmost + # available from prio list. + date_type_prio = ( + 'Valid', + 'Issued', + 'Available', + 'Accepted', + 'Submitted', + 'Copyrighted', + 'Collected', + 'Created', + 'Updated', + ) + + release_year, release_date = None, None + for prio in date_type_prio: + dates = attributes.get('dates', []) or [] # Never be None. + for item in dates: + if not item.get('dateType') == prio: + continue + result = dateparser.parse(item.get('date')) + if result is None: + # Unparsable date. + continue + release_date = result + release_year = result.year + if 1000 < release_year < datetime.date.today().year + 5: + # Skip possibly bogus dates. + continue + break + else: + continue + break + + # > Publisher + # + # A few NA values. A few bogus values. + # + publisher = attributes.get('publisher') + + if publisher in ('(:unav)', 'Unknown', 'n.a.', '[s.n.]', '(:unap)'): + publisher = None + if publisher is not None and len(publisher) > 80: + # Arbitrary magic value, TODO(martin): better heuristic. + # Example: "ETH-Bibliothek Zürich, Bildarchiv / Fotograf: Feller, + # Elisabeth, Empfänger, Unbekannt, Fotograf / Fel_041033-RE / Unbekannt, + # Nutzungsrechte müssen durch den Nutzer abgeklärt werden", + # TODO(martin): log misses. + publisher = None + + # > Container + # + # For the moment, only ISSN as container. + # + # "container": { + # "type": "Journal", + # "issue": "8", + # "title": "Angewandte Chemie International Edition", + # "volume": "57", + # "lastPage": "2080", + # "firstPage": "2077", + # "identifier": "14337851", + # "identifierType": "ISSN" + # }, + # + # "attributes.container.type": [ + # "DataRepository", + # "Journal", + # "Series", + # "Book Series" + # ], + # + # "attributes.container.identifierType": [ + # "Handle", + # "ISBN", + # "LISSN", + # "DOI", + # "EISSN", + # "URL", + # "ISSN" + # ], + + container_id = None + container = attributes.get('container', {}) or {} + if container.get('type') in CONTAINER_TYPE_MAP.keys(): + container_type = CONTAINER_TYPE_MAP.get(container['type']) + if container.get('identifier') and container.get('identifierType') == 'ISSN': + issn = container.get('identifier') + if len(issn) == 8: + issn = issn[:4] + "-" + issn[4:] + issnl = self.issn2issnl(issn) + container_id = self.lookup_issnl(issnl) + + if container_id is None and container.get('title'): + ce = fatcat_openapi_client.ContainerEntity( + issnl=issnl, + container_type=container_type, + name=container.get('title'), + ) + ce_edit = self.create_container(ce) + container_id = ce_edit.ident + self._issnl_id_map[issnl] = container_id + + # > License + # + # attributes.rightsList[].rightsUri + # attributes.rightsList[].rights + # attributes.rightsList[].lang + # + + license_slug = None + license_extra = [] + for l in attributes.get('rightsList', []): + slug = lookup_license_slug(l.get('rightsUri')) + if slug: + license_slug = slug + license_extra.append(l) + + # > Release type. + # + # Datacite has some fine granular typing (e.g. "Supplementary + # Collection of Datasets", "Taxonomic treatment", "blog_entry", ... + # + # Additional, coarse: resourceTypeGeneral + # + # "attributes.types.resourceTypeGeneral": [ + # "Image", + # "Dataset", + # "PhysicalObject", + # "Collection", + # "Text", + # "Sound", + # "InteractiveResource", + # "Event", + # "Software", + # "Other", + # "Workflow", + # "Audiovisual" + # ], + + # > Extra information. + extra, extra_datacite = dict(), dict() + if license_extra: + extra_datacite['license'] = license_extra + + if extra_datacite: + extra['datacite'] = extra_datacite + + # https://guide.fatcat.wiki/entity_release.html + re = fatcat_openapi_client.ReleaseEntity( + work_id=None, + container_id=container_id, + release_type=None, + release_stage=None, + title=title, # attributes.titles, various titleType + subtitle=subtitle, + original_title=title, # AlternativeTitle? + release_year=release_year, # publicationYear + release_date=release_date, # date issues/available? + publisher=publisher, # attributes.publisher + ext_ids=fatcat_openapi_client.ReleaseExtIds( + doi=attributes.get('doi'), # attributes.doi, + # Can we add handle.net link? + ), + contribs=contribs, + volume=None, + issue=None, + pages=None, + language=None, + abstracts=None, + refs=None, + extra=extra, + license_slug=license_slug, + ) + return re + + def try_update(self, re, debug=True): + if debug is True: + # print(type(re)) + print(json.dumps(re.to_dict(), default=extended_encoder)) + return + return False + + def insert_batch(self, batch): + # Debugging. + for item in batch: + print(item) + return + + # Orig. + self.api.create_release_auto_batch(fatcat_openapi_client.ReleaseAutoBatch( + editgroup=fatcat_openapi_client.Editgroup( + description=self.editgroup_description, + extra=self.editgroup_extra), + entity_list=batch)) + +def extended_encoder(value): + """ + Can be used with json.dumps(value, default=extended_encoder) to serialize + value not serializable by default. https://docs.python.org/3/library/json.html#basic-usage + """ + if isinstance(value, (datetime.datetime, datetime.date)): + return value.isoformat() + if isinstance(value, set): + return list(value) + +def lookup_license_slug(raw): + """ + TODO(martin): reuse from crossref, maybe. + """ + if not raw: + return None + raw = raw.strip().replace('http://', '//').replace('https://', '//') + if 'creativecommons.org' in raw.lower(): + raw = raw.lower() + raw = raw.replace('/legalcode', '/').replace('/uk', '') + if not raw.endswith('/'): + raw = raw + '/' + return LICENSE_SLUG_MAP.get(raw) diff --git a/python/tests/import_datacite.py b/python/tests/import_datacite.py new file mode 100644 index 00000000..0bbaba2e --- /dev/null +++ b/python/tests/import_datacite.py @@ -0,0 +1,25 @@ +""" +Test datacite importer. + +Datacite is a aggregator, hence inputs are quite varied. + +Here is small sample of ID types taken from a sample: + + 497344 "DOI" + 65013 "URL" + 22210 "CCDC" + 17853 "GBIF" + 17635 "Other" + 11474 "uri" + 9170 "Publisher ID" + 7775 "URN" + 6196 "DUCHAS" + 5624 "Handle" + 5056 "publisherId" + +A nice tool, not yet existing tool (maybe named indigo) would do the following: + + $ shuf -n 100000 datacite.ndjson | indigo -t md > data.md + +TODO(martin): Write tests. +""" |