#!/usr/bin/env python3 """ This file must be moved to the fatcat:python/ directory (aka, not in fatcat:extra/fixups) to run. It's a "one-off", so probably will bitrot pretty quickly. There are no tests. Example invocation: zcat /srv/fatcat/datasets/2018-09-23-0405.30-dumpgrobidmetainsertable.longtail_join.filtered.tsv.gz | ./fixup_longtail_issnl_unique.py /srv/fatcat/datasets/single_domain_issnl.tsv - See also: - bnewbold/scratch:mellon/201904_longtail_issn.md - aitio:/rapida/OA-JOURNAL-TESTCRAWL-TWO-2018 - https://archive.org/details/OA-JOURNAL-TESTCRAWL-TWO-2018-extra = https://archive.org/download/ia_longtail_dumpgrobidmetainsertable_2018-09-23/2018-09-23-0405.30-dumpgrobidmetainsertable.longtail_join.filtered.tsv.gz QA notes: - everything on revistas.uv.mx linked to 2395-9495, which is only one journal on that domain. blacklist 'revistas' in the domain? - afjg3yjdjbf2dad47t5jq7nlbe => 2305-7254 ok match but not perfect (wrong year of conference). probably better than nothing. - elib.mi.sanu.ac.rs has several journals on domain. container_name was correct. - revistavirtual.ucn.edu.co has 2x journals - lpchkxkp5jecdgrab33fxodd7y bad match - k36web33jvf25by64gop4yil7q an IR, not a journal (ok) - hvxercwasjhotpewb5xfadyyle good match, though only an abstract (in URL). full articles get DOIs - release_epkiok6y3zhsnp3no2lkljznza not a paper; journal match batch (cato, wtf) - release_b3jolh25mbg4djrqotgosyeike jfr.unibo.it good - release_bzr35evb4bdd3mxex6gxn6dcyy conf.ostis.net good? - uzspace.uzulu.ac.za IR, not a container - release_5lt36yy3vre2nnig46toy67kdi wrong, multiple journals - release_54hmv5gvtjghjk7rpcbp2pn2ky good - release_6h7doxfaxnao3jm7f6jkfdpdwm good - release_6pio5hz6bvawfnodhkvmfk4jei correct but stub - release_7oobqygqczapbgdvvgbxfyvqli correct - release_tsljmbevpzfpxiezzv7puwbilq good general notes: - GROBID works pretty well. references look pretty good, should match. there is a non-trivial fraction of non-journal content, but it isn't too bad - this "single-journal domain" premise doesn't work - could probably do a subset based on "is the journal name in the domain name", or "is domain acronym of journal name" - surprising number of IRs with ISSNs in here - might have better luck blacklisting out latin american TLDs, which tend to host many journals? """ import os, sys, argparse import json import sqlite3 import itertools import fatcat_openapi_client from fatcat_tools import authenticated_api from fatcat_tools.importers.common import EntityImporter, clean, LinePusher from fatcat_tools.importers.arabesque import b32_hex class LongtailIssnlSingleDomainFixup(EntityImporter): """ Fixup script for bootstrap longtail OA release entities which don't have a container but are confidently associated with an ISSN-L based on file domain. Expected to be a one-time fixup impacting about 600k entities (around half the longtail OA batch). Reads in a mapping of unique domain-ISSNL mappings, and then iterates over the original matched import batch file. For each line in the later: - checks if in-scope based on domain-ISSNL map - uses API to lookup file (by SHA-1) and confirm domain in URL list - look up releases for file and retain the longtail-oa ones (an extra flag) - if release is longtail-oa and no container, set the container based on ISSN-L (using cached lookup) - use EntityImporter stuff to manage update/editgroup queue """ def __init__(self, api, domain_issnl_tsv_file, **kwargs): eg_desc = kwargs.pop('editgroup_description', "Fixup for longtail OA releases that can be matched to specific container by file domain / ISSN-L mapping") eg_extra = kwargs.pop('editgroup_extra', dict()) eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.LongtailIssnlSingleDomainFixup') super().__init__(api, editgroup_description=eg_desc, editgroup_extra=eg_extra, **kwargs) self._domain_issnl_map = self.load_domain_issnl(domain_issnl_tsv_file) self._issnl_container_map = dict() def load_domain_issnl(self, tsv_file): print("Loading domain ISSN-L file...") m = dict() for l in tsv_file: l = l.strip().split('\t') assert len(l) == 2 domain = l[0].lower() issnl = l[1] assert len(issnl) == 9 and issnl[4] == '-' m[domain] = issnl print("Got {} matchings.".format(len(m))) return m def want(self, raw_record): # do it all in parse_record() return True def parse_record(self, row): """ TSV rows: - sha1 b32 key - JSON string: CDX-ish - surt - url - - mime - size (?) - JSON string: grobid metadata """ # parse row row = row.split('\t') assert len(row) == 5 sha1 = b32_hex(row[0][5:]) cdx_dict = json.loads(row[1]) url = cdx_dict['url'] domain = url.split('/')[2].lower() if not domain: self.counts['skip-domain-blank'] += 1 return None # domain in scope? issnl = self._domain_issnl_map.get(domain) if not issnl: self.counts['skip-domain-scope'] += 1 return None if 'revistas' in domain.lower().split('.'): self.counts['skip-domain-revistas'] += 1 return None # lookup file #print(sha1) try: file_entity = self.api.lookup_file(sha1=sha1, expand="releases") except fatcat_openapi_client.rest.ApiException as err: if err.status == 404: self.counts['skip-file-not-found'] += 1 return None else: raise err # container ident container_id = self.lookup_issnl(issnl) if not container_id: self.counts['skip-container-not-found'] += 1 return None # confirm domain url_domain_match = False for furl in file_entity.urls: fdomain = furl.url.split('/')[2].lower() if domain == fdomain: url_domain_match = True break if not url_domain_match: self.counts['skip-no-domain-match'] += 1 return None # fetch releases releases = [r for r in file_entity.releases if (r.extra.get('longtail_oa') == True and r.container_id == None)] if not releases: #print(file_entity.releases) self.counts['skip-no-releases'] += 1 return None # fetch full release objects (need abstract, etc, for updating) releases = [self.api.get_release(r.ident) for r in releases] # set container_id for r in releases: r.container_id = container_id return releases def try_update(self, re_list): for re in re_list: self.api.update_release(self.get_editgroup_id(), re.ident, re) self.counts['update'] += 1 return False def insert_batch(self, batch): raise NotImplementedError def run_fixup(args): fmi = LongtailIssnlSingleDomainFixup(args.api, args.domain_issnl_tsv_file, edit_batch_size=args.batch_size) LinePusher(fmi, args.insertable_tsv_file).run() def main(): parser = argparse.ArgumentParser() parser.add_argument('--api-host-url', default="http://localhost:9411/v0", help="connect to this host/port") parser.add_argument('--batch-size', help="size of batch to send", default=50, type=int) parser.add_argument('domain_issnl_tsv_file', help="domain/ISSNL mapping TSV file", type=argparse.FileType('r')) parser.add_argument('insertable_tsv_file', help="dumpgrobidmetainsertable TSV file to work over", default=sys.stdin, type=argparse.FileType('r')) auth_var = "FATCAT_AUTH_SANDCRAWLER" args = parser.parse_args() args.api = authenticated_api( args.api_host_url, # token is an optional kwarg (can be empty string, None, etc) token=os.environ.get(auth_var)) run_fixup(args) if __name__ == '__main__': main()