clean up extra/ folder a bit

1 files changed, 0 insertions, 232 deletions

diff --git a/extra/fixups/fixup_longtail_issnl_unique.py b/extra/fixups/fixup_longtail_issnl_unique.py
deleted file mode 100755
index ea615a13..00000000
--- a/extra/fixups/fixup_longtail_issnl_unique.py
+++ /dev/null
@@ -1,232 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-This file must be moved to the fatcat:python/ directory (aka, not in
-fatcat:extra/fixups) to run. It's a "one-off", so probably will bitrot pretty
-quickly. There are no tests.
-
-Example invocation:
-
-    zcat /srv/fatcat/datasets/2018-09-23-0405.30-dumpgrobidmetainsertable.longtail_join.filtered.tsv.gz | ./fixup_longtail_issnl_unique.py /srv/fatcat/datasets/single_domain_issnl.tsv -
-
-See also:
-- bnewbold/scratch:mellon/201904_longtail_issn.md
-- aitio:/rapida/OA-JOURNAL-TESTCRAWL-TWO-2018
-- https://archive.org/details/OA-JOURNAL-TESTCRAWL-TWO-2018-extra
-= https://archive.org/download/ia_longtail_dumpgrobidmetainsertable_2018-09-23/2018-09-23-0405.30-dumpgrobidmetainsertable.longtail_join.filtered.tsv.gz
-
-QA notes:
-
-- everything on revistas.uv.mx linked to 2395-9495, which is only one journal
-  on that domain. blacklist 'revistas' in the domain?
-- afjg3yjdjbf2dad47t5jq7nlbe => 2305-7254 ok match but not perfect (wrong year
-  of conference). probably better than nothing. 
-- elib.mi.sanu.ac.rs has several journals on domain. container_name was correct.
-- revistavirtual.ucn.edu.co has 2x journals
-- lpchkxkp5jecdgrab33fxodd7y bad match
-- k36web33jvf25by64gop4yil7q an IR, not a journal (ok)
-- hvxercwasjhotpewb5xfadyyle good match, though only an abstract (in URL). full
-  articles get DOIs
-- release_epkiok6y3zhsnp3no2lkljznza not a paper; journal match batch (cato, wtf)
-- release_b3jolh25mbg4djrqotgosyeike jfr.unibo.it good
-- release_bzr35evb4bdd3mxex6gxn6dcyy conf.ostis.net good?
-- uzspace.uzulu.ac.za IR, not a container
-- release_5lt36yy3vre2nnig46toy67kdi wrong, multiple journals
-- release_54hmv5gvtjghjk7rpcbp2pn2ky good
-- release_6h7doxfaxnao3jm7f6jkfdpdwm good
-- release_6pio5hz6bvawfnodhkvmfk4jei correct but stub
-- release_7oobqygqczapbgdvvgbxfyvqli correct
-- release_tsljmbevpzfpxiezzv7puwbilq good
-
-general notes:
-- GROBID works pretty well. references look pretty good, should match. there is
-  a non-trivial fraction of non-journal content, but it isn't too bad
-- this "single-journal domain" premise doesn't work
-- could probably do a subset based on "is the journal name in the domain name",
-  or "is domain acronym of journal name"
-- surprising number of IRs with ISSNs in here
-- might have better luck blacklisting out latin american TLDs, which tend to
-  host many journals?
-"""
-
-import os, sys, argparse
-import json
-import sqlite3
-import itertools
-
-import fatcat_openapi_client
-from fatcat_tools import authenticated_api
-from fatcat_tools.importers.common import EntityImporter, clean, LinePusher
-from fatcat_tools.importers.arabesque import b32_hex
-
-
-class LongtailIssnlSingleDomainFixup(EntityImporter):
-    """
-    Fixup script for bootstrap longtail OA release entities which don't have a
-    container but are confidently associated with an ISSN-L based on file
-    domain.
-
-    Expected to be a one-time fixup impacting about 600k entities (around half
-    the longtail OA batch).
-
-    Reads in a mapping of unique domain-ISSNL mappings, and then iterates over
-    the original matched import batch file. For each line in the later:
-    
-    - checks if in-scope based on domain-ISSNL map
-    - uses API to lookup file (by SHA-1) and confirm domain in URL list
-    - look up releases for file and retain the longtail-oa ones (an extra flag)
-    - if release is longtail-oa and no container, set the container based on
-      ISSN-L (using cached lookup)
-    - use EntityImporter stuff to manage update/editgroup queue
-    """
-
-    def __init__(self, api, domain_issnl_tsv_file, **kwargs):
-
-        eg_desc = kwargs.pop('editgroup_description',
-            "Fixup for longtail OA releases that can be matched to specific container by file domain / ISSN-L mapping")
-        eg_extra = kwargs.pop('editgroup_extra', dict())
-        eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.LongtailIssnlSingleDomainFixup')
-        super().__init__(api,
-            editgroup_description=eg_desc,
-            editgroup_extra=eg_extra,
-            **kwargs)
-
-        self._domain_issnl_map = self.load_domain_issnl(domain_issnl_tsv_file)
-        self._issnl_container_map = dict()
-
-    def load_domain_issnl(self, tsv_file):
-        print("Loading domain ISSN-L file...")
-        m = dict()
-        for l in tsv_file:
-            l = l.strip().split('\t')
-            assert len(l) == 2
-            domain = l[0].lower()
-            issnl = l[1]
-            assert len(issnl) == 9 and issnl[4] == '-'
-            m[domain] = issnl
-        print("Got {} matchings.".format(len(m)))
-        return m
-
-    def want(self, raw_record):
-        # do it all in parse_record()
-        return True
-
-    def parse_record(self, row):
-        """
-        TSV rows:
-        - sha1 b32 key
-        - JSON string: CDX-ish
-            - surt
-            - url
-            - <etc>
-        - mime
-        - size (?)
-        - JSON string: grobid metadata
-        """
-
-        # parse row
-        row = row.split('\t')
-        assert len(row) == 5
-        sha1 = b32_hex(row[0][5:])
-        cdx_dict = json.loads(row[1])
-        url = cdx_dict['url']
-        domain = url.split('/')[2].lower()
-
-        if not domain:
-            self.counts['skip-domain-blank'] += 1
-            return None
-
-        # domain in scope?
-        issnl = self._domain_issnl_map.get(domain)
-        if not issnl:
-            self.counts['skip-domain-scope'] += 1
-            return None
-        if 'revistas' in domain.lower().split('.'):
-            self.counts['skip-domain-revistas'] += 1
-            return None
-
-        # lookup file
-        #print(sha1)
-        try:
-            file_entity = self.api.lookup_file(sha1=sha1, expand="releases")
-        except fatcat_openapi_client.rest.ApiException as err:
-            if err.status == 404:
-                self.counts['skip-file-not-found'] += 1
-                return None
-            else:
-                raise err
-
-        # container ident
-        container_id = self.lookup_issnl(issnl)
-        if not container_id:
-            self.counts['skip-container-not-found'] += 1
-            return None
-
-        # confirm domain
-        url_domain_match = False
-        for furl in file_entity.urls:
-            fdomain = furl.url.split('/')[2].lower()
-            if domain == fdomain:
-                url_domain_match = True
-                break
-        if not url_domain_match:
-            self.counts['skip-no-domain-match'] += 1
-            return None
-
-        # fetch releases
-        releases = [r for r in file_entity.releases if (r.extra.get('longtail_oa') == True and r.container_id == None)]
-        if not releases:
-            #print(file_entity.releases)
-            self.counts['skip-no-releases'] += 1
-            return None
-
-        # fetch full release objects (need abstract, etc, for updating)
-        releases = [self.api.get_release(r.ident) for r in releases]
-
-        # set container_id
-        for r in releases:
-            r.container_id = container_id
-        return releases
-
-    def try_update(self, re_list):
-        for re in re_list:
-            self.api.update_release(self.get_editgroup_id(), re.ident, re)
-            self.counts['update'] += 1
-        return False
-
-    def insert_batch(self, batch):
-        raise NotImplementedError
-
-def run_fixup(args):
-    fmi = LongtailIssnlSingleDomainFixup(args.api,
-        args.domain_issnl_tsv_file,
-        edit_batch_size=args.batch_size)
-    LinePusher(fmi, args.insertable_tsv_file).run()
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--api-host-url',
-        default="http://localhost:9411/v0",
-        help="connect to this host/port")
-    parser.add_argument('--batch-size',
-        help="size of batch to send",
-        default=50, type=int)
-    parser.add_argument('domain_issnl_tsv_file',
-        help="domain/ISSNL mapping TSV file",
-        type=argparse.FileType('r'))
-    parser.add_argument('insertable_tsv_file',
-        help="dumpgrobidmetainsertable TSV file to work over",
-        default=sys.stdin, type=argparse.FileType('r'))
-
-    auth_var = "FATCAT_AUTH_SANDCRAWLER"
-
-    args = parser.parse_args()
-
-    args.api = authenticated_api(
-        args.api_host_url,
-        # token is an optional kwarg (can be empty string, None, etc)
-        token=os.environ.get(auth_var))
-    run_fixup(args)
-
-if __name__ == '__main__':
-    main()