diff options
Diffstat (limited to 'old')
| -rwxr-xr-x | old/fix_invalid_issnl.py | 75 | ||||
| -rw-r--r-- | old/invalid_fatcat_issnl.tsv | 118 | ||||
| -rwxr-xr-x | old/parse_merge_metadata.py | 674 |
3 files changed, 867 insertions, 0 deletions
diff --git a/old/fix_invalid_issnl.py b/old/fix_invalid_issnl.py new file mode 100755 index 0000000..521f334 --- /dev/null +++ b/old/fix_invalid_issnl.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +""" +This is a one-off script for pushing ISSN-L fixes to fatcat via the API. + +It excpects to have fatcat python libraries available. Run it like: + + export FATCAT_API_AUTH_TOKEN="..." + ./fix_invalid_issnl.py ~/code/chocula/invalid_fatcat_issnl.tsv + +It creates a new editgroup, which you'll need to merge/accept manually. + +Defaults to QA API endpoint; edit the file to switch to prod. +""" + +import os, sys +import csv + +from fatcat_tools import authenticated_api +from fatcat_client import Editgroup, ContainerEntity +from fatcat_client.rest import ApiException + +API_ENDPOINT = "https://api.qa.fatcat.wiki/v0" + + +def run(api, row_iter): + + eg = api.create_editgroup(Editgroup(description= + "Update or merge containers with invalid (by checksum) ISSN-L. Using the fix_invalid_issnl.py script from chocula repo.")) + print("Editgroup ident: {}".format(eg.editgroup_id)) + for row in row_iter: + #print(row) + fixed_issnl = row['fixed_issnl'].strip() + if not fixed_issnl: + print("SKIP") + continue + assert row['issnl'].strip() != fixed_issnl + invalid = api.get_container(row['fatcat_ident']) + assert invalid.state == "active" + try: + fixed = api.lookup_container(issnl=row['fixed_issnl']) + except ApiException as ae: + if ae.status != 404: + raise ae + fixed = None + + if fixed: + # merge/redirect + assert fixed.state == "active" + print("MERGE: {} -> {}".format(invalid.ident, fixed.ident)) + invalid.redirect = fixed.ident + api.update_container(eg.editgroup_id, invalid.ident, + ContainerEntity(redirect=fixed.ident)) + else: + # update in-place with fixed ISSN-L + print("FIX: {}: {}".format(invalid.ident, fixed_issnl)) + invalid.issnl = fixed_issnl + api.update_container(eg.editgroup_id, invalid.ident, invalid) + + # intentionally not merging editgroup + print("Editgroup ident: {}".format(eg.editgroup_id)) + print("DONE") + +def main(): + api = authenticated_api( + API_ENDPOINT, + # token is an optional kwarg (can be empty string, None, etc) + token=os.environ.get("FATCAT_API_AUTH_TOKEN")) + + path = sys.argv[1] + reader = csv.DictReader(open(path), delimiter='\t') + run(api, reader) + +if __name__ == '__main__': + main() diff --git a/old/invalid_fatcat_issnl.tsv b/old/invalid_fatcat_issnl.tsv new file mode 100644 index 0000000..14a8dec --- /dev/null +++ b/old/invalid_fatcat_issnl.tsv @@ -0,0 +1,118 @@ +fixed_issnl issnl fatcat_ident name +1530-1311 1550-1311 2nklacmgkjdfjib7kqi3hdh76a International Symposium on Temporal Representation and Reasoning/TIME, Proceedings of the +2306-0441 2223-0441 rngvdeed65ffhmgfxti7s2z6by Journal of Local and Global Health Science +1641-6554 1641-6565 yviicehmubf4bcxo23q43sbkzu Kolposkopia +1526-7539 1526-7639 pnyefvclqfabjlnl4suox6pdte International Symposium on Modeling, Analysis and Simulation of Computer and Telecommunication Systems, Proceedings of the +0276-6574 0276-6547 hcrk2xeoknf7daxwurje2vg3n4 Computers in Cardiology Conference +0018-9413 0359-4237 ngthxcdwgzhovfgimtll6owdnm IEEE Transactions on Geoscience Electronics + 2630-4301 drgmggxvfjbjjkakrjegaujkgy Food Modelling Journal +1089-7771 1086-7771 sjjfcknh3zawndw6jdmvfrix7a IEEE transactions on information technology in biomedicine +1093-1139 1039-1139 nnrvd2qmhzbebk2hsopvtchodq Academic Physician and Scientist +0037-7996 0037-7796 klgctd3pbvbwpf6ynvd4wns53m The Social studies +1520-5363 1520-5263 v3zwf2ujd5flzlopo2h4ggmdhu Document Analysis and Recognition (ICDAR), Proceedings of the International Conference on +1992-4712 1992-4721 mmptdalzizepxi7uqfohnjj3uy NAFU Farmer +0066-7374 0066-7372 s5fjnmhtkbggtcn2itw4g34bxq Proceedings of the Aristotelian Society (Hardback) + 1234-5678 lkoo6d6lcvba7hhwswxc6x2sly Test s Publication +1088-7725 1080-7725 ikqqv7cnwfhpfkqy2lfugiz6ae AUTOTESTCON IEEE Systems Readiness Technology Conference, Proceedings of the + 0537-4737 kklzxgby4zfqvow7klnneyjj7e IEE Telecommunications Hot Topics Forum +1527-4160 1528-1145 q7j7ro4qfnbjdb2kkkcxatesna Journal of Psychiatric Practice +2468-4295 2648-4309 qtvq67fxfje4ro5mldpan35s34 Brill Research Perspectives in Art and Law + 1113-1114 qqkn2huytzfj5fw573bc5ihxyi A-to-Z Guide to Thermodynamics Heat and Mass Transfer and Fluids Engineering +1080-241X 1082-241X c3pwhzd5hvbqzfgqdrbxnyfn3i Proceedings Annual Simulation Symposium +1050-4729 1059-4729 b5exnobxjvd65flifftm6gy3f4 IEEE International Conference on Engineering of Complex Computer Systems, Proceedings of the +0295-1630 0296-1630 u7tretgoaffblbvtsxbtml6pbu Pleine Marge +1071-0485 1071-0483 6x6ijsddbva7xldh5ubqoxcf7a IEEE International Workshop on Future Trends of Distributed Computing Systems (FTDCS) +0093-2582 0093-2592 ajsbz23tbnghzmwleirmr7or3a The St. Croix review +1544-6301 1544-6307 tl5eyaqxbfcllmy7ip63up2sma Journal Of Cancer Integrative Medicine +2155-5745 2155-5754 b6s7uywtdfannklqblq6tghkte International Radar Symposium +0767-3701 3767-3701 wyh37ssfibcdtlncbi53m7ms4q Recherche et Applications en Marketing +2255-8691 2225-8691 bl6y7mpqgfdtxoqqbcq7bikwvu Applied Computer Systems +2561-3103 2561-3101 46uvtmw72zgpjhmbkvllrnwlau Journal of Contact Lens Research and Science +0939-7248 0939-7148 xma7s7w65zed3g4ojkmfz7r6ca European journal of pediatric surgery +1071-6084 1071-6048 dqthad6e3vfhxjf4px42u76dbe The Journal of technology studies +1528-8463 1538-8463 j4cxdxvaxrb2thxtvbda47gy2y Hospital law's Regan report + 2352-3951 pc6ftuth7ffvreuolz5rxbaye4 Nanoelectronics and Spintronics +2006-9820 2006-9802 wmugzbvj6rcrzg5hf5jtug6tja Journal of Toxicology and Environmental Health Sciences +0953-9182 0955-9182 mcpenq5ytrd6zpmuyc5crmx2fi Contemporary reviews in obstetrics and gynaecology +0259-188X 0459-188X iw7xtyujbrax7ojqqbgxjlrsem Indicator South Africa +0894-9115 0894-9112 wa3xjixz6ve75j2ecrk23sposq American Journal of Physical Medicine & Rehabilitation +0165-9367 0165-0937 32qxo574yjcbbbchqijyakggzm BABesch - Bulletin Antieke Beschaving +2586-1247 2968-9853 whexebkzebh5ncyb6hvjppkyqa International Journal of Sustainable Lighting +1811-7813 1511-7813 ipu26yclwnhw7ekbsve6wpb44a Point-Based Graphics (PBG), IEEE VGTC Symposium Proceedings +0394-1051 2532-4681 opjfte47erhb7dke5aeap5wbse Eupalino : cultura della città e della casa + 0538-9989 cjkaddjcfzhmveuuffmtycjxmm Professional Development Course on Railway Electrification Infrastructure and Systems +1081-7735 1085-7735 oam6q4knpjaq5enpfzra3jazfi Asian Test Symposium, Proceedings of the +1064-7546 1064-7564 su2chgwy2vf6zavx6ekbyv4sem Journal of Environmental Polymer Degradation +1817-4426 1817-4428 kwqvwiwyx5b5vbjxfetnpxt6ky The Retail and Marketing Review + 1432-3123 sa73qgbh25bgnd5q6o262u7inm china Frontiers +2141-2634 2142-2634 g2z2ld65nve2pdwjk7gr3qe25e Journal of Civil Engineering and Construction Technology + 9999-9998 nvya624wtbbgxl6bdsrqatslum Space Resolver 15 + 0006-0809 5gtixxmzm5aodhqpd4bqp3hqiq The Bridge +0547-3578 7964-0977 bonch7zygnadddv2fqoecmj4yy IEEE National Aerospace and Electronics Conference +0003-3839 0003-3829 yqozzqdrwvedvafwrokyx5puna Annales agronomiques +2213-9885 2213-0886 hdfn46db7bgw5e7lmmlx3iubxe Current Chemical Genomics and Translational Medicine +2378-5225 2333-5225 tgcsgbazhrfqzodbpdkjgtvv6i Pathophysiology of Cell Injury Journal +1761-7227 1761-7727 uynmrtrspngoxdjdihrwx3x6oi International orthodontics +1079-5545 1990-1993 goo5ob7jqzh7ro5pft7j3qdkaq Academy of Management Executive +1536-0040 1111-1111 7au72xtbafdwpdsz6d53elvk2a SIAM E-Journal on Applied Dynamical Systems +0163-514X 0613-514X jjup253sjjc2pkg3zz3vfz3nye The Journal of Prevention +1020-1491 1020-1497 gtn3v33zcrg5zndvyir7l65mna Image & Text : a Journal for Design +0972-8279 0972-8270 laktqf5ag5fpje2kfzzutnooyq Journal of maxillofacial and oral surgery + 0133-0189 kixvoietq5bhjeub36kqu6kwpa DCDS Supplements +1071-9458 1071-9658 xmwrlaklfbe53ayhlzrhinb5cu International Symposium on Software Reliability Engineering, Proceedings of the +0040-6325 0040-5325 qpbm4in22jevvfeht6mhawmllq The Thomist: A Speculative Quarterly Review +0039-4157 0039-4156 5xp3jodttzhmrp22xl5femz52q Study encounter +1122-5300 1125-5300 zeclu7ljprfkvauia5vljzvnfy Parolechiave + 1453-212X dhdssmbdbjfyfe53dpbfgcejti Research Journal of International Studies +1530-1834 1550-1834 d57htlky2nexlmy4wa36lva6wy Brazilian Symposium on Computer Graphics and Image Processing + 0686-3174 ajqi4yc43feitknsozbzgpjnre Y hoc Viet Nam +0015-8208 1521-3979 hoifudhajjgwfauzsgevzgogpa Fortschritte der Physik/Progress of Physics +2141-6613 2141-6113 xghlebvnzzco3j6eb6ywlwal3m International Journal of Water Resources and Environmental Engineering +0112-9120 0110-7903 6c6fql7levae5lyunnm6aiuzaq New Zealand monthly review + 0798-1881 xnblbncvezfuveglr5loa2rqym Construccion +0017-3703 0017-3701 5dfgot47anfwxk65i3biooh6we Greater amusements +0015-8577 0015-1287 xjwp4btqb5gojjcmdegglcyrli The Forumeer +0160-8371 1060-8371 jbubc4zldfegxfkstwvg2oh47i IEEE Photovoltaic Specialists Conference, Conference Record of the +2378-9530 2378-8953 4x4y2afasfg45iqoo47vphr6iq Veterinary Research and Case Reports +2162-4933 2162-4923 azachjfocbf6xciiz3m3ze564u Current Dermatology Reports +0882-5645 0882-5646 7fyviche6rbyzfsznpfrvvlyv4 Topics in Pain Management +0033-7196 0033-7916 hsp5k2eij5dwpe4tbqf2n7gkpa RWDSU record +1823-3902 1832-3902 sdw3bp3mk5b7vpkf22nj3dbttu Journal of tropical biology & conservation +1066-6192 1066-6172 jlzpb6temzauliox6i7ecma3mq Euromicro Workshop on Parallel and Distributed Processing, Proceedings of the +1818-9091 1819-9091 gom2wpnbtje6zfssyliobeup24 SA Horseman +2504-2289 8888-0000 eyngovpvzfen5hsd55udttyk2u Big Data and Cognitive Computing + 8164-9547 knfumdwvafcnjnwge24v3kaip4 IEEE International Conference on Communications (2006) +0369-8718 0368-8718 fatiuzqz7veebcr4n3t7wvn2n4 Proceedings of the Chemical Society +2379-6227 2397-6227 37z2akit2nbtbea5qcn5shreru Computational Psychiatry +0025-6978 0026-6978 vq6o3qyo3rezvercvuai2dciti Medical and biological illustration +2588-4174 4174-2588 lr5kb4tnq5bpjf3otub5umkdmy Afzalipour Journal of Clinical Research +2641-7928 1089-796X cff4m4os3bdznnc3fxbh2vv6ha International Conference on Parallel Architectures and Compilation Techniques, Proceedings of the +1931-7611 1931-7811 gmtif5qt6vddroc7uox4hxtch4 International journal of sexual health +0538-933X 0583-9364 xt6yxm3j7jdyzb6viwcltb6vsq International Reading Association. Proceedings of the annual convention + 7777-8888 3va7v7macvbalhxfujg5qfb7vi A Journal with Only 1 Volume +2157-3689 2157-3698 qegnzb72dzhf5jdunf3ge7jmji Limnology & Oceanography Fluids & Environment +1673-3487 1673-3508 z3jjoyzt45ecrh3c62afjmmxjm Frontiers of Physical in China +2571-6255 0013-0013 kwk3mhqddfddrd6hjqdnkg3f3y Fire +2141-2618 2141-2648 w5aoztaff5dg7harb4d2fyxzji Journal of Medical Laboratory and Diagnosis + 0804-3321 bqecated6jcwzoe56hmlxxydsq Utdanning + 0842-3446 wiidl3rotncxfpqoj46uegf5mq African Journal of Farm Child and Youth Development + 0539-9989 c7abz7liw5d53bskkcq7fcetvy Institution of Engineering and Technology International Conference on Medical Electrical Devices and Technology +1051-9173 1057-9173 auy44witavcmnejiss5h4ndlnq IEEE Symposium on Mass Storage Systems +0276-2234 0726-2234 6cbknzfrcfchhj73pmrlull2ky Oncology Times UK +0029-0564 1949-1986 s4poo6asmffuhlfw4lsneh6fq4 Nineteenth-Century Fiction + 9999-9999 zjdb7jqlxzcmlatzcjfjlrna6m Space Resolver 16 +0026-475X 0027-475X xmgc2fswtjaghmrocercgewwnu Minerva dietologica + 8164-2284 jkjfyzs3ijcnfbbfwxpewlwuaq IEEE International Electron Devices Meeting +1972-3792 2035-0167 nbo7hjouubbjba3kk5nn6oucmu Quaderni materialisti +0307-0565 0307-9565 mfgzm4p6zzgsxja4jjy7etl35q International journal of obesity +0738-6176 0723-6176 ynfkc2j7lzbpnchzpjic2orwlq The Psychotherapy patient +0963-0252 0965-0252 4s5mbkeut5g5tass6gz6qfivny Plasma sources science & technology +0144-3887 0143-3887 kdtbqfuuynd2fc64lprajnauvm Current Psychological Research +1470-3556 0741-9738 23er6bey4rgclaudewagavnf7m Advances in Mind-Body Medicine +0040-3261 0400-3261 arakq24urzgfhbz4abv6kz4mha Tennessee historical quarterly +0957-4484 0957-9207 k4v6ng56q5fb5p7yjyi7fsmv5a Nanotechnology +1087-9870 1089-9870 ubsj5pa5abbe7gqqdttkfibqle Thermal and Thermomechanical Phenomena in Electronic Systems (ITHERM), Intersociety Conference on +1812-6731 1812-6371 lhhrrjjxynd3lcadty56vqjk4u New Voices in Psychology +0163-0067 0613-0067 mlluqww5qnet7gqptazojr32bu Information world +1080-9775 1080-9175 yxjwuy4kj5hqvorfbmv4widyqa Biomedical Safety & Standards + 9999-9997 o6xy2oixgzag5o6oe26fdmmr4e Space Resolver diff --git a/old/parse_merge_metadata.py b/old/parse_merge_metadata.py new file mode 100755 index 0000000..b1d038b --- /dev/null +++ b/old/parse_merge_metadata.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python3 + +import sys, csv, json +import ftfy +import pycountry +from collections import Counter + +ISSNL_FILE = 'data/20190220.ISSN-to-ISSN-L.txt' + +ENTREZ_FILE = 'data/entrez-journals.csv' +ROAD_FILE = 'data/road-2018-01-24.tsv' +ROAD_DATE = '2018-01-24' +DOAJ_FILE = 'data/doaj_20190124.csv' +DOAJ_DATE = '2019-01-24' +CROSSREF_FILE = 'data/doi_titles_file_2019-01-24.csv' +SHERPA_ROMEO_JOURNAL_FILE = 'data/romeo-journals.csv' +SHERPA_ROMEO_POLICY_FILE = 'data/romeo-policies.csv' +NORWEGIAN_FILE = 'data/2018-03-02 Norwegian Register for Scientific Journals and Series.csv' +NORWEGIAN_DATE = '2018-03-02' +LOCKSS_FILE = 'data/kbart_LOCKSS.txt' +CLOCKSS_FILE = 'data/kbart_CLOCKSS.txt' +PORTICO_FILE = 'data/Portico_Holding_KBart.txt' +JSTOR_FILE = 'data/jstor_all-archive-titles.txt' +SIM_FILE = 'data/MASTER TITLE_METADATA_LIST_20171019.converted.csv' +IA_CRAWL_FILE = 'data/journal_homepage_results.partial.tsv' + +# NOTE: this is a partial list, focusing on non-publisher hosted platforms and +# software frameworks +PLATFORM_MAP = { + 'OJS': 'ojs', + 'BMC': 'bmc', + 'SciELO Brazil': 'scielo', + 'SciELO Argentina': 'scielo', + 'SciELO': 'scielo', + 'SciELO Mexico': 'scielo', + 'SciELO Spain': 'scielo', + 'SciELO Portugal': 'scielo', + 'WordPress': 'wordpress', + 'Sciendo': 'sciendo', + 'Drupal': 'drupal', + 'revues.org': 'openedition', +} + +MIMETYPE_MAP = { + 'PDF': 'application/pdf', + 'HTML': 'text/html', + 'XML': 'application/xml', +} + +def unquote(s): + if s.startswith('"'): + s = s[1:] + if s.endswith('"'): + s = s[:-1] + if s.endswith('.'): + s = s[:-1] + return s.strip() + +def parse_lang(s): + if not s or s in ('Not applicable', 'Multiple languages', 'Unknown'): + return None + try: + if len(s) == 2: + lang = pycountry.languages.get(alpha2=s.lower()) + elif len(s) == 3: + lang = pycountry.languages.get(alpha3=s.lower()) + else: + lang = pycountry.languages.get(name=s) + return lang.alpha2.lower() + except KeyError: + return None + except AttributeError: + return None + +def parse_country(s): + if not s or s in ('Unknown'): + return None + try: + if len(s) == 2: + country = pycountry.countries.get(alpha2=s.lower()) + else: + country = pycountry.countries.get(name=s) + except KeyError: + return None + return country.alpha2.lower() + +def gaps_to_spans(first, last, gaps): + if not gaps: + return [[first, last]] + if not (last >= first and max(gaps) < last and min(gaps) > first): + # mangled + print("mangled years: {}".format((first, last, gaps))) + return [] + full = list(range(first, last+1)) + for missing in gaps: + full.remove(missing) + spans = [] + low = None + last = None + for year in full: + if not low: + low = year + last = year + continue + if year != last+1: + spans.append([low, last]) + low = year + last = year + last = year + if low: + spans.append([low, last]) + return spans + +def test_gaps(): + assert gaps_to_spans(1900, 1900, None) == \ + [[1900, 1900]] + assert gaps_to_spans(1900, 1903, None) == \ + [[1900, 1903]] + assert gaps_to_spans(1900, 1902, [1901]) == \ + [[1900, 1900], [1902, 1902]] + assert gaps_to_spans(1950, 1970, [1955, 1956, 1965]) == \ + [[1950, 1954], [1957, 1964], [1966, 1970]] + +def merge_spans(old, new): + if not new: + return old + if not old: + old = [] + old.extend(new) + years = set() + for span in old: + for y in range(span[0], span[1]+1): + years.add(y) + if not years: + return [] + spans = [] + start = None + last = None + todo = False + for y in sorted(list(years)): + if start == None: + # very first + start = y + last = y + todo = True + continue + if y == last + 1: + # span continues + last = y + todo = True + continue + # a gap just happened! + spans.append([start, last]) + start = y + last = y + todo = True + if todo: + spans.append([start, last]) + return spans + +def test_merge_spans(): + assert merge_spans([[5, 10]], [[10, 20]]) == \ + [[5, 20]] + assert merge_spans([[5, 9]], [[10, 20]]) == \ + [[5, 20]] + assert merge_spans([[5, 11]], [[10, 20]]) == \ + [[5, 20]] + assert merge_spans([], []) == \ + [] + assert merge_spans([[9, 11]], []) == \ + [[9,11]] + assert merge_spans([[2000, 2000]], [[1450, 1900]]) == \ + [[1450, 1900], [2000, 2000]] + +class Munger(): + """ + Top-level fields we'd like to fill in if possible: + + issnp: string + issne: string + first_year: year (integer) + last_year: if publishing has stopped + languages: array of ISO codes; first is the "primary" language + country: ISO shortcode of country published from + urls: homepage links + abbrev: string + default_license: slug + original_name: native name (if name is translated) + platform: hosting platform: OJS, wordpress, scielo, etc + mimetypes: array of strings (eg, 'application/pdf', 'text/html') + aliases: array of "also known as" + + Lower priority (TODO/later): + coden: string + oclc_id: string (lookup?) + lccn_id: string (lookup?) + dblb_id: string + region: TODO: continent/world-region + discipline: TODO: highest-level subject; "life science", "humanities", etc + field: TODO: narrower description of field + subjects: TODO? + + TODO: so many missing ISSN/ISSN-L + TODO: abbrev + """ + + def __init__(self): + self.data = dict() + with open(ISSNL_FILE, 'r') as f: + self.read_issn_map_file(f) + + def run(self, out_path): + self.load_doaj(DOAJ_FILE) + self.load_norwegian(NORWEGIAN_FILE) + self.load_crossref(CROSSREF_FILE) + self.load_sherpa_romeo(SHERPA_ROMEO_JOURNAL_FILE, SHERPA_ROMEO_POLICY_FILE) + self.load_road(ROAD_FILE) + self.load_kbart('lockss', LOCKSS_FILE) + self.load_kbart('clockss', CLOCKSS_FILE) + self.load_kbart('portico', PORTICO_FILE) + self.load_kbart('jstor', JSTOR_FILE) + self.load_entrez(ENTREZ_FILE) + self.load_sim(SIM_FILE) + self.load_homepage_crawl(IA_CRAWL_FILE) + self.summarize() + self.dump(out_path) + print("Done!") + + def dump(self, out_path): + print("#### Dumping to {}".format(out_path)) + with open(out_path, 'w') as out: + for issnl in self.data: + out.write(json.dumps(self.data[issnl]) + "\n") + + def summarize(self): + print("##### Loaded {} unique entries".format(len(self.data))) + + def read_issn_map_file(self, issn_map_file): + print("##### Loading ISSN map file...") + self._issn_issnl_map = dict() + for line in issn_map_file: + if line.startswith("ISSN") or len(line) == 0: + continue + (issn, issnl) = line.split()[0:2] + self._issn_issnl_map[issn] = issnl + # double mapping makes lookups easy + self._issn_issnl_map[issnl] = issnl + print("Got {} ISSN-L mappings.".format(len(self._issn_issnl_map))) + + def issn2issnl(self, issn): + if issn is None: + return None + return self._issn_issnl_map.get(issn) + + def add_issn(self, raw_issn=None, issne=None, issnp=None, name=None, publisher=None): + # do ISSN => ISSN-L mappings for any raw ISSNs + issnl = None + if not (raw_issn or issne or issnp): + return None, 'no-issn' + for lookup in (issnp, issne, raw_issn): + if not lookup: + continue + lookup = lookup.strip().upper() + #if not (len(lookup) == 9 and lookup[4] == '-'): + # print(lookup) + # print(len(lookup)) + # print(lookup[4]) + # return None, 'invalid-issn' + #assert len(lookup) == 9 and lookup[4] == '-' + issnl = self.issn2issnl(lookup) + if issnl: + break + if not issnl: + #print((raw_issn, issne, issnp)) + # UGH. + issnl = issne or issnp or raw_issn + if not issnl: + return None, 'no-issnl' + issnl = issnl.strip().upper() + assert len(issnl) == 9 and issnl[4] == '-' + status = 'found-munge' + else: + status = 'found' + # lookup ISSN-Ls in data (or create one) + if not issnl in self.data: + status = 'created' + self.data[issnl] = dict(issnl=issnl) + d = self.data[issnl] + # if name/publisher not set, do so + if name and not 'name' in d: + name = unquote(ftfy.fix_text(name)) + if name: + self.data[issnl]['name'] = name + if publisher and not 'publisher' in d: + publisher = unquote(ftfy.fix_text(publisher)) + if publisher: + self.data[issnl]['publisher'] = publisher + if issne and not 'issne' in d: + self.data[issnl]['issne'] = issne + if issnp and not 'issnp' in d: + self.data[issnl]['issnp'] = issnp + # always return ISSN-L + return issnl, status + + def add_lang(self, issnl, lang): + if not (lang and issnl): + return + lang = parse_lang(lang) + if not lang: + return + if 'languages' not in self.data[issnl]: + self.data[issnl]['languages'] = [lang] + elif lang not in self.data[issnl]['languages']: + self.data[issnl]['languages'].append(lang) + + def add_url(self, issnl, url): + if not (url and issnl) or 'mailto:' in url.lower() or url in ('http://n/a', 'http://N/A'): + return + if url.startswith('www.'): + url = "http://" + url + url.replace('Http://', 'http://') + if 'urls' not in self.data[issnl]: + self.data[issnl]['urls'] = [url] + elif url not in self.data[issnl]['urls']: + self.data[issnl]['urls'].append(url) + + def add_country(self, issnl, country): + if not (country and issnl): + return + country = parse_country(country) + if not country: + return + if 'country' not in self.data[issnl]: + self.data[issnl]['country'] = country + + def add_mimetype(self, issnl, val): + if not (val and issnl): + return + mimetype = None + if '/' in val: + mimetype = val + else: + mimetype = MIMETYPE_MAP.get(val) + if not mimetype: + return + if 'mimetypes' not in self.data[issnl]: + self.data[issnl]['mimestypes'] = [mimetype] + elif mimetype not in self.data[issnl]['mimestypes']: + self.data[issnl]['mimestypes'].append(mimetype) + + def load_entrez(self, path): + print("##### Loading Entrez...") + # JrId,JournalTitle,MedAbbr,"ISSN (Print)","ISSN (Online)",IsoAbbr,NlmId + reader = csv.DictReader(open(path)) + counts = Counter() + for row in reader: + if not (row.get('ISSN (Online)') or row.get('ISSN (Print)')): + counts['skipped'] += 1 + continue + issnl, status = self.add_issn( + issne=row.get('ISSN (Online)'), + issnp=row.get('ISSN (Print)'), + name=row['JournalTitle'], + ) + if row['IsoAbbr'] and not 'abbrev' in self.data[issnl]: + self.data[issnl]['abbrev'] = row['IsoAbbr'].strip() + counts[status] += 1 + print(counts) + + def load_road(self, path): + print("##### Loading ROAD...") + reader = csv.DictReader(open(path), delimiter='\t', + fieldnames=("ISSN", "ISSN-L", "Short Title", "Title", "Publisher", "URL1", "URL2", "Region", "Lang1", "Lang2") + ) + counts = Counter() + for row in reader: + issnl, status = self.add_issn( + raw_issn=row['ISSN-L'], + name=row['Short Title'], + publisher=row['Publisher'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + if row['URL1']: + self.add_url(issnl, row['URL1']) + if row['URL2']: + self.add_url(issnl, row['URL2']) + if row['Lang1']: + self.add_lang(issnl, row['Lang1']) + if row['Lang2']: + self.add_lang(issnl, row['Lang2']) + # TODO: region mapping: "Europe and North America" + # TODO: lang mapping: already alpha-3 + self.data[issnl]['road'] = dict(as_of=ROAD_DATE) + print(counts) + + def load_doaj(self, path): + print("##### Loading DOAJ...") + #Journal title,Journal URL,Alternative title,Journal ISSN (print version),Journal EISSN (online version),Publisher,Society or institution,"Platform, host or aggregator",Country of publisher,Journal article processing charges (APCs),APC information URL,APC amount,Currency,Journal article submission fee,Submission fee URL,Submission fee amount,Submission fee currency,Number of articles publish in the last calendar year,Number of articles information URL,Journal waiver policy (for developing country authors etc),Waiver policy information URL,Digital archiving policy or program(s),Archiving: national library,Archiving: other,Archiving infomation URL,Journal full-text crawl permission,Permanent article identifiers,Journal provides download statistics,Download statistics information URL,First calendar year journal provided online Open Access content,Full text formats,Keywords,Full text language,URL for the Editorial Board page,Review process,Review process information URL,URL for journal's aims & scope,URL for journal's instructions for authors,Journal plagiarism screening policy,Plagiarism information URL,Average number of weeks between submission and publication,URL for journal's Open Access statement,Machine-readable CC licensing information embedded or displayed in articles,URL to an example page with embedded licensing information,Journal license,License attributes,URL for license terms,Does this journal allow unrestricted reuse in compliance with BOAI?,Deposit policy directory,Author holds copyright without restrictions,Copyright information URL,Author holds publishing rights without restrictions,Publishing rights information URL,DOAJ Seal,Tick: Accepted after March 2014,Added on Date,Subjects + reader = csv.DictReader(open(path)) + counts = Counter() + for row in reader: + issnl, status = self.add_issn( + issnp=row['Journal ISSN (print version)'], + issne=row['Journal EISSN (online version)'], + name=row['Journal title'], + publisher=row['Publisher'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + doaj = dict(as_of=DOAJ_DATE) + # TODO: work_level: bool (are work-level publications deposited with DOAJ?) + + if row['Digital archiving policy or program(s)']: + doaj['archive'] = [a.strip() for a in row['Digital archiving policy or program(s)'].split(',') if a.strip()] + elif row['Archiving: national library']: + doaj['archive'] = ['national-library'] + + crawl_permission = row['Journal full-text crawl permission'] + if crawl_permission: + doaj['crawl-permission'] = dict(Yes=True, No=False)[crawl_permission] + # TODO: Permanent article identifiers + default_license = row['Journal license'] + if default_license and default_license.startswith('CC'): + self.data[issnl]['default_license'] = default_license.replace('CC ', 'CC-').strip() + + self.add_mimetype(issnl, row['Full text formats']) + platform = PLATFORM_MAP.get(row['Platform, host or aggregator']) + if platform: + self.data[issnl]['platform'] = platform + if row['DOAJ Seal']: + doaj['seal'] = {"no": False, "yes": True}[row['DOAJ Seal'].lower()] + if row['Country of publisher']: + self.add_country(issnl, row['Country of publisher']) + if row['Full text language']: + self.add_lang(issnl, row['Full text language']) + if row['Journal URL']: + self.add_url(issnl, row['Journal URL']) + # TODO: Subjects + self.data[issnl]['doaj'] = doaj + print(counts) + + def load_sherpa_romeo(self, journal_path, policy_path): + # first load policies + print("##### Loading SHERPA/ROMEO policies...") + #RoMEO Record ID,Publisher,Policy Heading,Country,RoMEO colour,Published Permission,Published Restrictions,Published Max embargo,Accepted Prmission,Accepted Restrictions,Accepted Max embargo,Submitted Permission,Submitted Restrictions,Submitted Max embargo,Open Access Publishing,Record Status,Updated + policies = dict() + fixed_policy_file = ftfy.fix_file(open(policy_path, 'rb')) + policy_reader = csv.DictReader(fixed_policy_file) + for row in policy_reader: + policies[row['RoMEO Record ID']] = row + print("##### Loading SHERPA/ROMEO journal metadata...") + #Journal Title,ISSN,ESSN,URL,RoMEO Record ID,Updated + # super mangled :( + raw_file = open(journal_path, 'rb').read().decode(errors='replace') + fixed_file = ftfy.fix_text(raw_file) + reader = csv.DictReader(fixed_file.split('\n')) + counts = Counter() + for row in reader: + #row['Journal Title'] = row.pop('\ufeffJournal Title') + row.update(policies[row['RoMEO Record ID']]) + issnl, status = self.add_issn( + issnp=row['ISSN'], + issne=row['ESSN'], + name=row['Journal Title'], + publisher=row['Publisher'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + sherpa_romeo = dict() + if row['RoMEO colour']: + sherpa_romeo['color'] = row['RoMEO colour'] + # row['Open Access Publishing'] + if row['Country']: + self.add_country(issnl, row['Country']) + self.data[issnl]['sherpa_romeo'] = sherpa_romeo + print(counts) + + def load_norwegian(self, path): + print("##### Loading Norwegian Registry...") + #pandas.read_csv(NORWEGIAN_FILE, sep=';', encoding="ISO-8859-1") + #NSD tidsskrift_id;Original title;International title;Present Level (2018);Print ISSN;Online ISSN;Open Access;NPI Scientific Field;NPI Academic Discipline;URL;Publishing Company;Publisher;Country of publication;Language;Level 2019;Level 2018;Level 2017;Level 2016;Level 2015;Level 2014;Level 2013;Level 2012;Level 2011;Level 2010;Level 2009;Level 2008;Level 2007;Level 2006;Level 2005;Level 2004;itar_id + reader = csv.DictReader(open(path, encoding="ISO-8859-1"), delimiter=";") + counts = Counter() + for row in reader: + issnp = row['Print ISSN'] + issne = row['Online ISSN'] + if issne and len(issne.strip()) != 9: + issne = None + if issnp and len(issnp.strip()) != 9: + issnp = None + if not (issnp or issne): + counts['no-issn'] += 1 + continue + issnl, status = self.add_issn( + issnp=issnp, + issne=issne, + name=row['International title'], + publisher=row['Publisher'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + norwegian = dict(as_of=NORWEGIAN_DATE) + norwegian['level'] = int(row['Present Level (2018)']) + norwegian['id'] = int(row['NSD tidsskrift_id']) + + if row['Original title'] != row['International title'] and not 'original_name' in d: + self.data[issnl]['original_name'] = row['Original title'] + if row['Country of publication']: + self.add_country(issnl, row['Country of publication']) + if row['Language']: + self.add_lang(issnl, row['Language']) + if row['URL']: + self.add_url(issnl, row['URL']) + self.data[issnl]['norwegian'] = norwegian + print(counts) + + def load_kbart(self, name, path): + print("##### Loading KBART file for {}...".format(name)) + #publication_title print_identifier online_identifier date_first_issue_online num_first_vol_online num_first_issue_online date_last_issue_online num_last_vol_online num_last_issue_online title_url first_author title_id embargo_info coverage_depth coverage_notes publisher_name + raw_file = open(path, 'rb').read().decode(errors='replace') + fixed_file = ftfy.fix_text(raw_file) + reader = csv.DictReader(fixed_file.split('\n'), delimiter='\t') + counts = Counter() + for row in reader: + if not row['print_identifier'] and not row['online_identifier']: + counts['no-issn'] += 1 + continue + issnl, status = self.add_issn( + issnp=row['print_identifier'], + issne=row['online_identifier'], + name=row['publication_title'], + publisher=row['publisher_name'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + if not 'kbart' in d: + self.data[issnl]['kbart'] = dict() + d = self.data[issnl] + if not name in d['kbart']: + self.data[issnl]['kbart'][name] = dict() + old_spans = self.data[issnl]['kbart'].get(name, dict()).get('year_spans', []) + kbart = dict() + if row['date_first_issue_online'] and row['date_last_issue_online']: + start = int(row['date_first_issue_online'][:4]) + end = int(row['date_last_issue_online'][:4]) + if not start <= end: + print("{}: {} not before {}! er, mangling".format( + issnl, + row['date_first_issue_online'], + row['date_last_issue_online'])) + new_spans = [[end, start]] + else: + new_spans = [[start, end]] + self.data[issnl]['kbart'][name]['year_spans'] = merge_spans(old_spans, new_spans) + print(counts) + + def load_crossref(self, path): + print("##### Loading Crossref...") + #"JournalTitle","JournalID","Publisher","pissn","eissn","additionalIssns","doi","(year1)[volume1]issue1,issue2,issue3(year2)[volume2]issue4,issues5" + reader = csv.DictReader(open(path)) + counts = Counter() + for row in reader: + if row['pissn'] and len(row['pissn']) == 8: + row['pissn'] = row['pissn'][:4] + '-' + row['pissn'][4:] + if row['eissn'] and len(row['eissn']) == 8: + row['eissn'] = row['eissn'][:4] + '-' + row['eissn'][4:] + if not (row['pissn'] or row['eissn']): + counts['no-issn'] += 1 + continue + issnl, status = self.add_issn( + issnp=row['pissn'], + issne=row['eissn'], + name=row['JournalTitle'], + publisher=row['Publisher'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + crossref = dict() + if row['doi']: + crossref['doi'] = row['doi'] + crossref['any'] = True + self.data[issnl]['crossref'] = crossref + print(counts) + + def load_sim(self, path): + print("##### Loading SIM Metadata...") + #NA Pub Cat ID,Title,Publisher,ISSN,Impact Rank,Total Cities,Journal Impact Factor,Eigenfact or Score,First Volume,Last Volume,NA Gaps,"Scholarly / Peer-\n Reviewed","Peer-\n Reviewed",Pub Type,Pub Language,Subjects + reader = csv.DictReader(open(path)) + counts = Counter() + for row in reader: + if not row['ISSN'] or row['ISSN'] == "NULL": + counts['no-issn'] += 1 + continue + issnl, status = self.add_issn( + raw_issn=row['ISSN'][:9], + name=row['Title'], + publisher=row['Publisher'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + sim = dict() + sim['id'] = row['NA Pub Cat ID'] + first_year = row['First Volume'] + if first_year: + first_year = int(first_year) + sim['first_year'] = int(row['First Volume']) + else: + first_year = None + last_year = row['Last Volume'] + if last_year: + last_year = int(last_year) + sim['last_year'] = last_year + else: + last_year = None + gaps = [int(g) for g in row['NA Gaps'].split(';') if g.strip()] + if gaps: + sim['gaps'] = gaps + if first_year and last_year: + sim['year_spans'] = gaps_to_spans(first_year, last_year, gaps) + if row['Pub Language']: + self.add_lang(issnl, row['Pub Language']) + # TODO: 'Pub Type' + all_keys = list(sim.keys()) + for k in all_keys: + if not sim[k]: + sim.pop(k) + self.data[issnl]['sim'] = sim + print(counts) + + def load_homepage_crawl(self, path): + print("##### Loading IA Homepage Crawl Results...") + reader = csv.DictReader(open(path), delimiter='\t', + fieldnames=("ISSN", "first_url", "first_status", "last_status", "last_url") + ) + counts = Counter() + for row in reader: + issnl, status = self.add_issn( + raw_issn=row['ISSN'], + ) + counts[status] += 1 + if not issnl: + continue + d = self.data[issnl] + ia = d.get('ia', dict()) + ia['homepage_status'] = int(row['last_status']) + if ia['homepage_status'] == 200: + ia['homepage_url'] = row['last_url'] + else: + ia['homepage_url'] = row['first_url'] + self.data[issnl]['ia'] = ia + print(counts) + +if __name__=='__main__': + if len(sys.argv) != 2 or sys.argv[1].startswith('-'): + print("pass me path for an output JSON lines file") + sys.exit(-1) + munger = Munger() + munger.run(sys.argv[1]) + |