aboutsummaryrefslogtreecommitdiffstats
path: root/extra/journal_metadata/chocula.py
diff options
context:
space:
mode:
Diffstat (limited to 'extra/journal_metadata/chocula.py')
-rwxr-xr-xextra/journal_metadata/chocula.py228
1 files changed, 171 insertions, 57 deletions
diff --git a/extra/journal_metadata/chocula.py b/extra/journal_metadata/chocula.py
index 9ec4f1a8..1d50e850 100755
--- a/extra/journal_metadata/chocula.py
+++ b/extra/journal_metadata/chocula.py
@@ -23,14 +23,15 @@ Commands:
index_szczepanski
index_ezb
- fatcat_load
- fatcat_stats
+ load_fatcat
+ load_fatcat_stats
export_urls
update_url_status
Future commands:
+ fatcat_edits
index_jurn
index_wikidata
index_datacite
@@ -38,9 +39,22 @@ Future commands:
preserve_sim
TODO:
+- KBART imports (with JSON, so only a single row per slug)
+- check that all fields actually getting imported reasonably
+- imprint/publisher distinction (publisher is big group)
+- summary table should be superset of fatcat table
+- add timestamp columns to enable updates?
+- "GOLD" importer (for scopus/WoS)
+- fatcat export (filters for changes to make, writes out as JSON)
+- homepage crawl/status script
+- update_url_status (needs re-write)
- index -> directory
- log out index issues (duplicate ISSN-L, etc)
-
+- validate against GOLD OA list
+- decide what to do with JURN... match? create missing fatcat?
+x load_fatcat
+x fatcat_stats (also add timestamp column)
+x export_url
"""
import sys, csv, json
@@ -76,7 +90,9 @@ JSTOR_FILE = 'data/jstor_all-archive-titles.txt'
SIM_FILE = 'data/MASTER TITLE_METADATA_LIST_20171019.converted.csv'
IA_CRAWL_FILE = 'data/journal_homepage_results.partial.tsv'
SZCZEPANSKI_FILE = 'data/Jan-Szczepanski-Open-Access-Journals-2018_0.fixed.json'
-EZB_FILE = 'data/ezb_sample.json'
+EZB_FILE = 'data/ezb_metadata.json'
+FATCAT_CONTAINER_FILE = 'data/container_export.json'
+FATCAT_STATS_FILE = 'data/container_stats.json'
################### Utilities
@@ -249,36 +265,6 @@ def test_merge_spans():
################### Main Class
class ChoculaDatabase():
- """
- Top-level fields we'd like to fill in if possible:
-
- issnp: string
- issne: string
- first_year: year (integer)
- last_year: if publishing has stopped
- languages: array of ISO codes; first is the "primary" language
- country: ISO shortcode of country published from
- urls: homepage links
- abbrev: string
- default_license: slug
- original_name: native name (if name is translated)
- platform: hosting platform: OJS, wordpress, scielo, etc
- mimetypes: array of strings (eg, 'application/pdf', 'text/html')
- aliases: array of "also known as"
-
- Lower priority (TODO/later):
- coden: string
- oclc_id: string (lookup?)
- lccn_id: string (lookup?)
- dblb_id: string
- region: TODO: continent/world-region
- discipline: TODO: highest-level subject; "life science", "humanities", etc
- field: TODO: narrower description of field
- subjects: TODO?
-
- TODO: so many missing ISSN/ISSN-L
- TODO: abbrev
- """
def __init__(self, db_file):
self._issn_issnl_map = dict()
@@ -715,7 +701,7 @@ class ChoculaDatabase():
if row['additionalIssns'] and len(row['additionalIssns']) == 8:
row['additionalIssns'] = row['additionalIssns'][:4] + '-' + row['additionalIssns'][4:]
if not (row['pissn'] or row['eissn'] or row['additionalIssns']):
- print(row)
+ #print(row)
counts['no-issn'] += 1
continue
extra = dict()
@@ -787,7 +773,8 @@ class ChoculaDatabase():
self.db.commit()
print(counts)
- def load_homepage_crawl(self, path):
+ def update_url_status(self, args):
+ path = args.input_file or IA_CRAWL_FILE
print("##### Loading IA Homepage Crawl Results...")
reader = csv.DictReader(open(path), delimiter='\t',
fieldnames=("ISSN", "first_url", "first_status", "last_status", "last_url")
@@ -795,36 +782,166 @@ class ChoculaDatabase():
counts = Counter()
self.c = self.db.cursor()
for row in reader:
- issnl, status = self.add_issn(
- raw_issn=row['ISSN'],
- )
- counts[status] += 1
- if not issnl:
+ counts['total'] += 1
+ url = row['first_url']
+ assert(url)
+ self.c.execute("UPDATE homepage SET status_code=?, terminal_url=?, terminal_status_code=? WHERE url=?",
+ (row['first_status'], row['last_url'], row['last_status'], url))
+ counts['updated'] += 1
+ self.c.close()
+ self.db.commit()
+ print(counts)
+
+ def load_fatcat(self, args):
+ path = args.input_file or FATCAT_CONTAINER_FILE
+ print("##### Loading Fatcat Container Entities...")
+ # JSON
+ json_file = open(path, 'r')
+ counts = Counter()
+ self.c = self.db.cursor()
+ for row in json_file:
+ if not row:
continue
- d = self.data[issnl]
- ia = d.get('ia', dict())
- ia['homepage_status'] = int(row['last_status'])
- if ia['homepage_status'] == 200:
- ia['homepage_url'] = row['last_url']
+ row = json.loads(row)
+ if row['state'] != 'active':
+ continue
+ counts['total'] += 1
+ extra = row.get('extra', dict())
+ issne = extra.get('issne')
+ issnp = extra.get('issnp')
+ country = extra.get('country')
+ languages = extra.get('languages', [])
+ lang = None
+ if languages:
+ lang = languages[0]
+ try:
+ self.c.execute("INSERT OR REPLACE INTO fatcat_container (issnl, ident, revision, issne, issnp, wikidata_qid, name, container_type, publisher, country, lang) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+ (row['issnl'], row['ident'], row['revision'], issne, issnp,
+ row.get('wikidata_qid'), row['name'],
+ row.get('container_type'), extra.get('publisher'), country,
+ lang))
+ except sqlite3.IntegrityError as ie:
+ if str(ie).startswith("UNIQUE"):
+ return None, "duplicate-issnl"
+ raise ie
+ counts['inserted'] += 1
+ if row.get('issnl'):
+ urls = extra.get('urls', [])
+ for url in urls:
+ self.add_url(row['issnl'], url)
+ self.c.close()
+ self.db.commit()
+ print(counts)
+
+ def load_fatcat_stats(self, args):
+ path = args.input_file or FATCAT_STATS_FILE
+ print("##### Loading Fatcat Container Stats...")
+ # JSON
+ json_file = open(path, 'r')
+ counts = Counter()
+ self.c = self.db.cursor()
+ for row in json_file:
+ if not row:
+ continue
+ row = json.loads(row)
+ total = int(row['total'])
+ if total > 0:
+ ia_frac = float(row['in_web'])/total
+ preserved_frac = float(row['is_preserved'])/total
else:
- ia['homepage_url'] = row['first_url']
- self.data[issnl]['ia'] = ia
+ ia_frac = None
+ preserved_frac = None
+ self.c.execute("UPDATE fatcat_container SET release_count = ?, ia_count = ?, ia_frac = ?, preserved_count = ?, preserved_frac = ? WHERE issnl = ?",
+ (total, row['in_web'], ia_frac, row['is_preserved'], preserved_frac, row['issnl']))
+ counts['updated'] += 1
+ self.c.close()
+ self.db.commit()
+ print(counts)
+
+ def export_urls(self, args):
+ self.c = self.db.cursor()
+ self.db.row_factory = sqlite3.Row
+ cur = self.db.execute("SELECT issnl, url FROM homepage;")
+ for hrow in cur:
+ assert(hrow['url'])
+ assert(len(hrow['url'].split()) == 1)
+ print('\t'.join((hrow['issnl'], hrow['url'])))
+
+ def summarize(self, args):
+ print("##### Summarizing Everything...")
+ counts = Counter()
+ self.c = self.db.cursor()
+ self.db.row_factory = sqlite3.Row
+ index_issnls = list(self.c.execute('SELECT DISTINCT issnl FROM journal_index'))
+ fatcat_issnls = list(self.c.execute('SELECT DISTINCT issnl FROM fatcat_container'))
+ all_issnls = set([i[0] for i in index_issnls + fatcat_issnls])
+ print("{} total ISSN-Ls".format(len(all_issnls)))
+ for issnl in list(all_issnls):
+ #print(issnl)
+ counts['total'] += 1
+
+ out = dict()
+
+ fatcat_row = list(self.db.execute("SELECT * FROM fatcat_container WHERE issnl = ?;", [issnl]))
+ if fatcat_row:
+ frow = fatcat_row[0]
+ out['fatcat_ident'] = frow['ident']
+ for k in ('name', 'publisher', 'issne', 'issnp', 'lang', 'country'):
+ if not out.get(k) and frow[k]:
+ out[k] = frow[k]
+ any_preservation = bool(frow['preserved_count'])
+ any_ia = bool(frow['ia_count'])
+
+ cur = self.db.execute("SELECT * FROM journal_index WHERE issnl = ?;", [issnl])
+ for irow in cur:
+ if irow['slug'] in ('crossref',):
+ out['has_dois'] = True
+ if irow['slug'] in ('doaj','road','szczepanski'):
+ out['is_oa'] = True
+ # TODO: or if sz color is green
+ # TODO: define longtail, based on publisher_type?
+ for k in ('name',):
+ if not out.get(k) and irow[k]:
+ out[k] = irow[k]
+ if irow['extra']:
+ extra = json.loads(irow['extra'])
+ for k in ('country', 'lang', 'issne', 'issnp', 'publisher'):
+ if not out.get(k) and extra.get(k):
+ out[k] = extra[k]
+
+ cur = self.db.execute("SELECT * FROM homepage WHERE issnl = ?;", [issnl])
+ for hrow in cur:
+ if hrow['terminal_status_code'] == 200:
+ out['any_live_homepage'] = True
+
+ self.c.execute("INSERT OR REPLACE INTO journal_summary (issnl, issne, issnp, fatcat_ident, name, publisher, country, lang, is_oa, is_longtail, has_dois, any_live_homepage, any_preservation, any_ia) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+ (issnl, out.get('issne'), out.get('issnp'),
+ out.get('fatcat_ident'), out.get('name'),
+ out.get('publisher'), out.get('country'), out.get('lang'),
+ out.get('is_oa'), out.get('is_longtail'), out.get('has_dois', False),
+ out.get('any_live_homepage', False), out.get('any_preservation', False),
+ out.get('any_ia')))
self.c.close()
self.db.commit()
print(counts)
def everything(self, args):
+ self.init_db(args)
self.index_doaj(args)
self.index_norwegian(args)
self.index_crossref(args)
self.index_sherpa_romeo(args)
self.index_road(args)
self.index_entrez(args)
+ self.index_ezb(args)
+ self.load_fatcat(args)
+ self.load_fatcat_stats(args)
+ self.update_url_status(args)
#self.load_kbart('lockss', LOCKSS_FILE)
#self.load_kbart('clockss', CLOCKSS_FILE)
#self.load_kbart('portico', PORTICO_FILE)
#self.load_kbart('jstor', JSTOR_FILE)
- self.index_sim(args)
+ #self.index_sim(args)
#self.load_homepage_crawl(IA_CRAWL_FILE)
self.summarize(args)
print("### Done with everything!")
@@ -841,9 +958,6 @@ class ChoculaDatabase():
self.db.executescript(fschema.read())
print("Done!")
- def summarize(self, args):
- pass
-
def main():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
@@ -871,11 +985,11 @@ def main():
sub = subparsers.add_parser('index_{}'.format(ind))
sub.set_defaults(func='index_{}'.format(ind))
- sub = subparsers.add_parser('fatcat_load')
- sub.set_defaults(func='fatcat_load')
+ sub = subparsers.add_parser('load_fatcat')
+ sub.set_defaults(func='load_fatcat')
- sub = subparsers.add_parser('fatcat_stats')
- sub.set_defaults(func='fatcat_stats')
+ sub = subparsers.add_parser('load_fatcat_stats')
+ sub.set_defaults(func='load_fatcat_stats')
sub = subparsers.add_parser('export_urls')
sub.set_defaults(func='export_urls')
@@ -889,7 +1003,7 @@ def main():
sys.exit(-1)
cdb = ChoculaDatabase(args.db_file)
- if args.func.startswith('index_'):
+ if args.func.startswith('index_') or args.func in ('everything',):
cdb.read_issn_map_file(ISSNL_FILE)
func = getattr(cdb, args.func)
func(args)