1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
#!/usr/bin/env python3
"""
Count Chocula - online serials metadata and stats
"one, two, three, un-preserved web-native open-access long-tail indie
journals, hah, hah, hah!"
(yeah, I know, this name isn't very good)
(see also: https://teamyacht.com/ernstchoukula.com/Ernst-Choukula.html)
Commands:
everything
init_db
summarize
export
export_fatcat
export_urls
directory <source>
doaj
road
crossref
entrez
norwegian
szczepanski
ezb
wikidata
openapc
sim
load <source>
fatcat_containers
fatcat_stats
homepage_status
kbart <source>
jstor
clockss
lockss
portico
See TODO.md for more work-in-progress
"""
import sys
import csv
import argparse
from chocula import ChoculaDatabase, ChoculaConfig, IssnDatabase,\
ALL_CHOCULA_DIR_CLASSES, ALL_CHOCULA_KBART_CLASSES
def run_everything(config, database):
database.init_db()
for cls in ALL_CHOCULA_DIR_CLASSES:
loader = cls(config)
counts = loader.index_file(database)
print(counts)
for cls in ALL_CHOCULA_KBART_CLASSES:
loader = cls(config)
counts = loader.index_file(database)
print(counts)
database.load_fatcat_containers(config)
database.load_fatcat_stats(config)
database.load_homepage_status(config)
database.summarize()
print("### Done with everything!")
def run_directory(config, database, source):
for cls in ALL_CHOCULA_DIR_CLASSES:
if cls.source_slug == source:
loader = cls(config)
counts = loader.index_file(database)
print(counts)
return
raise NotImplementedError(f"unknown source: {source}")
def run_kbart(config, database, source):
for cls in ALL_CHOCULA_KBART_CLASSES:
if cls.source_slug == source:
loader = cls(config)
counts = loader.index_file(database)
print(counts)
return
raise NotImplementedError(f"unknown source: {source}")
def run_load(config, database, source):
if source == 'fatcat_stats':
print(database.load_fatcat_stats(config))
elif source == 'fatcat_containers':
print(database.load_fatcat_containers(config))
elif source == 'homepage_status':
print(database.load_homepage_status(config))
else:
raise NotImplementedError(f"unknown source: {source}")
def main():
parser = argparse.ArgumentParser(
prog="python -m chocula",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
subparsers = parser.add_subparsers()
parser.add_argument("--db-file",
help="sqlite database file",
default='chocula.sqlite',
type=str)
sub = subparsers.add_parser('everything',
help="run all the commands")
sub.set_defaults(func='everything')
sub = subparsers.add_parser('init_db',
help="create sqlite3 output file and tables")
sub.set_defaults(func='init_db')
sub = subparsers.add_parser('summarize',
help="aggregate metadata from all tables into 'journals' table")
sub.set_defaults(func='summarize')
sub = subparsers.add_parser('export',
help="dump JSON output")
sub.set_defaults(func='export')
sub = subparsers.add_parser('export_fatcat',
help="dump JSON output in a format that can load into fatcat")
sub.set_defaults(func='export_fatcat')
sub = subparsers.add_parser('export_urls',
help="dump homepage URLs (eg, to crawl for status)")
sub.set_defaults(func='export_urls')
sub = subparsers.add_parser('directory',
help="index directory metadata from a given source")
sub.add_argument("source", type=str, help="short name of source to index")
sub.set_defaults(func=run_directory)
sub = subparsers.add_parser('load',
help="load metadata of a given type")
sub.add_argument("source", type=str, help="short name of source to index")
sub.set_defaults(func=run_load)
sub = subparsers.add_parser('kbart',
help="index KBART holding metadata for a given source")
sub.add_argument("source", type=str, help="short name of source to index")
sub.set_defaults(func=run_kbart)
args = parser.parse_args()
if not args.__dict__.get("func"):
parser.print_help()
sys.exit(-1)
config = ChoculaConfig.from_file()
issn_db: Optional[IssnDatabase] = None
if args.func in ('everything', 'summarize', run_directory, run_kbart):
issn_db = IssnDatabase(config.issnl.filepath)
cdb = ChoculaDatabase(args.db_file, issn_db)
if args.func == 'everything':
run_everything(config, cdb)
elif args.func in (run_directory, run_load, run_kbart):
args.func(config, cdb, args.source)
else:
# all other commands run on the ChoculaDatabase itself
func = getattr(cdb, args.func)
print(func(), file=sys.stderr)
if __name__ == '__main__':
main()
|
