summaryrefslogtreecommitdiffstats
path: root/python/fatcat_import.py
diff options
context:
space:
mode:
Diffstat (limited to 'python/fatcat_import.py')
-rwxr-xr-xpython/fatcat_import.py37
1 files changed, 37 insertions, 0 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index 19cf43ec..ff6c94dc 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -256,6 +256,24 @@ def run_datacite(args):
else:
JsonLinePusher(dci, args.json_file).run()
+def run_doaj_article(args):
+ dai = DoajArticleImporter(args.api,
+ args.issn_map_file,
+ edit_batch_size=args.batch_size,
+ do_updates=args.do_updates,
+ )
+ if args.kafka_mode:
+ KafkaJsonPusher(
+ dai,
+ args.kafka_hosts,
+ args.kafka_env,
+ "api-doaj",
+ "fatcat-{}-import-doaj".format(args.kafka_env),
+ consume_batch_size=args.batch_size,
+ ).run()
+ else:
+ JsonLinePusher(dai, args.json_file).run()
+
def run_file_meta(args):
# do_updates defaults to true for this importer
fmi = FileMetaImporter(args.api,
@@ -606,6 +624,25 @@ def main():
auth_var="FATCAT_AUTH_WORKER_DATACITE",
)
+ sub_doaj_article = subparsers.add_parser('doaj-article',
+ help="import doaj.org article metadata")
+ sub_doaj_article.add_argument('json_file',
+ help="File with JSON lines from DOAJ API (or bulk dump) to import from",
+ default=sys.stdin, type=argparse.FileType('r'))
+ sub_doaj_article.add_argument('--issn-map-file',
+ help="ISSN to ISSN-L mapping file",
+ default=None, type=argparse.FileType('r'))
+ sub_doaj_article.add_argument('--kafka-mode',
+ action='store_true',
+ help="consume from kafka topic (not stdin)")
+ sub_doaj_article.add_argument('--do-updates',
+ action='store_true',
+ help="update any pre-existing release entities")
+ sub_doaj_article.set_defaults(
+ func=run_doaj_article,
+ auth_var="FATCAT_AUTH_WORKER_DOAJ",
+ )
+
sub_file_meta = subparsers.add_parser('file-meta',
help="simple update-only importer for file metadata")
sub_file_meta.set_defaults(