arxiv importer robustification and CLI impl

1 files changed, 21 insertions, 0 deletions

diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index 94c90ea5..e80c5d5b 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -22,6 +22,15 @@ def run_jalc(args):
         extid_map_file=args.extid_map_file)
     Bs4XmlLinesPusher(ji, args.xml_file, "<rdf:Description").run()
 
+def run_arxiv(args):
+    ari = ArxivRawImporter(args.api,
+        edit_batch_size=args.batch_size)
+    if args.kafka_mode:
+        raise NotImplementedError
+        #KafkaBs4XmlPusher(ari, args.kafka_hosts, args.kafka_env, "api-arxiv", "fatcat-import").run()
+    else:
+        Bs4XmlFilePusher(ari, args.xml_file, "record").run()
+
 def run_orcid(args):
     foi = OrcidImporter(args.api,
         edit_batch_size=args.batch_size)
@@ -164,6 +173,18 @@ def main():
         help="DOI-to-other-identifiers sqlite3 database",
         default=None, type=str)
 
+    sub_arxiv = subparsers.add_parser('arxiv')
+    sub_arxiv.set_defaults(
+        func=run_arxiv,
+        auth_var="FATCAT_AUTH_WORKER_ARXIV",
+    )
+    sub_arxiv.add_argument('xml_file',
+        help="arXivRaw XML file to import from",
+        default=sys.stdin, type=argparse.FileType('r'))
+    sub_arxiv.add_argument('--kafka-mode',
+        action='store_true',
+        help="consume from kafka topic (not stdin)")
+
     sub_orcid = subparsers.add_parser('orcid')
     sub_orcid.set_defaults(
         func=run_orcid,