arabesque import tweaks

2 files changed, 9 insertions, 1 deletions

diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index f04a63ef..cd325697 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -34,6 +34,7 @@ def run_matched(args):
 def run_arabesque_matched(args):
     ami = ArabesqueMatchImporter(args.api,
         do_updates=args.do_updates,
+        require_grobid=(not args.no_require_grobid),
         extid_type=args.extid_type,
         crawl_id=args.crawl_id,
         default_link_rel=args.default_link_rel,
@@ -171,6 +172,9 @@ def main():
     sub_arabesque_matched.add_argument('--do-updates',
         action='store_true',
         help="update pre-existing file entities if new match (instead of skipping)")
+    sub_arabesque_matched.add_argument('--no-require-grobid',
+        action='store_true',
+        help="whether postproc_status column must be '200'")
     sub_arabesque_matched.add_argument('--extid-type',
         default="doi",
         help="identifer type in the database (eg, 'doi', 'pmcid'")
diff --git a/python/tests/import_arabesque.py b/python/tests/import_arabesque.py
index 516b0ec2..9d74f96c 100644
--- a/python/tests/import_arabesque.py
+++ b/python/tests/import_arabesque.py
@@ -1,7 +1,7 @@
 
 import json
 import pytest
-from fatcat_tools.importers import ArabesqueMatchImporter, SqlitePusher
+from fatcat_tools.importers import ArabesqueMatchImporter, SqlitePusher, JsonLinePusher
 from fixtures import api
 
 
@@ -13,6 +13,10 @@ def arabesque_importer(api):
 def test_arabesque_importer_basic(arabesque_importer):
     SqlitePusher(arabesque_importer, 'tests/files/arabesque_example.sqlite3', "crawl_result").run()
 
+def test_arabesque_importer_json(arabesque_importer):
+    with open('tests/files/arabesque_example.json', 'r') as f:
+        JsonLinePusher(arabesque_importer, f).run()
+
 def test_arabesque_importer(arabesque_importer):
     last_index = arabesque_importer.api.get_changelog(limit=1)[0].index
     arabesque_importer.bezerk_mode = True