early version of arabesque importer

2 files changed, 29 insertions, 0 deletions

diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index aea8c757..f04a63ef 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -31,6 +31,16 @@ def run_matched(args):
         edit_batch_size=args.batch_size)
     JsonLinePusher(fmi, args.json_file).run()
 
+def run_arabesque_matched(args):
+    ami = ArabesqueMatchImporter(args.api,
+        do_updates=args.do_updates,
+        extid_type=args.extid_type,
+        crawl_id=args.crawl_id,
+        default_link_rel=args.default_link_rel,
+        edit_batch_size=args.batch_size)
+    SqlitePusher(ami, args.db_file, "crawl_result",
+        ARABESQUE_MATCH_WHERE_CLAUSE).run()
+
 def run_grobid_metadata(args):
     fmi = GrobidMetadataImporter(args.api,
         edit_batch_size=args.batch_size,
@@ -151,6 +161,24 @@ def main():
         action='store_true',
         help="don't lookup existing files, just insert (clobbers; only for fast bootstrap)")
 
+    sub_arabesque_matched = subparsers.add_parser('arabesque_matched')
+    sub_arabesque_matched.set_defaults(
+        func=run_arabesque_matched,
+        auth_var="FATCAT_API_AUTH_TOKEN",
+    )
+    sub_arabesque_matched.add_argument('db_file',
+        help="sqlite database file to import from")
+    sub_arabesque_matched.add_argument('--do-updates',
+        action='store_true',
+        help="update pre-existing file entities if new match (instead of skipping)")
+    sub_arabesque_matched.add_argument('--extid-type',
+        default="doi",
+        help="identifer type in the database (eg, 'doi', 'pmcid'")
+    sub_arabesque_matched.add_argument('--crawl-id',
+        help="crawl ID (optionally included in editgroup metadata)")
+    sub_arabesque_matched.add_argument('--default-link-rel',
+        help="default URL rel for matches (eg, 'publisher', 'web')")
+
     sub_grobid_metadata = subparsers.add_parser('grobid-metadata')
     sub_grobid_metadata.set_defaults(
         func=run_grobid_metadata,
diff --git a/python/fatcat_tools/importers/__init__.py b/python/fatcat_tools/importers/__init__.py
index 94802915..f5ff43e5 100644
--- a/python/fatcat_tools/importers/__init__.py
+++ b/python/fatcat_tools/importers/__init__.py
@@ -18,6 +18,7 @@ from .grobid_metadata import GrobidMetadataImporter
 from .journal_metadata import JournalMetadataImporter
 from .matched import MatchedImporter
 from .orcid import OrcidImporter
+from .arabesque import ArabesqueMatchImporter, ARABESQUE_MATCH_WHERE_CLAUSE
 from .wayback_static import auto_wayback_static
 from .cdl_dash_dat import auto_cdl_dash_dat
 #from .kafka_source import KafkaSource