aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xpython/fatcat_import.py28
-rw-r--r--python/fatcat_tools/importers/__init__.py1
2 files changed, 29 insertions, 0 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index aea8c757..f04a63ef 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -31,6 +31,16 @@ def run_matched(args):
edit_batch_size=args.batch_size)
JsonLinePusher(fmi, args.json_file).run()
+def run_arabesque_matched(args):
+ ami = ArabesqueMatchImporter(args.api,
+ do_updates=args.do_updates,
+ extid_type=args.extid_type,
+ crawl_id=args.crawl_id,
+ default_link_rel=args.default_link_rel,
+ edit_batch_size=args.batch_size)
+ SqlitePusher(ami, args.db_file, "crawl_result",
+ ARABESQUE_MATCH_WHERE_CLAUSE).run()
+
def run_grobid_metadata(args):
fmi = GrobidMetadataImporter(args.api,
edit_batch_size=args.batch_size,
@@ -151,6 +161,24 @@ def main():
action='store_true',
help="don't lookup existing files, just insert (clobbers; only for fast bootstrap)")
+ sub_arabesque_matched = subparsers.add_parser('arabesque_matched')
+ sub_arabesque_matched.set_defaults(
+ func=run_arabesque_matched,
+ auth_var="FATCAT_API_AUTH_TOKEN",
+ )
+ sub_arabesque_matched.add_argument('db_file',
+ help="sqlite database file to import from")
+ sub_arabesque_matched.add_argument('--do-updates',
+ action='store_true',
+ help="update pre-existing file entities if new match (instead of skipping)")
+ sub_arabesque_matched.add_argument('--extid-type',
+ default="doi",
+ help="identifer type in the database (eg, 'doi', 'pmcid'")
+ sub_arabesque_matched.add_argument('--crawl-id',
+ help="crawl ID (optionally included in editgroup metadata)")
+ sub_arabesque_matched.add_argument('--default-link-rel',
+ help="default URL rel for matches (eg, 'publisher', 'web')")
+
sub_grobid_metadata = subparsers.add_parser('grobid-metadata')
sub_grobid_metadata.set_defaults(
func=run_grobid_metadata,
diff --git a/python/fatcat_tools/importers/__init__.py b/python/fatcat_tools/importers/__init__.py
index 94802915..f5ff43e5 100644
--- a/python/fatcat_tools/importers/__init__.py
+++ b/python/fatcat_tools/importers/__init__.py
@@ -18,6 +18,7 @@ from .grobid_metadata import GrobidMetadataImporter
from .journal_metadata import JournalMetadataImporter
from .matched import MatchedImporter
from .orcid import OrcidImporter
+from .arabesque import ArabesqueMatchImporter, ARABESQUE_MATCH_WHERE_CLAUSE
from .wayback_static import auto_wayback_static
from .cdl_dash_dat import auto_cdl_dash_dat
#from .kafka_source import KafkaSource