diff options
Diffstat (limited to 'python/fatcat_import.py')
| -rwxr-xr-x | python/fatcat_import.py | 33 |
1 files changed, 33 insertions, 0 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py index 1dcfec21..b82e81c7 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -159,6 +159,7 @@ def run_ingest_web(args): "fatcat-{}-ingest-web-result".format(args.kafka_env), kafka_namespace="sandcrawler", consume_batch_size=args.batch_size, + force_flush=True, ).run() else: JsonLinePusher(iwri, args.json_file).run() @@ -176,6 +177,25 @@ def run_savepapernow_file(args): "fatcat-{}-savepapernow-file-result".format(args.kafka_env), kafka_namespace="sandcrawler", consume_batch_size=args.batch_size, + force_flush=True, + ).run() + else: + JsonLinePusher(ifri, args.json_file).run() + +def run_savepapernow_web(args): + ifri = SavePaperNowWebImporter(args.api, + editgroup_description=args.editgroup_description_override, + edit_batch_size=args.batch_size) + if args.kafka_mode: + KafkaJsonPusher( + ifri, + args.kafka_hosts, + args.kafka_env, + "ingest-file-results", + "fatcat-{}-savepapernow-web-result".format(args.kafka_env), + kafka_namespace="sandcrawler", + consume_batch_size=args.batch_size, + force_flush=True, ).run() else: JsonLinePusher(ifri, args.json_file).run() @@ -554,6 +574,19 @@ def main(): action='store_true', help="consume from kafka topic (not stdin)") + sub_savepapernow_web = subparsers.add_parser('savepapernow-web-results', + help="add webcapture entities crawled due to async Save Paper Now request") + sub_savepapernow_web.set_defaults( + func=run_savepapernow_web, + auth_var="FATCAT_AUTH_WORKER_SAVEPAPERNOW", + ) + sub_savepapernow_web.add_argument('json_file', + help="ingest-file JSON file to import from", + default=sys.stdin, type=argparse.FileType('r')) + sub_savepapernow_web.add_argument('--kafka-mode', + action='store_true', + help="consume from kafka topic (not stdin)") + sub_grobid_metadata = subparsers.add_parser('grobid-metadata', help="create release and file entities based on GROBID PDF metadata extraction") sub_grobid_metadata.set_defaults( |