From 8577d50b644dd45bce5275675eed4d43bb816b67 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 3 Nov 2021 18:35:44 -0700 Subject: crossref persist: batch size depends on whether parsing refs --- python/persist_tool.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'python/persist_tool.py') diff --git a/python/persist_tool.py b/python/persist_tool.py index 5cf5776..069bef7 100755 --- a/python/persist_tool.py +++ b/python/persist_tool.py @@ -120,6 +120,9 @@ def run_ingest_request(args): def run_crossref(args): + batch_size = 200 + if args.parse_refs: + batch_size = 10 grobid_client = GrobidClient( host_url=args.grobid_host, ) @@ -131,7 +134,7 @@ def run_crossref(args): pusher = JsonLinePusher( worker, args.json_file, - batch_size=10, + batch_size=batch_size, ) pusher.run() -- cgit v1.2.3