diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2021-10-01 17:39:40 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2021-10-01 17:39:43 -0700 |
| commit | b72c18e3518e827bd09044deaadcbf0b0ca50335 (patch) | |
| tree | 9c79a4907971ea3f940a4069d5c3c48d4acb6361 /python/fatcat_import.py | |
| parent | 9618d5146eea046342b69895e68b937a056d2816 (diff) | |
| download | fatcat-b72c18e3518e827bd09044deaadcbf0b0ca50335.tar.gz fatcat-b72c18e3518e827bd09044deaadcbf0b0ca50335.zip | |
kafka import: optional 'force-flush' mode for some importers
Behavior and motivation described in the kafka json import comment.
Diffstat (limited to 'python/fatcat_import.py')
| -rwxr-xr-x | python/fatcat_import.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py index 7e790fa4..b82e81c7 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -159,6 +159,7 @@ def run_ingest_web(args): "fatcat-{}-ingest-web-result".format(args.kafka_env), kafka_namespace="sandcrawler", consume_batch_size=args.batch_size, + force_flush=True, ).run() else: JsonLinePusher(iwri, args.json_file).run() @@ -176,6 +177,7 @@ def run_savepapernow_file(args): "fatcat-{}-savepapernow-file-result".format(args.kafka_env), kafka_namespace="sandcrawler", consume_batch_size=args.batch_size, + force_flush=True, ).run() else: JsonLinePusher(ifri, args.json_file).run() @@ -193,6 +195,7 @@ def run_savepapernow_web(args): "fatcat-{}-savepapernow-web-result".format(args.kafka_env), kafka_namespace="sandcrawler", consume_batch_size=args.batch_size, + force_flush=True, ).run() else: JsonLinePusher(ifri, args.json_file).run() |