diff options
author | Bryan Newbold <bnewbold@archive.org> | 2020-03-05 00:41:14 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2020-03-05 00:41:14 -0800 |
commit | 97ffdc17941b1272a8c7f05c0d1353cd28761280 (patch) | |
tree | 07309077043139328c9be809ac5f26971a2729c9 /python/persist_tool.py | |
parent | 173e5e88de4160a63949ff6e263123c4a25b2017 (diff) | |
download | sandcrawler-97ffdc17941b1272a8c7f05c0d1353cd28761280.tar.gz sandcrawler-97ffdc17941b1272a8c7f05c0d1353cd28761280.zip |
persist: ingest_request tool (with no ingest_file_result)
Diffstat (limited to 'python/persist_tool.py')
-rwxr-xr-x | python/persist_tool.py | 18 |
1 files changed, 18 insertions, 0 deletions
diff --git a/python/persist_tool.py b/python/persist_tool.py index 80b1156..f0beef8 100755 --- a/python/persist_tool.py +++ b/python/persist_tool.py @@ -85,6 +85,17 @@ def run_ingest_file_result(args): ) pusher.run() +def run_ingest_request(args): + worker = PersistIngestRequestWorker( + db_url=args.db_url, + ) + pusher = JsonLinePusher( + worker, + args.json_file, + batch_size=200, + ) + pusher.run() + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -149,6 +160,13 @@ def main(): help="ingest_file_result file to import from (or '-' for stdin)", type=argparse.FileType('r')) + sub_ingest_request = subparsers.add_parser('ingest-request', + help="backfill a ingest_request JSON dump into postgresql") + sub_ingest_request.set_defaults(func=run_ingest_file_result) + sub_ingest_request.add_argument('json_file', + help="ingest_request to import from (or '-' for stdin)", + type=argparse.FileType('r')) + args = parser.parse_args() if not args.__dict__.get("func"): print("Tell me what to do!", file=sys.stderr) |