aboutsummaryrefslogtreecommitdiffstats
path: root/python/persist_tool.py
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-03-05 00:41:14 -0800
committerBryan Newbold <bnewbold@archive.org>2020-03-05 00:41:14 -0800
commit97ffdc17941b1272a8c7f05c0d1353cd28761280 (patch)
tree07309077043139328c9be809ac5f26971a2729c9 /python/persist_tool.py
parent173e5e88de4160a63949ff6e263123c4a25b2017 (diff)
downloadsandcrawler-97ffdc17941b1272a8c7f05c0d1353cd28761280.tar.gz
sandcrawler-97ffdc17941b1272a8c7f05c0d1353cd28761280.zip
persist: ingest_request tool (with no ingest_file_result)
Diffstat (limited to 'python/persist_tool.py')
-rwxr-xr-xpython/persist_tool.py18
1 files changed, 18 insertions, 0 deletions
diff --git a/python/persist_tool.py b/python/persist_tool.py
index 80b1156..f0beef8 100755
--- a/python/persist_tool.py
+++ b/python/persist_tool.py
@@ -85,6 +85,17 @@ def run_ingest_file_result(args):
)
pusher.run()
+def run_ingest_request(args):
+ worker = PersistIngestRequestWorker(
+ db_url=args.db_url,
+ )
+ pusher = JsonLinePusher(
+ worker,
+ args.json_file,
+ batch_size=200,
+ )
+ pusher.run()
+
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -149,6 +160,13 @@ def main():
help="ingest_file_result file to import from (or '-' for stdin)",
type=argparse.FileType('r'))
+ sub_ingest_request = subparsers.add_parser('ingest-request',
+ help="backfill a ingest_request JSON dump into postgresql")
+ sub_ingest_request.set_defaults(func=run_ingest_file_result)
+ sub_ingest_request.add_argument('json_file',
+ help="ingest_request to import from (or '-' for stdin)",
+ type=argparse.FileType('r'))
+
args = parser.parse_args()
if not args.__dict__.get("func"):
print("Tell me what to do!", file=sys.stderr)