From 8f85ab294eae50e31efa9e31bb0bca1bca76cf8b Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 7 Jul 2022 13:19:40 -0700 Subject: ingest: targeted 2022-04 notes --- notes/ingest/2022-04_targeted.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'notes') diff --git a/notes/ingest/2022-04_targeted.md b/notes/ingest/2022-04_targeted.md index 89fe40a..23fd35f 100644 --- a/notes/ingest/2022-04_targeted.md +++ b/notes/ingest/2022-04_targeted.md @@ -138,5 +138,7 @@ TODO: are there any cases where we do a bulk ingest request, fail, and `terminal cd /srv/sandcrawler/src/python sudo su sandcrawler pipenv run ./scripts/ingestrequest_row2json.py /srv/sandcrawler/tasks/patch_ingest_request_$PATCHDATE.rows.json | pv -l > /srv/sandcrawler/tasks/patch_ingest_request_$PATCHDATE.ingest_request.json + => 4.84M 0:03:14 [24.9k/s] - cat /srv/sandcrawler/tasks/patch_ingest_request_$PATCHDATE.ingest_request.json | rg -v "\\\\" | jq . -c | kafkacat -P -b wbgrp-svc263.us.archive.org -t sandcrawler-prod.ingest-file-requests-bulk -p -1 + cat /srv/sandcrawler/tasks/patch_ingest_request_$PATCHDATE.ingest_request.json | rg -v "\\\\" | jq . -c | kafkacat -P -b wbgrp-svc350.us.archive.org -t sandcrawler-prod.ingest-file-requests-bulk -p -1 + => started 2022-05-11 -- cgit v1.2.3