diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2022-02-08 16:57:16 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2022-02-08 16:57:16 -0800 |
| commit | b76630806d43427ddcdef8cd6efd110c731668a5 (patch) | |
| tree | 2b0d9c940ab6abc13ed36e4c4ecf0a4011ba5426 /sql/reingest_bulk.sh | |
| parent | 963fc4a50e0eddc99c14c6ff571d099e8873abac (diff) | |
| download | sandcrawler-b76630806d43427ddcdef8cd6efd110c731668a5.tar.gz sandcrawler-b76630806d43427ddcdef8cd6efd110c731668a5.zip | |
sql: script to reingest recent spn2 lookup failure in bulk mode
Diffstat (limited to 'sql/reingest_bulk.sh')
| -rwxr-xr-x | sql/reingest_bulk.sh | 19 |
1 files changed, 19 insertions, 0 deletions
diff --git a/sql/reingest_bulk.sh b/sql/reingest_bulk.sh new file mode 100755 index 0000000..d5d3e35 --- /dev/null +++ b/sql/reingest_bulk.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e # fail on error +set -u # fail if variable not set in substitution +set -o pipefail # fail if part of a '|' command fails + +sudo -u postgres psql sandcrawler < dump_reingest_bulk.sql + +cd ../python +sudo -u sandcrawler pipenv run \ + ./scripts/ingestrequest_row2json.py /srv/sandcrawler/tasks/reingest_bulk_current.rows.json \ + > /srv/sandcrawler/tasks/reingest_bulk_current.json + +cat /srv/sandcrawler/tasks/reingest_bulk_current.json \ + | shuf \ + | head -n1000000 \ + | jq . -c \ + | kafkacat -P -b wbgrp-svc263.us.archive.org -t sandcrawler-prod.ingest-file-requests-bulk -p -1 + |