diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2021-09-30 16:49:47 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2021-09-30 16:49:47 -0700 |
| commit | f6125848f627ae9bfd3a36d807d2349e1c66bfe3 (patch) | |
| tree | 8419947e17dd27621a5b17098eacaedf1a0227a5 /sql/reingest_spn.sh | |
| parent | 24edc1c754d35df1020907a9c8941f16eaf71440 (diff) | |
| download | sandcrawler-f6125848f627ae9bfd3a36d807d2349e1c66bfe3.tar.gz sandcrawler-f6125848f627ae9bfd3a36d807d2349e1c66bfe3.zip | |
refactor reingest scripts
Diffstat (limited to 'sql/reingest_spn.sh')
| -rwxr-xr-x | sql/reingest_spn.sh | 19 |
1 files changed, 19 insertions, 0 deletions
diff --git a/sql/reingest_spn.sh b/sql/reingest_spn.sh new file mode 100755 index 0000000..6fb1e4b --- /dev/null +++ b/sql/reingest_spn.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e # fail on error +set -u # fail if variable not set in substitution +set -o pipefail # fail if part of a '|' command fails + +sudo -u postgres psql sandcrawler < dump_reingest_spn.sql + +cd ../python +sudo -u sandcrawler pipenv run \ + ./scripts/ingestrequest_row2json.py /srv/sandcrawler/tasks/reingest_spn.rows.json \ + > /srv/sandcrawler/tasks/reingest_spn.json + +cat /srv/sandcrawler/tasks/reingest_spn.json \ + | shuf \ + | head -n60000 \ + | jq . -c \ + | kafkacat -P -b wbgrp-svc263.us.archive.org -t sandcrawler-prod.ingest-file-requests-priority -p -1 + |