aboutsummaryrefslogtreecommitdiffstats
path: root/sql/reingest_bulk.sh
blob: d39a171efb5bf06a1f46fd8d0c0c827afca347b5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/bin/bash

set -e              # fail on error
set -u              # fail if variable not set in substitution
set -o pipefail     # fail if part of a '|' command fails

sudo -u postgres psql sandcrawler < dump_reingest_bulk.sql

cd ../python
sudo -u sandcrawler pipenv run \
    ./scripts/ingestrequest_row2json.py /srv/sandcrawler/tasks/reingest_bulk_current.rows.json \
    > /srv/sandcrawler/tasks/reingest_bulk_current.json

cat /srv/sandcrawler/tasks/reingest_bulk_current.json \
    | shuf \
    | head -n1000000 \
    | jq . -c \
    | kafkacat -P -b wbgrp-svc350.us.archive.org -t sandcrawler-prod.ingest-file-requests-bulk -p -1