From 54e14814080d9a706ff6f15694b3b54918200169 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 3 Oct 2022 10:16:26 -0700 Subject: reingests: update scripts and SQL --- sql/reingest_terminalstatus_forcerecrawl.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 sql/reingest_terminalstatus_forcerecrawl.sh (limited to 'sql/reingest_terminalstatus_forcerecrawl.sh') diff --git a/sql/reingest_terminalstatus_forcerecrawl.sh b/sql/reingest_terminalstatus_forcerecrawl.sh new file mode 100755 index 0000000..5cb6d51 --- /dev/null +++ b/sql/reingest_terminalstatus_forcerecrawl.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e # fail on error +set -u # fail if variable not set in substitution +set -o pipefail # fail if part of a '|' command fails + +sudo -u postgres psql sandcrawler < dump_reingest_terminalstatus.sql + +cd ../python +sudo -u sandcrawler pipenv run \ + ./scripts/ingestrequest_row2json.py --force-recrawl /srv/sandcrawler/tasks/reingest_terminalstatus_current.rows.json \ + > /srv/sandcrawler/tasks/reingest_terminalstatus_current.json + +cat /srv/sandcrawler/tasks/reingest_terminalstatus_current.json \ + | shuf \ + | head -n100000 \ + | jq . -c \ + | kafkacat -P -b wbgrp-svc350.us.archive.org -t sandcrawler-prod.ingest-file-requests-daily -p -1 + -- cgit v1.2.3