From 1a42a73e8c494d13e671d0fc6729665d85010669 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 24 Feb 2020 12:54:59 -0800 Subject: ingest backfill notes --- notes/tasks/2020-02-18_ingest_backfills.md | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 notes/tasks/2020-02-18_ingest_backfills.md (limited to 'notes/tasks/2020-02-18_ingest_backfills.md') diff --git a/notes/tasks/2020-02-18_ingest_backfills.md b/notes/tasks/2020-02-18_ingest_backfills.md new file mode 100644 index 0000000..1ab18f4 --- /dev/null +++ b/notes/tasks/2020-02-18_ingest_backfills.md @@ -0,0 +1,42 @@ + +Select: + + COPY ( + SELECT row_to_json(ingest_request.*) FROM ingest_request + LEFT JOIN ingest_file_result ON ingest_file_result.base_url = ingest_request.base_url + WHERE ingest_request.ingest_type = 'pdf' + AND ingest_file_result.ingest_type = 'pdf' + AND ingest_file_result.updated < NOW() - '2 day'::INTERVAL + AND ingest_file_result.hit = false + AND ingest_file_result.status like 'spn2-error%' + ) TO '/grande/snapshots/reingest_spn2err_20200218.rows.json'; + => COPY 6537 + + COPY ( + SELECT row_to_json(ingest_request.*) FROM ingest_request + LEFT JOIN ingest_file_result ON ingest_file_result.base_url = ingest_request.base_url + WHERE ingest_request.ingest_type = 'pdf' + AND ingest_file_result.ingest_type = 'pdf' + AND ingest_file_result.hit = false + AND ingest_file_result.status like 'wayback-error' + ) TO '/grande/snapshots/reingest_waybackerr_20200218.rows.json'; + => COPY 33022 + +Transform: + + ./scripts/ingestrequest_row2json.py reingest_spn2err_20200218.rows.json > reingest_spn2err_20200218.json + ./scripts/ingestrequest_row2json.py reingest_waybackerr_20200218.rows.json > reingest_waybackerr_20200218.json + +Push to kafka: + + cat reingest_spn2err_20200218.json | jq . -c | kafkacat -P -b wbgrp-svc263.us.archive.org -t sandcrawler-prod.ingest-file-requests -p -1 + cat reingest_waybackerr_20200218.json | jq . -c | kafkacat -P -b wbgrp-svc263.us.archive.org -t sandcrawler-prod.ingest-file-requests -p -1 + +Many had null `ingest_request_source`, so won't actually import into fatcat: + + bnewbold@ia601101$ cat reingest_waybackerr_20200218.json | jq .ingest_request_source | sort | uniq -c | sort -n + 1 "savepapernow-web" + 112 "fatcat-ingest-container" + 11750 "fatcat-changelog" + 21159 null + -- cgit v1.2.3