aboutsummaryrefslogtreecommitdiffstats
path: root/notes/tasks/2020-02-18_ingest_backfills.md
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-02-24 12:54:59 -0800
committerBryan Newbold <bnewbold@archive.org>2020-02-24 12:54:59 -0800
commit1a42a73e8c494d13e671d0fc6729665d85010669 (patch)
tree1f6fcad2a15e859183bf54d2ab678cb5c212dd74 /notes/tasks/2020-02-18_ingest_backfills.md
parentcfe04b60a28e90c694d2558eadb71c0b280c40e8 (diff)
downloadsandcrawler-1a42a73e8c494d13e671d0fc6729665d85010669.tar.gz
sandcrawler-1a42a73e8c494d13e671d0fc6729665d85010669.zip
ingest backfill notes
Diffstat (limited to 'notes/tasks/2020-02-18_ingest_backfills.md')
-rw-r--r--notes/tasks/2020-02-18_ingest_backfills.md42
1 files changed, 42 insertions, 0 deletions
diff --git a/notes/tasks/2020-02-18_ingest_backfills.md b/notes/tasks/2020-02-18_ingest_backfills.md
new file mode 100644
index 0000000..1ab18f4
--- /dev/null
+++ b/notes/tasks/2020-02-18_ingest_backfills.md
@@ -0,0 +1,42 @@
+
+Select:
+
+ COPY (
+ SELECT row_to_json(ingest_request.*) FROM ingest_request
+ LEFT JOIN ingest_file_result ON ingest_file_result.base_url = ingest_request.base_url
+ WHERE ingest_request.ingest_type = 'pdf'
+ AND ingest_file_result.ingest_type = 'pdf'
+ AND ingest_file_result.updated < NOW() - '2 day'::INTERVAL
+ AND ingest_file_result.hit = false
+ AND ingest_file_result.status like 'spn2-error%'
+ ) TO '/grande/snapshots/reingest_spn2err_20200218.rows.json';
+ => COPY 6537
+
+ COPY (
+ SELECT row_to_json(ingest_request.*) FROM ingest_request
+ LEFT JOIN ingest_file_result ON ingest_file_result.base_url = ingest_request.base_url
+ WHERE ingest_request.ingest_type = 'pdf'
+ AND ingest_file_result.ingest_type = 'pdf'
+ AND ingest_file_result.hit = false
+ AND ingest_file_result.status like 'wayback-error'
+ ) TO '/grande/snapshots/reingest_waybackerr_20200218.rows.json';
+ => COPY 33022
+
+Transform:
+
+ ./scripts/ingestrequest_row2json.py reingest_spn2err_20200218.rows.json > reingest_spn2err_20200218.json
+ ./scripts/ingestrequest_row2json.py reingest_waybackerr_20200218.rows.json > reingest_waybackerr_20200218.json
+
+Push to kafka:
+
+ cat reingest_spn2err_20200218.json | jq . -c | kafkacat -P -b wbgrp-svc263.us.archive.org -t sandcrawler-prod.ingest-file-requests -p -1
+ cat reingest_waybackerr_20200218.json | jq . -c | kafkacat -P -b wbgrp-svc263.us.archive.org -t sandcrawler-prod.ingest-file-requests -p -1
+
+Many had null `ingest_request_source`, so won't actually import into fatcat:
+
+ bnewbold@ia601101$ cat reingest_waybackerr_20200218.json | jq .ingest_request_source | sort | uniq -c | sort -n
+ 1 "savepapernow-web"
+ 112 "fatcat-ingest-container"
+ 11750 "fatcat-changelog"
+ 21159 null
+