diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-11-12 13:22:18 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2019-11-12 13:22:18 -0800 |
| commit | e0ad2e3be5286b2703df8f0a98b450658e28d28b (patch) | |
| tree | 4f2781af5cffe42531b8585b1d5e79607c613d28 /sql/dump_regrobid_pdf.sql | |
| parent | 2e3611f0e66615ae007d4e46bb5905e2220fb690 (diff) | |
| download | sandcrawler-e0ad2e3be5286b2703df8f0a98b450658e28d28b.tar.gz sandcrawler-e0ad2e3be5286b2703df8f0a98b450658e28d28b.zip | |
SQL stats and commands (mostly from sept 2019)
Diffstat (limited to 'sql/dump_regrobid_pdf.sql')
| -rw-r--r-- | sql/dump_regrobid_pdf.sql | 15 |
1 files changed, 15 insertions, 0 deletions
diff --git a/sql/dump_regrobid_pdf.sql b/sql/dump_regrobid_pdf.sql new file mode 100644 index 0000000..b846834 --- /dev/null +++ b/sql/dump_regrobid_pdf.sql @@ -0,0 +1,15 @@ + +-- Run like: +-- psql sandcrawler < dump_regrobid_pdf.sql | sort -S 4G | uniq -w 40 | cut -f2 > dump_regrobid_pdf.2019-11-12.json + +BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE; + +COPY ( + SELECT cdx.sha1hex, row_to_json(cdx) FROM cdx + WHERE cdx.mimetype = 'application/pdf' + AND EXISTS (SELECT grobid.sha1hex FROM grobid WHERE cdx.sha1hex = grobid.sha1hex AND grobid.grobid_version IS NULL) +) +TO STDOUT +WITH NULL ''; + +ROLLBACK; |