diff options
Diffstat (limited to 'sql/dump_unextracted_pdf.sql')
| -rw-r--r-- | sql/dump_unextracted_pdf.sql | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/sql/dump_unextracted_pdf.sql b/sql/dump_unextracted_pdf.sql index 4d264b9..a7fb920 100644 --- a/sql/dump_unextracted_pdf.sql +++ b/sql/dump_unextracted_pdf.sql @@ -1,16 +1,22 @@ -- Run like: --- psql sandcrawler < dump_unextracted_pdf.sql > dump_unextracted_pdf.2019-09-23.json +-- psql sandcrawler < dump_unextracted_pdf.sql BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE; COPY ( - SELECT row_to_json(cdx) + SELECT DISTINCT ON (cdx.sha1hex) row_to_json(cdx) FROM grobid LEFT JOIN cdx ON grobid.sha1hex = cdx.sha1hex + --LEFT JOIN fatcat_file ON grobid.sha1hex = fatcat_file.sha1hex + LEFT JOIN ingest_file_result ON grobid.sha1hex = ingest_file_result.terminal_sha1hex + LEFT JOIN pdf_meta ON grobid.sha1hex = pdf_meta.sha1hex + WHERE cdx.sha1hex IS NOT NULL + --AND fatcat_file.sha1hex IS NOT NULL + AND ingest_file_result.terminal_sha1hex IS NOT NULL + AND pdf_meta.sha1hex IS NULL ) --- TO '/grande/snapshots/dump_unextracted_pdf.2020-06-25.json'; -TO STDOUT +TO '/srv/sandcrawler/tasks/dump_unextracted_pdf.ingest.2020-10-21.json' WITH NULL ''; ROLLBACK; |