diff options
author | Bryan Newbold <bnewbold@archive.org> | 2020-08-05 13:10:56 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2020-08-05 13:10:56 -0700 |
commit | af140307a025738767e740fea8da8d15e20fb983 (patch) | |
tree | 39b3413e2db69ea7568f45eff0eb228be040a59f /sql/dump_unextracted_pdf.sql | |
parent | ec09418b88c0bf77fa2e8b4319529fa8044918c3 (diff) | |
download | sandcrawler-af140307a025738767e740fea8da8d15e20fb983.tar.gz sandcrawler-af140307a025738767e740fea8da8d15e20fb983.zip |
grobid+pdftext missing catch-up commands
Diffstat (limited to 'sql/dump_unextracted_pdf.sql')
-rw-r--r-- | sql/dump_unextracted_pdf.sql | 11 |
1 files changed, 6 insertions, 5 deletions
diff --git a/sql/dump_unextracted_pdf.sql b/sql/dump_unextracted_pdf.sql index 5c7e7a7..7b5e823 100644 --- a/sql/dump_unextracted_pdf.sql +++ b/sql/dump_unextracted_pdf.sql @@ -1,19 +1,20 @@ -- Run like: --- psql sandcrawler < dump_unextracted_pdf.sql > dump_unextracted_pdf.2019-09-23.json +-- psql sandcrawler < dump_unextracted_pdf.sql BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE; COPY ( - SELECT row_to_json(cdx) + SELECT DISTINCT ON (cdx.sha1hex) row_to_json(cdx) FROM grobid LEFT JOIN cdx ON grobid.sha1hex = cdx.sha1hex --LEFT JOIN fatcat_file ON grobid.sha1hex = fatcat_file.sha1hex + LEFT JOIN pdf_meta ON grobid.sha1hex = pdf_meta.sha1hex WHERE cdx.sha1hex IS NOT NULL - --AND fatcat_file.sha1hex IS NOT NULL + --AND fatcat_file.sha1hex IS NOT NULL + AND pdf_meta.sha1hex IS NULL ) ---TO '/grande/snapshots/dump_unextracted_pdf.2020-06-25.json'; -TO STDOUT +TO '/grande/snapshots/dump_unextracted_pdf.fatcat.2020-07-22.json' WITH NULL ''; ROLLBACK; |