From af140307a025738767e740fea8da8d15e20fb983 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 5 Aug 2020 13:10:56 -0700 Subject: grobid+pdftext missing catch-up commands --- sql/dump_unextracted_pdf.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'sql/dump_unextracted_pdf.sql') diff --git a/sql/dump_unextracted_pdf.sql b/sql/dump_unextracted_pdf.sql index 5c7e7a7..7b5e823 100644 --- a/sql/dump_unextracted_pdf.sql +++ b/sql/dump_unextracted_pdf.sql @@ -1,19 +1,20 @@ -- Run like: --- psql sandcrawler < dump_unextracted_pdf.sql > dump_unextracted_pdf.2019-09-23.json +-- psql sandcrawler < dump_unextracted_pdf.sql BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE; COPY ( - SELECT row_to_json(cdx) + SELECT DISTINCT ON (cdx.sha1hex) row_to_json(cdx) FROM grobid LEFT JOIN cdx ON grobid.sha1hex = cdx.sha1hex --LEFT JOIN fatcat_file ON grobid.sha1hex = fatcat_file.sha1hex + LEFT JOIN pdf_meta ON grobid.sha1hex = pdf_meta.sha1hex WHERE cdx.sha1hex IS NOT NULL - --AND fatcat_file.sha1hex IS NOT NULL + --AND fatcat_file.sha1hex IS NOT NULL + AND pdf_meta.sha1hex IS NULL ) ---TO '/grande/snapshots/dump_unextracted_pdf.2020-06-25.json'; -TO STDOUT +TO '/grande/snapshots/dump_unextracted_pdf.fatcat.2020-07-22.json' WITH NULL ''; ROLLBACK; -- cgit v1.2.3