From af140307a025738767e740fea8da8d15e20fb983 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Wed, 5 Aug 2020 13:10:56 -0700
Subject: grobid+pdftext missing catch-up commands

---
 sql/dump_unextracted_pdf.sql | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

(limited to 'sql/dump_unextracted_pdf.sql')

diff --git a/sql/dump_unextracted_pdf.sql b/sql/dump_unextracted_pdf.sql
index 5c7e7a7..7b5e823 100644
--- a/sql/dump_unextracted_pdf.sql
+++ b/sql/dump_unextracted_pdf.sql
@@ -1,19 +1,20 @@
 
 -- Run like:
---   psql sandcrawler < dump_unextracted_pdf.sql > dump_unextracted_pdf.2019-09-23.json
+--   psql sandcrawler < dump_unextracted_pdf.sql
 
 BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE;
 
 COPY (
-  SELECT row_to_json(cdx)
+  SELECT DISTINCT ON (cdx.sha1hex) row_to_json(cdx)
   FROM grobid
   LEFT JOIN cdx ON grobid.sha1hex = cdx.sha1hex
   --LEFT JOIN fatcat_file ON grobid.sha1hex = fatcat_file.sha1hex
+  LEFT JOIN pdf_meta ON grobid.sha1hex = pdf_meta.sha1hex
   WHERE cdx.sha1hex IS NOT NULL
-  --AND fatcat_file.sha1hex IS NOT NULL
+    --AND fatcat_file.sha1hex IS NOT NULL
+    AND pdf_meta.sha1hex IS NULL
 )
---TO '/grande/snapshots/dump_unextracted_pdf.2020-06-25.json';
-TO STDOUT
+TO '/grande/snapshots/dump_unextracted_pdf.fatcat.2020-07-22.json'
 WITH NULL '';
 
 ROLLBACK;
-- 
cgit v1.2.3

