diff options
author | Bryan Newbold <bnewbold@archive.org> | 2020-08-05 13:10:56 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2020-08-05 13:10:56 -0700 |
commit | af140307a025738767e740fea8da8d15e20fb983 (patch) | |
tree | 39b3413e2db69ea7568f45eff0eb228be040a59f /sql/dump_ungrobid_pdf_petabox.sql | |
parent | ec09418b88c0bf77fa2e8b4319529fa8044918c3 (diff) | |
download | sandcrawler-af140307a025738767e740fea8da8d15e20fb983.tar.gz sandcrawler-af140307a025738767e740fea8da8d15e20fb983.zip |
grobid+pdftext missing catch-up commands
Diffstat (limited to 'sql/dump_ungrobid_pdf_petabox.sql')
-rw-r--r-- | sql/dump_ungrobid_pdf_petabox.sql | 17 |
1 files changed, 17 insertions, 0 deletions
diff --git a/sql/dump_ungrobid_pdf_petabox.sql b/sql/dump_ungrobid_pdf_petabox.sql new file mode 100644 index 0000000..f758ec2 --- /dev/null +++ b/sql/dump_ungrobid_pdf_petabox.sql @@ -0,0 +1,17 @@ + +-- Run like: +-- psql sandcrawler < dump_ungrobid_pdf_petabox.sql + +BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE; + +COPY ( + SELECT DISTINCT ON (petabox.sha1hex) row_to_json(petabox) + FROM petabox + WHERE NOT EXISTS (SELECT grobid.sha1hex FROM grobid WHERE petabox.sha1hex = grobid.sha1hex AND grobid.status IS NOT NULL) + -- uncomment/comment this to control whether only fatcat files are included + AND EXISTS (SELECT fatcat_file.sha1hex FROM fatcat_file WHERE petabox.sha1hex = fatcat_file.sha1hex) +) +TO '/grande/snapshots/dump_ungrobided_pdf_petabox.2020-08-04.json' +WITH NULL ''; + +ROLLBACK; |