aboutsummaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-04-10 17:20:40 -0700
committerBryan Newbold <bnewbold@archive.org>2020-04-10 17:20:40 -0700
commitb123a324d9ac5faad3167052710e71edaadf1bfe (patch)
tree4c3efea2a5f58d888056cb46831aae8e9e3a3791 /bin
parent44f12b817be9d4ccd4b2c0c6ea07ea401ae86ffa (diff)
downloadfatcat-covid19-b123a324d9ac5faad3167052710e71edaadf1bfe.tar.gz
fatcat-covid19-b123a324d9ac5faad3167052710e71edaadf1bfe.zip
pipeline scripts
Diffstat (limited to 'bin')
-rwxr-xr-xbin/make_pdf_derivatives.sh4
-rwxr-xr-xbin/run_pipeline.sh61
2 files changed, 63 insertions, 2 deletions
diff --git a/bin/make_pdf_derivatives.sh b/bin/make_pdf_derivatives.sh
index 8727119..76c9ebf 100755
--- a/bin/make_pdf_derivatives.sh
+++ b/bin/make_pdf_derivatives.sh
@@ -14,9 +14,9 @@ fi
echo "processing: $PDFPATH"
if [[ ! -f "$FULLTEXTDIR/pdftotext/$BLOBPATH.txt" ]]; then
- pdftotext $PDFPATH $FULLTEXTDIR/pdftotext/$BLOBPATH.txt
+ pdftotext $PDFPATH $FULLTEXTDIR/pdftotext/$BLOBPATH.txt || true
fi
if [[ ! -f "$FULLTEXTDIR/thumbnail/$BLOBPATH.png" ]]; then
- pdftocairo -png -singlefile -scale-to-x 400 -scale-to-y -1 $PDFPATH $FULLTEXTDIR/thumbnail/$BLOBPATH
+ pdftocairo -png -singlefile -scale-to-x 400 -scale-to-y -1 $PDFPATH $FULLTEXTDIR/thumbnail/$BLOBPATH || true
fi
diff --git a/bin/run_pipeline.sh b/bin/run_pipeline.sh
new file mode 100755
index 0000000..7ace3b3
--- /dev/null
+++ b/bin/run_pipeline.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+
+# invoke this script like:
+#
+# pipenv shell
+# CORDDATE=2020-04-03 ./bin/run_pipeline.sh
+
+set -e -u -o pipefail
+
+export TODAY="`TZ=UTC date --iso-8601=date`"
+
+echo "TODAY=$TODAY"
+echo "CORDDATE=$CORDDATE"
+
+# versbose:
+set -x
+
+mkdir -p metadata fulltext_web
+
+if [ ! -f metadata/cord19.$CORDDATE.csv ]; then
+ wget https://archive.org/download/s2-cord19-dataset/cord19.$CORDDATE.csv -O metadata/cord19.$CORDDATE.csv --no-clobber || true
+fi
+
+if [ ! -f metadata/cord19.$CORDDATE.json ]; then
+ ./covid19_tool.py parse-cord19 metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json
+fi
+
+if [ ! -f metadata/cord19.$CORDDATE.enrich.json ]; then
+ cat metadata/cord19.$CORDDATE.json | parallel -j10 --linebuffer --round-robin --pipe ./covid19_tool.py enrich-fatcat - | pv -l > metadata/cord19.$CORDDATE.enrich.json
+fi
+
+if [ ! -f metadata/cord19.$CORDDATE.missing.json ]; then
+ cat metadata/cord19.$CORDDATE.enrich.json | jq 'select(.release_id == null) | .cord19_paper' -c > metadata/cord19.$CORDDATE.missing.json
+fi
+
+if [ ! -f metadata/fatcat_hits.$TODAY.enrich.json ]; then
+ ./covid19_tool.py query-fatcat | pv -l > metadata/fatcat_hits.$TODAY.enrich.json
+fi
+
+if [ ! -f metadata/combined.$TODAY.enrich.json ]; then
+ cat metadata/fatcat_hits.$TODAY.enrich.json metadata/cord19.$CORDDATE.enrich.json | ./covid19_tool.py dedupe | pv -l > metadata/combined.$TODAY.enrich.json
+fi
+
+if [ ! -f fatcat_web_$TODAY.log ]; then
+ cat metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$TODAY.log
+fi
+
+if [ ! -f metadata/derivatives.$TODAY.stamp ]; then
+ ./bin/make_dir_derivatives.sh fulltext_web
+ touch metadata/derivatives.$TODAY.stamp
+fi
+
+if [ ! -f metadata/combined.$TODAY.fulltext.json ]; then
+ ./covid19_tool.py enrich-derivatives metadata/combined.$TODAY.enrich.json --base-dir fulltext_web/ | pv -l > metadata/combined.$TODAY.fulltext.json
+fi
+
+echo "## Fulltext Inclusion Counts"
+cat metadata/combined.$TODAY.fulltext.json | jq .fulltext_status -r | sort | uniq -c | sort -nr
+
+./covid19_tool.py transform-es metadata/combined.$TODAY.fulltext.json | pv -l | esbulk -verbose -size 1000 -id fatcat_ident -w 8 -index covid19_fatcat_fulltext -type release