From d2c95fb7a98e051639ef875c1d9aa4ba4b794509 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Wed, 13 May 2020 14:09:43 -0700
Subject: improve pipeline commands

---
 bin/run_pipeline.sh        | 25 +++++++++++++++++--------
 notes/pipeline_commands.md |  6 ++++++
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/bin/run_pipeline.sh b/bin/run_pipeline.sh
index 7ace3b3..5c1f4de 100755
--- a/bin/run_pipeline.sh
+++ b/bin/run_pipeline.sh
@@ -19,31 +19,39 @@ set -x
 mkdir -p metadata fulltext_web
 
 if [ ! -f metadata/cord19.$CORDDATE.csv ]; then
-    wget https://archive.org/download/s2-cord19-dataset/cord19.$CORDDATE.csv -O metadata/cord19.$CORDDATE.csv --no-clobber || true
+    rm metadata/cord19.$CORDDATE.csv.wip || true
+    wget https://archive.org/download/s2-cord19-dataset/cord19.$CORDDATE.csv -O metadata/cord19.$CORDDATE.csv.wip --no-clobber
+    mv metadata/cord19.$CORDDATE.csv.wip metadata/cord19.$CORDDATE.csv
 fi
 
 if [ ! -f metadata/cord19.$CORDDATE.json ]; then
-    ./covid19_tool.py parse-cord19 metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json
+    ./covid19_tool.py parse-cord19 metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json.wip
+    mv metadata/cord19.$CORDDATE.json.wip metadata/cord19.$CORDDATE.json
 fi
 
 if [ ! -f metadata/cord19.$CORDDATE.enrich.json ]; then
-    cat metadata/cord19.$CORDDATE.json | parallel -j10 --linebuffer --round-robin --pipe ./covid19_tool.py enrich-fatcat - | pv -l > metadata/cord19.$CORDDATE.enrich.json
+    cat metadata/cord19.$CORDDATE.json | parallel -j10 --linebuffer --round-robin --pipe ./covid19_tool.py enrich-fatcat - | pv -l > metadata/cord19.$CORDDATE.enrich.json.wip
+    mv metadata/cord19.$CORDDATE.enrich.json.wip metadata/cord19.$CORDDATE.enrich.json
 fi
 
 if [ ! -f metadata/cord19.$CORDDATE.missing.json ]; then
-    cat metadata/cord19.$CORDDATE.enrich.json | jq 'select(.release_id == null) | .cord19_paper' -c > metadata/cord19.$CORDDATE.missing.json
+    cat metadata/cord19.$CORDDATE.enrich.json | jq 'select(.release_id == null) | .cord19_paper' -c > metadata/cord19.$CORDDATE.missing.json.wip
+    mv metadata/cord19.$CORDDATE.missing.json.wip metadata/cord19.$CORDDATE.missing.json
 fi
 
 if [ ! -f metadata/fatcat_hits.$TODAY.enrich.json ]; then
-    ./covid19_tool.py query-fatcat | pv -l > metadata/fatcat_hits.$TODAY.enrich.json
+    ./covid19_tool.py query-fatcat | pv -l > metadata/fatcat_hits.$TODAY.enrich.json.wip
+    mv metadata/fatcat_hits.$TODAY.enrich.json.wip metadata/fatcat_hits.$TODAY.enrich.json
 fi
 
 if [ ! -f metadata/combined.$TODAY.enrich.json ]; then
-    cat metadata/fatcat_hits.$TODAY.enrich.json metadata/cord19.$CORDDATE.enrich.json | ./covid19_tool.py dedupe | pv -l > metadata/combined.$TODAY.enrich.json
+    cat metadata/fatcat_hits.$TODAY.enrich.json metadata/cord19.$CORDDATE.enrich.json | ./covid19_tool.py dedupe | pv -l > metadata/combined.$TODAY.enrich.json.wip
+    mv metadata/combined.$TODAY.enrich.json.wip metadata/combined.$TODAY.enrich.json
 fi
 
 if [ ! -f fatcat_web_$TODAY.log ]; then
-    cat metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$TODAY.log
+    cat metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$TODAY.log.wip
+    mv fatcat_web_$TODAY.log.wip fatcat_web_$TODAY.log
 fi
 
 if [ ! -f metadata/derivatives.$TODAY.stamp ]; then
@@ -52,7 +60,8 @@ if [ ! -f metadata/derivatives.$TODAY.stamp ]; then
 fi
 
 if [ ! -f metadata/combined.$TODAY.fulltext.json ]; then
-    ./covid19_tool.py enrich-derivatives metadata/combined.$TODAY.enrich.json --base-dir fulltext_web/ | pv -l > metadata/combined.$TODAY.fulltext.json
+    ./covid19_tool.py enrich-derivatives metadata/combined.$TODAY.enrich.json --base-dir fulltext_web/ | pv -l > metadata/combined.$TODAY.fulltext.json.wip
+    mv metadata/combined.$TODAY.fulltext.json.wip metadata/combined.$TODAY.fulltext.json
 fi
 
 echo "## Fulltext Inclusion Counts"
diff --git a/notes/pipeline_commands.md b/notes/pipeline_commands.md
index ca3106c..94d7480 100644
--- a/notes/pipeline_commands.md
+++ b/notes/pipeline_commands.md
@@ -67,8 +67,14 @@ Create and index existing `fatcat_release` schema:
 
     # in fatcat python directory, pipenv shell
     export LC_ALL=C.UTF-8
+    export TODAY="`TZ=UTC date --iso-8601=date`"
     cat /srv/fatcat_covid19/src/metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | rg -v '^null$' | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 1000 -id ident -w 8 -index covid19_fatcat_release -type release
 
+Attempt crawl/ingest of missing papers:
+
+    ./fatcat_ingest.py --env prod --enqueue-kafka --elasticsearch-index covid19_fatcat_release --fatcat-api-url https://api.fatcat.wiki/v0 --kafka-hosts wbgrp-svc263.us.archive.org --allow-non-oa query "pmcid:*"
+    ./fatcat_ingest.py --env prod --enqueue-kafka --elasticsearch-index covid19_fatcat_release --fatcat-api-url https://api.fatcat.wiki/v0 --kafka-hosts wbgrp-svc263.us.archive.org --allow-non-oa query "-_exists_:pmcid doi:*"
+
 ## GROBID Processing
 
     zip -r fulltext_web.zip fulltext_web
-- 
cgit v1.2.3