aboutsummaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-05-13 14:09:43 -0700
committerBryan Newbold <bnewbold@archive.org>2020-05-13 14:09:43 -0700
commitd2c95fb7a98e051639ef875c1d9aa4ba4b794509 (patch)
tree8f8dec94ca329415a0b7d2e923865ee0d6cc210d /bin
parent959544196679de53c379d8d71feff20c59cd40f2 (diff)
downloadfatcat-covid19-d2c95fb7a98e051639ef875c1d9aa4ba4b794509.tar.gz
fatcat-covid19-d2c95fb7a98e051639ef875c1d9aa4ba4b794509.zip
improve pipeline commands
Diffstat (limited to 'bin')
-rwxr-xr-xbin/run_pipeline.sh25
1 files changed, 17 insertions, 8 deletions
diff --git a/bin/run_pipeline.sh b/bin/run_pipeline.sh
index 7ace3b3..5c1f4de 100755
--- a/bin/run_pipeline.sh
+++ b/bin/run_pipeline.sh
@@ -19,31 +19,39 @@ set -x
mkdir -p metadata fulltext_web
if [ ! -f metadata/cord19.$CORDDATE.csv ]; then
- wget https://archive.org/download/s2-cord19-dataset/cord19.$CORDDATE.csv -O metadata/cord19.$CORDDATE.csv --no-clobber || true
+ rm metadata/cord19.$CORDDATE.csv.wip || true
+ wget https://archive.org/download/s2-cord19-dataset/cord19.$CORDDATE.csv -O metadata/cord19.$CORDDATE.csv.wip --no-clobber
+ mv metadata/cord19.$CORDDATE.csv.wip metadata/cord19.$CORDDATE.csv
fi
if [ ! -f metadata/cord19.$CORDDATE.json ]; then
- ./covid19_tool.py parse-cord19 metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json
+ ./covid19_tool.py parse-cord19 metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json.wip
+ mv metadata/cord19.$CORDDATE.json.wip metadata/cord19.$CORDDATE.json
fi
if [ ! -f metadata/cord19.$CORDDATE.enrich.json ]; then
- cat metadata/cord19.$CORDDATE.json | parallel -j10 --linebuffer --round-robin --pipe ./covid19_tool.py enrich-fatcat - | pv -l > metadata/cord19.$CORDDATE.enrich.json
+ cat metadata/cord19.$CORDDATE.json | parallel -j10 --linebuffer --round-robin --pipe ./covid19_tool.py enrich-fatcat - | pv -l > metadata/cord19.$CORDDATE.enrich.json.wip
+ mv metadata/cord19.$CORDDATE.enrich.json.wip metadata/cord19.$CORDDATE.enrich.json
fi
if [ ! -f metadata/cord19.$CORDDATE.missing.json ]; then
- cat metadata/cord19.$CORDDATE.enrich.json | jq 'select(.release_id == null) | .cord19_paper' -c > metadata/cord19.$CORDDATE.missing.json
+ cat metadata/cord19.$CORDDATE.enrich.json | jq 'select(.release_id == null) | .cord19_paper' -c > metadata/cord19.$CORDDATE.missing.json.wip
+ mv metadata/cord19.$CORDDATE.missing.json.wip metadata/cord19.$CORDDATE.missing.json
fi
if [ ! -f metadata/fatcat_hits.$TODAY.enrich.json ]; then
- ./covid19_tool.py query-fatcat | pv -l > metadata/fatcat_hits.$TODAY.enrich.json
+ ./covid19_tool.py query-fatcat | pv -l > metadata/fatcat_hits.$TODAY.enrich.json.wip
+ mv metadata/fatcat_hits.$TODAY.enrich.json.wip metadata/fatcat_hits.$TODAY.enrich.json
fi
if [ ! -f metadata/combined.$TODAY.enrich.json ]; then
- cat metadata/fatcat_hits.$TODAY.enrich.json metadata/cord19.$CORDDATE.enrich.json | ./covid19_tool.py dedupe | pv -l > metadata/combined.$TODAY.enrich.json
+ cat metadata/fatcat_hits.$TODAY.enrich.json metadata/cord19.$CORDDATE.enrich.json | ./covid19_tool.py dedupe | pv -l > metadata/combined.$TODAY.enrich.json.wip
+ mv metadata/combined.$TODAY.enrich.json.wip metadata/combined.$TODAY.enrich.json
fi
if [ ! -f fatcat_web_$TODAY.log ]; then
- cat metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$TODAY.log
+ cat metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$TODAY.log.wip
+ mv fatcat_web_$TODAY.log.wip fatcat_web_$TODAY.log
fi
if [ ! -f metadata/derivatives.$TODAY.stamp ]; then
@@ -52,7 +60,8 @@ if [ ! -f metadata/derivatives.$TODAY.stamp ]; then
fi
if [ ! -f metadata/combined.$TODAY.fulltext.json ]; then
- ./covid19_tool.py enrich-derivatives metadata/combined.$TODAY.enrich.json --base-dir fulltext_web/ | pv -l > metadata/combined.$TODAY.fulltext.json
+ ./covid19_tool.py enrich-derivatives metadata/combined.$TODAY.enrich.json --base-dir fulltext_web/ | pv -l > metadata/combined.$TODAY.fulltext.json.wip
+ mv metadata/combined.$TODAY.fulltext.json.wip metadata/combined.$TODAY.fulltext.json
fi
echo "## Fulltext Inclusion Counts"