inclusion notes and pipeline update

2 files changed, 70 insertions, 16 deletions

diff --git a/notes/pipeline_commands.md b/notes/pipeline_commands.md
index a9eddcd..ca3106c 100644
--- a/notes/pipeline_commands.md
+++ b/notes/pipeline_commands.md
@@ -6,31 +6,34 @@ Dependencies:
 
 Context:
 
-    export CORDDATE="2020-03-27"
+    export TODAY="`TZ=UTC date --iso-8601=date`"
+    export CORDDATE="2020-04-03"
 
-Fetch and transform metadata:
+Fetch and transform CORD-19 metadata:
 
     mkdir -p metadata fulltext_web
     wget https://archive.org/download/s2-cord19-dataset/cord19.$CORDDATE.csv -O metadata/cord19.$CORDDATE.csv metadata
-    ./bin/parse_cord19_csv.py metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json
+    ./covid19_tool.py parse-cord19 metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json
     cat metadata/cord19.$CORDDATE.json | parallel -j10 --linebuffer --round-robin --pipe ./covid19_tool.py enrich-fatcat - | pv -l > metadata/cord19.$CORDDATE.enrich.json
     cat metadata/cord19.$CORDDATE.enrich.json | jq 'select(.release_id == null) | .cord19_paper' -c > metadata/cord19.$CORDDATE.missing.json
 
-Download fulltext from wayback:
+Fetch fatcat query metadata:
+
+    ./covid19_tool.py query-fatcat | pv -l > metadata/fatcat_hits.$TODAY.enrich.json
 
-    cat metadata/cord19.$CORDDATE.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$CORDDATE.log
+Combine and de-dupe:
 
-    cut -f1 fatcat_web_$CORDDATE.log | sort | uniq -c | sort -nr
+    cat metadata/fatcat_hits.$TODAY.enrich.json metadata/cord19.$CORDDATE.enrich.json | ./covid19_tool.py dedupe | pv -l > metadata/combined.$TODAY.enrich.json
+
+Download fulltext from wayback:
 
-Extract text from PDFs:
+    cat metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$TODAY.log
 
-    ls fulltext_web/pdf/ | parallel mkdir -p fulltext_web/pdftotext/{}
-    fd -I .pdf fulltext_web/pdf/ | cut -c18-60 | parallel -j10 pdftotext fulltext_web/pdf/{}.pdf fulltext_web/pdftotext/{}.txt
+    cut -f1 fatcat_web_$TODAY.log | sort | uniq -c | sort -nr
 
-Create thumbnails:
+Update PDF derivative files (pdftotext and PNG thumbnail):
 
-    ls fulltext_web/pdf/ | parallel mkdir -p fulltext_web/thumbnail/{}
-    fd -I .pdf fulltext_web/pdf/ | cut -c18-60 | parallel -j10 pdftocairo -png -singlefile -scale-to-x 400 -scale-to-y -1 fulltext_web/pdf/{}.pdf fulltext_web/thumbnail/{}
+    ./bin/make_dir_derivatives.sh fulltext_web
 
 Fetch GROBID:
 
@@ -43,9 +46,9 @@ Convert GROBID XML to JSON:
 
 Create large derivatives file (including extracted fulltext):
 
-    ./covid19_tool.py enrich-derivatives metadata/cord19.$CORDDATE.enrich.json --base-dir fulltext_web/ | pv -l > metadata/cord19.$CORDDATE.fulltext.json
+    ./covid19_tool.py enrich-derivatives metadata/combined.$TODAY.enrich.json --base-dir fulltext_web/ | pv -l > metadata/combined.$TODAY.fulltext.json
 
-    cat metadata/cord19.$CORDDATE.fulltext.json | jq .fulltext_status -r | sort | uniq -c | sort -nr
+    cat metadata/combined.$TODAY.fulltext.json | jq .fulltext_status -r | sort | uniq -c | sort -nr
 
 
 ## ES Indices
@@ -56,7 +59,7 @@ Create fulltext index, transform to ES schema and index:
     http put :9200/covid19_fatcat_fulltext < schema/fulltext_schema.v00.json
 
     # in fatcat_covid19, pipenv shell
-    ./covid19_tool.py transform-es metadata/cord19.$CORDDATE.fulltext.json | pv -l | esbulk -verbose -size 1000 -id fatcat_ident -w 8 -index covid19_fatcat_fulltext -type release
+    ./covid19_tool.py transform-es metadata/combined.$TODAY.fulltext.json | pv -l | esbulk -verbose -size 1000 -id fatcat_ident -w 8 -index covid19_fatcat_fulltext -type release
 
 Create and index existing `fatcat_release` schema:
 
@@ -64,7 +67,7 @@ Create and index existing `fatcat_release` schema:
 
     # in fatcat python directory, pipenv shell
     export LC_ALL=C.UTF-8
-    cat /srv/fatcat_covid19/src/metadata/cord19.$CORDDATE.enrich.json | jq .fatcat_release -c | rg -v '^null$' | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 1000 -id ident -w 8 -index covid19_fatcat_release -type release
+    cat /srv/fatcat_covid19/src/metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | rg -v '^null$' | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 1000 -id ident -w 8 -index covid19_fatcat_release -type release
 
 ## GROBID Processing
 
diff --git a/notes/should_include.md b/notes/should_include.md
new file mode 100644
index 0000000..6033eb6
--- /dev/null
+++ b/notes/should_include.md
@@ -0,0 +1,51 @@
+
+## Queries
+
+    pandemic influenza
+    epidemic influenza
+    pandemic ventilator
+    SARS
+    sars-cov-2
+    covid-19
+
+## Should not include?
+
+Duplicate releases:
+
+- zenodo versions
+- figshare versions
+    eg "Coronavirus Research on Figshare" (12 versions)
+
+Remove anything researchgate? Quality is low. DOI prefix: 
+
+"TOF-SARS" => time of flight physics thing
+
+These should not end up in the corpus:
+
+    "Description of a new Norwegian star-fish"
+    by M. Sars
+    https://fatcat.wiki/release/ngp3qkqf4fccbdlxz2u4h4taoe
+
+## Specific Articles
+
+Expect these to end up in the corpus (they are not already):
+
+    "100 Years of Medical Countermeasures and Pandemic Influenza Preparedness"
+    https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6187768/
+
+
+## Hacks
+
+    10.2210/pdb4njl/pdb
+    no release_type
+    => dataset
+    => published
+
+    no release_type
+    title starts "figure"
+    => graphic/figure, skip it
+
+    journal: "Emerald Expert Briefings"
+    container_id:fnllqvywjbec5eumrbavqipfym
+    => skip it
+