diff options
| -rw-r--r-- | notes/pipeline_commands.md | 35 | ||||
| -rw-r--r-- | notes/should_include.md | 51 |
2 files changed, 70 insertions, 16 deletions
diff --git a/notes/pipeline_commands.md b/notes/pipeline_commands.md index a9eddcd..ca3106c 100644 --- a/notes/pipeline_commands.md +++ b/notes/pipeline_commands.md @@ -6,31 +6,34 @@ Dependencies: Context: - export CORDDATE="2020-03-27" + export TODAY="`TZ=UTC date --iso-8601=date`" + export CORDDATE="2020-04-03" -Fetch and transform metadata: +Fetch and transform CORD-19 metadata: mkdir -p metadata fulltext_web wget https://archive.org/download/s2-cord19-dataset/cord19.$CORDDATE.csv -O metadata/cord19.$CORDDATE.csv metadata - ./bin/parse_cord19_csv.py metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json + ./covid19_tool.py parse-cord19 metadata/cord19.$CORDDATE.csv > metadata/cord19.$CORDDATE.json cat metadata/cord19.$CORDDATE.json | parallel -j10 --linebuffer --round-robin --pipe ./covid19_tool.py enrich-fatcat - | pv -l > metadata/cord19.$CORDDATE.enrich.json cat metadata/cord19.$CORDDATE.enrich.json | jq 'select(.release_id == null) | .cord19_paper' -c > metadata/cord19.$CORDDATE.missing.json -Download fulltext from wayback: +Fetch fatcat query metadata: + + ./covid19_tool.py query-fatcat | pv -l > metadata/fatcat_hits.$TODAY.enrich.json - cat metadata/cord19.$CORDDATE.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$CORDDATE.log +Combine and de-dupe: - cut -f1 fatcat_web_$CORDDATE.log | sort | uniq -c | sort -nr + cat metadata/fatcat_hits.$TODAY.enrich.json metadata/cord19.$CORDDATE.enrich.json | ./covid19_tool.py dedupe | pv -l > metadata/combined.$TODAY.enrich.json + +Download fulltext from wayback: -Extract text from PDFs: + cat metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | parallel -j20 --linebuffer --round-robin --pipe ./bin/deliver_file2disk.py --disk-dir fulltext_web - | pv -l > fatcat_web_$TODAY.log - ls fulltext_web/pdf/ | parallel mkdir -p fulltext_web/pdftotext/{} - fd -I .pdf fulltext_web/pdf/ | cut -c18-60 | parallel -j10 pdftotext fulltext_web/pdf/{}.pdf fulltext_web/pdftotext/{}.txt + cut -f1 fatcat_web_$TODAY.log | sort | uniq -c | sort -nr -Create thumbnails: +Update PDF derivative files (pdftotext and PNG thumbnail): - ls fulltext_web/pdf/ | parallel mkdir -p fulltext_web/thumbnail/{} - fd -I .pdf fulltext_web/pdf/ | cut -c18-60 | parallel -j10 pdftocairo -png -singlefile -scale-to-x 400 -scale-to-y -1 fulltext_web/pdf/{}.pdf fulltext_web/thumbnail/{} + ./bin/make_dir_derivatives.sh fulltext_web Fetch GROBID: @@ -43,9 +46,9 @@ Convert GROBID XML to JSON: Create large derivatives file (including extracted fulltext): - ./covid19_tool.py enrich-derivatives metadata/cord19.$CORDDATE.enrich.json --base-dir fulltext_web/ | pv -l > metadata/cord19.$CORDDATE.fulltext.json + ./covid19_tool.py enrich-derivatives metadata/combined.$TODAY.enrich.json --base-dir fulltext_web/ | pv -l > metadata/combined.$TODAY.fulltext.json - cat metadata/cord19.$CORDDATE.fulltext.json | jq .fulltext_status -r | sort | uniq -c | sort -nr + cat metadata/combined.$TODAY.fulltext.json | jq .fulltext_status -r | sort | uniq -c | sort -nr ## ES Indices @@ -56,7 +59,7 @@ Create fulltext index, transform to ES schema and index: http put :9200/covid19_fatcat_fulltext < schema/fulltext_schema.v00.json # in fatcat_covid19, pipenv shell - ./covid19_tool.py transform-es metadata/cord19.$CORDDATE.fulltext.json | pv -l | esbulk -verbose -size 1000 -id fatcat_ident -w 8 -index covid19_fatcat_fulltext -type release + ./covid19_tool.py transform-es metadata/combined.$TODAY.fulltext.json | pv -l | esbulk -verbose -size 1000 -id fatcat_ident -w 8 -index covid19_fatcat_fulltext -type release Create and index existing `fatcat_release` schema: @@ -64,7 +67,7 @@ Create and index existing `fatcat_release` schema: # in fatcat python directory, pipenv shell export LC_ALL=C.UTF-8 - cat /srv/fatcat_covid19/src/metadata/cord19.$CORDDATE.enrich.json | jq .fatcat_release -c | rg -v '^null$' | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 1000 -id ident -w 8 -index covid19_fatcat_release -type release + cat /srv/fatcat_covid19/src/metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | rg -v '^null$' | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 1000 -id ident -w 8 -index covid19_fatcat_release -type release ## GROBID Processing diff --git a/notes/should_include.md b/notes/should_include.md new file mode 100644 index 0000000..6033eb6 --- /dev/null +++ b/notes/should_include.md @@ -0,0 +1,51 @@ + +## Queries + + pandemic influenza + epidemic influenza + pandemic ventilator + SARS + sars-cov-2 + covid-19 + +## Should not include? + +Duplicate releases: + +- zenodo versions +- figshare versions + eg "Coronavirus Research on Figshare" (12 versions) + +Remove anything researchgate? Quality is low. DOI prefix: + +"TOF-SARS" => time of flight physics thing + +These should not end up in the corpus: + + "Description of a new Norwegian star-fish" + by M. Sars + https://fatcat.wiki/release/ngp3qkqf4fccbdlxz2u4h4taoe + +## Specific Articles + +Expect these to end up in the corpus (they are not already): + + "100 Years of Medical Countermeasures and Pandemic Influenza Preparedness" + https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6187768/ + + +## Hacks + + 10.2210/pdb4njl/pdb + no release_type + => dataset + => published + + no release_type + title starts "figure" + => graphic/figure, skip it + + journal: "Emerald Expert Briefings" + container_id:fnllqvywjbec5eumrbavqipfym + => skip it + |