diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-05-13 14:09:43 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-05-13 14:09:43 -0700 |
| commit | d2c95fb7a98e051639ef875c1d9aa4ba4b794509 (patch) | |
| tree | 8f8dec94ca329415a0b7d2e923865ee0d6cc210d /notes | |
| parent | 959544196679de53c379d8d71feff20c59cd40f2 (diff) | |
| download | fatcat-covid19-d2c95fb7a98e051639ef875c1d9aa4ba4b794509.tar.gz fatcat-covid19-d2c95fb7a98e051639ef875c1d9aa4ba4b794509.zip | |
improve pipeline commands
Diffstat (limited to 'notes')
| -rw-r--r-- | notes/pipeline_commands.md | 6 |
1 files changed, 6 insertions, 0 deletions
diff --git a/notes/pipeline_commands.md b/notes/pipeline_commands.md index ca3106c..94d7480 100644 --- a/notes/pipeline_commands.md +++ b/notes/pipeline_commands.md @@ -67,8 +67,14 @@ Create and index existing `fatcat_release` schema: # in fatcat python directory, pipenv shell export LC_ALL=C.UTF-8 + export TODAY="`TZ=UTC date --iso-8601=date`" cat /srv/fatcat_covid19/src/metadata/combined.$TODAY.enrich.json | jq .fatcat_release -c | rg -v '^null$' | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 1000 -id ident -w 8 -index covid19_fatcat_release -type release +Attempt crawl/ingest of missing papers: + + ./fatcat_ingest.py --env prod --enqueue-kafka --elasticsearch-index covid19_fatcat_release --fatcat-api-url https://api.fatcat.wiki/v0 --kafka-hosts wbgrp-svc263.us.archive.org --allow-non-oa query "pmcid:*" + ./fatcat_ingest.py --env prod --enqueue-kafka --elasticsearch-index covid19_fatcat_release --fatcat-api-url https://api.fatcat.wiki/v0 --kafka-hosts wbgrp-svc263.us.archive.org --allow-non-oa query "-_exists_:pmcid doi:*" + ## GROBID Processing zip -r fulltext_web.zip fulltext_web |