diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2020-02-26 22:05:43 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2020-02-26 22:05:43 -0800 |
| commit | ae50ee2274031ddc178fa4a10b59280e8440a24c (patch) | |
| tree | 5a17d5d5f875ab6ff770c955b25626c6f36d16a8 /extra/elasticsearch/README.md | |
| parent | 1556cdd7f0f5f4bc4fe5ccc9764c1598c852bb9b (diff) | |
| parent | 81e0784813500a39955c20278140e25d7940d9c6 (diff) | |
| download | fatcat-ae50ee2274031ddc178fa4a10b59280e8440a24c.tar.gz fatcat-ae50ee2274031ddc178fa4a10b59280e8440a24c.zip | |
Merge branch 'bnewbold-elastic-v03b'
Diffstat (limited to 'extra/elasticsearch/README.md')
| -rw-r--r-- | extra/elasticsearch/README.md | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/extra/elasticsearch/README.md b/extra/elasticsearch/README.md index 3a48a178..df4cb918 100644 --- a/extra/elasticsearch/README.md +++ b/extra/elasticsearch/README.md @@ -40,9 +40,11 @@ Drop and rebuild the schema: http delete :9200/fatcat_release http delete :9200/fatcat_container + http delete :9200/fatcat_file http delete :9200/fatcat_changelog http put :9200/fatcat_release < release_schema.json http put :9200/fatcat_container < container_schema.json + http put :9200/fatcat_file < file_schema.json http put :9200/fatcat_changelog < changelog_schema.json Put a single object (good for debugging): @@ -57,8 +59,9 @@ Bulk insert from a file on disk: Or, in a bulk production live-stream conversion: export LC_ALL=C.UTF-8 - time zcat /srv/fatcat/snapshots/release_export_expanded.json.gz | pv -l | parallel -j20 --linebuffer --round-robin --pipe ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_release -type release - time zcat /srv/fatcat/snapshots/container_export.json.gz | pv -l | ./fatcat_transform.py elasticsearch-containers - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_container -type container + time zcat /srv/fatcat/snapshots/release_export_expanded.json.gz | pv -l | parallel -j20 --linebuffer --round-robin --pipe ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 2000 -id ident -w 8 -index fatcat_release -type release + time zcat /srv/fatcat/snapshots/container_export.json.gz | pv -l | ./fatcat_transform.py elasticsearch-containers - - | esbulk -verbose -size 2000 -id ident -w 8 -index fatcat_container -type container + time zcat /srv/fatcat/snapshots/file_export.json.gz | pv -l | parallel -j20 --linebuffer --round-robin --pipe ./fatcat_transform.py elasticsearch-files - - | esbulk -verbose -size 2000 -id ident -w 8 -index fatcat_file -type file ## Index Aliases |