summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-04-30 15:44:26 -0700
committerBryan Newbold <bnewbold@robocracy.org>2019-04-30 15:44:26 -0700
commit81fd879bf00e452bfd1e8da95164c2a2f431a0c3 (patch)
tree64720e67d72d95bda1a8e94de93f08b6124bf5f3
parent15b4c8a29e9796c059f80f8dc3a884e1d1d4d2f1 (diff)
downloadfatcat-81fd879bf00e452bfd1e8da95164c2a2f431a0c3.tar.gz
fatcat-81fd879bf00e452bfd1e8da95164c2a2f431a0c3.zip
faster elasticsearch imports
-rw-r--r--extra/elasticsearch/README.md2
1 files changed, 1 insertions, 1 deletions
diff --git a/extra/elasticsearch/README.md b/extra/elasticsearch/README.md
index 70da77e4..15c00b4c 100644
--- a/extra/elasticsearch/README.md
+++ b/extra/elasticsearch/README.md
@@ -57,7 +57,7 @@ Bulk insert from a file on disk:
Or, in a bulk production live-stream conversion:
export LC_ALL=C.UTF-8
- time zcat /srv/fatcat/snapshots/release_export_expanded.json.gz | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_release -type release
+ time zcat /srv/fatcat/snapshots/release_export_expanded.json.gz | pv -l | parallel -j20 --linebuffer --round-robin --pipe ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_release -type release
time zcat /srv/fatcat/snapshots/container_export.json.gz | pv -l | ./fatcat_transform.py elasticsearch-containers - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_container -type container
## Full-Text Querying