From 81fd879bf00e452bfd1e8da95164c2a2f431a0c3 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 30 Apr 2019 15:44:26 -0700 Subject: faster elasticsearch imports --- extra/elasticsearch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/elasticsearch/README.md b/extra/elasticsearch/README.md index 70da77e4..15c00b4c 100644 --- a/extra/elasticsearch/README.md +++ b/extra/elasticsearch/README.md @@ -57,7 +57,7 @@ Bulk insert from a file on disk: Or, in a bulk production live-stream conversion: export LC_ALL=C.UTF-8 - time zcat /srv/fatcat/snapshots/release_export_expanded.json.gz | pv -l | ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_release -type release + time zcat /srv/fatcat/snapshots/release_export_expanded.json.gz | pv -l | parallel -j20 --linebuffer --round-robin --pipe ./fatcat_transform.py elasticsearch-releases - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_release -type release time zcat /srv/fatcat/snapshots/container_export.json.gz | pv -l | ./fatcat_transform.py elasticsearch-containers - - | esbulk -verbose -size 20000 -id ident -w 8 -index fatcat_container -type container ## Full-Text Querying -- cgit v1.2.3