From 25d39d27118d2b1d1e2f4e02e7988a17fee79031 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Thu, 8 Jul 2021 18:26:51 +0200 Subject: reduce: set default batch size --- skate/reduce.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/skate/reduce.go b/skate/reduce.go index b151926..8161955 100644 --- a/skate/reduce.go +++ b/skate/reduce.go @@ -31,6 +31,8 @@ import ( "github.com/segmentio/encoding/json" ) +const defaultBatchSize = 10000 + // groupLogf logs a message alongsize a serialized group for debugging. func groupLogf(g *zipkey.Group, s string, vs ...interface{}) { log.Printf(s, vs...) @@ -81,7 +83,7 @@ func ZippyExact(releases, refs io.Reader, matchResult MatchResult, w io.Writer) } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) // hard-code for now; on 24 cores 10K take up over 8G of RAM + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) // hard-code for now; on 24 cores 10K take up over 8G of RAM ) defer batcher.Close() zipper := zipkey.New(releases, refs, keyer, batcher.GroupFunc) @@ -131,7 +133,7 @@ func ZippyExactReleases(olr, releases io.Reader, matchResult MatchResult, w io.W } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(olr, releases, keyer, batcher.GroupFunc) @@ -174,7 +176,7 @@ func ZippyExactWiki(releases, wiki io.Reader, mr MatchResult, w io.Writer) error } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(releases, wiki, keyer, batcher.GroupFunc) @@ -218,7 +220,7 @@ func ZippyVerifyRefs(releases, refs io.Reader, w io.Writer) error { } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(releases, refs, keyer, batcher.GroupFunc) @@ -314,7 +316,7 @@ func ZippyVerifyRefsOpenLibrary(olr, refs io.Reader, w io.Writer) error { } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(olr, refs, keyer, batcher.GroupFunc) @@ -374,7 +376,7 @@ func ZippyBrefAugment(bref, raw io.Reader, w io.Writer) error { } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(bref, raw, keyer, batcher.GroupFunc) -- cgit v1.2.3