diff options
| -rw-r--r-- | skate/reduce.go | 14 |
1 files changed, 8 insertions, 6 deletions
diff --git a/skate/reduce.go b/skate/reduce.go index b151926..8161955 100644 --- a/skate/reduce.go +++ b/skate/reduce.go @@ -31,6 +31,8 @@ import ( "github.com/segmentio/encoding/json" ) +const defaultBatchSize = 10000 + // groupLogf logs a message alongsize a serialized group for debugging. func groupLogf(g *zipkey.Group, s string, vs ...interface{}) { log.Printf(s, vs...) @@ -81,7 +83,7 @@ func ZippyExact(releases, refs io.Reader, matchResult MatchResult, w io.Writer) } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) // hard-code for now; on 24 cores 10K take up over 8G of RAM + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) // hard-code for now; on 24 cores 10K take up over 8G of RAM ) defer batcher.Close() zipper := zipkey.New(releases, refs, keyer, batcher.GroupFunc) @@ -131,7 +133,7 @@ func ZippyExactReleases(olr, releases io.Reader, matchResult MatchResult, w io.W } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(olr, releases, keyer, batcher.GroupFunc) @@ -174,7 +176,7 @@ func ZippyExactWiki(releases, wiki io.Reader, mr MatchResult, w io.Writer) error } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(releases, wiki, keyer, batcher.GroupFunc) @@ -218,7 +220,7 @@ func ZippyVerifyRefs(releases, refs io.Reader, w io.Writer) error { } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(releases, refs, keyer, batcher.GroupFunc) @@ -314,7 +316,7 @@ func ZippyVerifyRefsOpenLibrary(olr, refs io.Reader, w io.Writer) error { } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(olr, refs, keyer, batcher.GroupFunc) @@ -374,7 +376,7 @@ func ZippyBrefAugment(bref, raw io.Reader, w io.Writer) error { } return nil } - batcher = zipkey.NewBatcherSize(grouper, 10000) + batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) ) defer batcher.Close() zipper := zipkey.New(bref, raw, keyer, batcher.GroupFunc) |