reduce: set default batch size

1 files changed, 8 insertions, 6 deletions

diff --git a/skate/reduce.go b/skate/reduce.go
index b151926..8161955 100644
--- a/skate/reduce.go
+++ b/skate/reduce.go
@@ -31,6 +31,8 @@ import (
 	"github.com/segmentio/encoding/json"
 )
 
+const defaultBatchSize = 10000
+
 // groupLogf logs a message alongsize a serialized group for debugging.
 func groupLogf(g *zipkey.Group, s string, vs ...interface{}) {
 	log.Printf(s, vs...)
@@ -81,7 +83,7 @@ func ZippyExact(releases, refs io.Reader, matchResult MatchResult, w io.Writer)
 			}
 			return nil
 		}
-		batcher = zipkey.NewBatcherSize(grouper, 10000) // hard-code for now; on 24 cores 10K take up over 8G of RAM
+		batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize) // hard-code for now; on 24 cores 10K take up over 8G of RAM
 	)
 	defer batcher.Close()
 	zipper := zipkey.New(releases, refs, keyer, batcher.GroupFunc)
@@ -131,7 +133,7 @@ func ZippyExactReleases(olr, releases io.Reader, matchResult MatchResult, w io.W
 			}
 			return nil
 		}
-		batcher = zipkey.NewBatcherSize(grouper, 10000)
+		batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize)
 	)
 	defer batcher.Close()
 	zipper := zipkey.New(olr, releases, keyer, batcher.GroupFunc)
@@ -174,7 +176,7 @@ func ZippyExactWiki(releases, wiki io.Reader, mr MatchResult, w io.Writer) error
 			}
 			return nil
 		}
-		batcher = zipkey.NewBatcherSize(grouper, 10000)
+		batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize)
 	)
 	defer batcher.Close()
 	zipper := zipkey.New(releases, wiki, keyer, batcher.GroupFunc)
@@ -218,7 +220,7 @@ func ZippyVerifyRefs(releases, refs io.Reader, w io.Writer) error {
 			}
 			return nil
 		}
-		batcher = zipkey.NewBatcherSize(grouper, 10000)
+		batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize)
 	)
 	defer batcher.Close()
 	zipper := zipkey.New(releases, refs, keyer, batcher.GroupFunc)
@@ -314,7 +316,7 @@ func ZippyVerifyRefsOpenLibrary(olr, refs io.Reader, w io.Writer) error {
 			}
 			return nil
 		}
-		batcher = zipkey.NewBatcherSize(grouper, 10000)
+		batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize)
 	)
 	defer batcher.Close()
 	zipper := zipkey.New(olr, refs, keyer, batcher.GroupFunc)
@@ -374,7 +376,7 @@ func ZippyBrefAugment(bref, raw io.Reader, w io.Writer) error {
 			}
 			return nil
 		}
-		batcher = zipkey.NewBatcherSize(grouper, 10000)
+		batcher = zipkey.NewBatcherSize(grouper, defaultBatchSize)
 	)
 	defer batcher.Close()
 	zipper := zipkey.New(bref, raw, keyer, batcher.GroupFunc)