diff options
Diffstat (limited to 'skate/cmd/skate-bref-id/main.go')
-rw-r--r-- | skate/cmd/skate-bref-id/main.go | 42 |
1 files changed, 42 insertions, 0 deletions
diff --git a/skate/cmd/skate-bref-id/main.go b/skate/cmd/skate-bref-id/main.go new file mode 100644 index 0000000..ca3d7c4 --- /dev/null +++ b/skate/cmd/skate-bref-id/main.go @@ -0,0 +1,42 @@ +// skate-bref-id is a temporary helper to generate an id for a bref doc. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "runtime" + "time" + + jsoniter "github.com/json-iterator/go" + "git.archive.org/martin/cgraph/skate" + "git.archive.org/martin/cgraph/skate/parallel" +) + +var ( + numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") + batchSize = flag.Int("b", 100000, "batch size") + json = jsoniter.ConfigCompatibleWithStandardLibrary + + newlineB = []byte("\n") +) + +func main() { + pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) { + var bref skate.BiblioRef + if err := json.Unmarshal(p, &bref); err != nil { + return nil, err + } + bref.Key = fmt.Sprintf("%s_%d", bref.SourceReleaseIdent, bref.RefIndex) + bref.UpdateTs = time.Now().Unix() + b, err := json.Marshal(bref) + b = append(b, newlineB...) + return b, err + }) + pp.NumWorkers = *numWorkers + pp.BatchSize = *batchSize + if err := pp.Run(); err != nil { + log.Fatal(err) + } +} |