diff options
author | Martin Czygan <martin.czygan@gmail.com> | 2021-03-21 01:17:38 +0100 |
---|---|---|
committer | Martin Czygan <martin.czygan@gmail.com> | 2021-03-21 01:17:38 +0100 |
commit | 09a7e8c9d013f13a1aa1ef4e9b7f397647b79967 (patch) | |
tree | 122b474e27afbc66cba1182e983ef5c8555ed12f /skate/cmd/skate-bref-id/main.go | |
parent | a7e0cf191ebf8fb499e0ab9a3b6cae45727f1286 (diff) | |
download | refcat-09a7e8c9d013f13a1aa1ef4e9b7f397647b79967.tar.gz refcat-09a7e8c9d013f13a1aa1ef4e9b7f397647b79967.zip |
initial import of skate
Diffstat (limited to 'skate/cmd/skate-bref-id/main.go')
-rw-r--r-- | skate/cmd/skate-bref-id/main.go | 42 |
1 files changed, 42 insertions, 0 deletions
diff --git a/skate/cmd/skate-bref-id/main.go b/skate/cmd/skate-bref-id/main.go new file mode 100644 index 0000000..ca3d7c4 --- /dev/null +++ b/skate/cmd/skate-bref-id/main.go @@ -0,0 +1,42 @@ +// skate-bref-id is a temporary helper to generate an id for a bref doc. +package main + +import ( + "flag" + "fmt" + "log" + "os" + "runtime" + "time" + + jsoniter "github.com/json-iterator/go" + "git.archive.org/martin/cgraph/skate" + "git.archive.org/martin/cgraph/skate/parallel" +) + +var ( + numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") + batchSize = flag.Int("b", 100000, "batch size") + json = jsoniter.ConfigCompatibleWithStandardLibrary + + newlineB = []byte("\n") +) + +func main() { + pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) { + var bref skate.BiblioRef + if err := json.Unmarshal(p, &bref); err != nil { + return nil, err + } + bref.Key = fmt.Sprintf("%s_%d", bref.SourceReleaseIdent, bref.RefIndex) + bref.UpdateTs = time.Now().Unix() + b, err := json.Marshal(bref) + b = append(b, newlineB...) + return b, err + }) + pp.NumWorkers = *numWorkers + pp.BatchSize = *batchSize + if err := pp.Run(); err != nil { + log.Fatal(err) + } +} |