diff options
Diffstat (limited to 'skate/cmd/skate-bref-id')
-rw-r--r-- | skate/cmd/skate-bref-id/main.go | 41 |
1 files changed, 0 insertions, 41 deletions
diff --git a/skate/cmd/skate-bref-id/main.go b/skate/cmd/skate-bref-id/main.go deleted file mode 100644 index ba52219..0000000 --- a/skate/cmd/skate-bref-id/main.go +++ /dev/null @@ -1,41 +0,0 @@ -// skate-bref-id is a temporary helper to generate an id for a bref doc. -package main - -import ( - "flag" - "fmt" - "log" - "os" - "runtime" - "time" - - "git.archive.org/martin/cgraph/skate" - "git.archive.org/martin/cgraph/skate/parallel" - "github.com/segmentio/encoding/json" -) - -var ( - numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") - batchSize = flag.Int("b", 100000, "batch size") - - newlineB = []byte("\n") -) - -func main() { - pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) { - var bref skate.BiblioRef - if err := json.Unmarshal(p, &bref); err != nil { - return nil, err - } - bref.Key = fmt.Sprintf("%s_%d", bref.SourceReleaseIdent, bref.RefIndex) - bref.IndexedTs = time.Now().Format(time.RFC3339) - b, err := json.Marshal(bref) - b = append(b, newlineB...) - return b, err - }) - pp.NumWorkers = *numWorkers - pp.BatchSize = *batchSize - if err := pp.Run(); err != nil { - log.Fatal(err) - } -} |