From e8ed1ff2a60b694b242669a50c5a37346f3b6d79 Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Tue, 30 Mar 2021 01:35:58 +0200 Subject: stub wikipedia converter --- skate/cmd/skate-biblioref-from-wikipedia/main.go | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 skate/cmd/skate-biblioref-from-wikipedia/main.go (limited to 'skate/cmd') diff --git a/skate/cmd/skate-biblioref-from-wikipedia/main.go b/skate/cmd/skate-biblioref-from-wikipedia/main.go new file mode 100644 index 0000000..552b625 --- /dev/null +++ b/skate/cmd/skate-biblioref-from-wikipedia/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "flag" + "log" + "os" + "runtime" + + "git.archive.org/martin/cgraph/skate" + "git.archive.org/martin/cgraph/skate/parallel" + jsoniter "github.com/json-iterator/go" +) + +var ( + numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") + batchSize = flag.Int("b", 100000, "batch size") + json = jsoniter.ConfigCompatibleWithStandardLibrary + bytesNewline = []byte("\n") +) + +func main() { + flag.Parse() + pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) { + var w skate.MinimalCitations + if err := json.Unmarshal(p, &w); err != nil { + return nil, err + } + return nil, nil + }) + pp.NumWorkers = *numWorkers + pp.BatchSize = *batchSize + if err := pp.Run(); err != nil { + log.Fatal(err) + } +} -- cgit v1.2.3