package main import ( "flag" "fmt" "log" "os" "runtime" "strings" "git.archive.org/martin/cgraph/skate" "git.archive.org/martin/cgraph/skate/parallel" json "github.com/segmentio/encoding/json" ) var ( numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") batchSize = flag.Int("b", 100000, "batch size") bytesNewline = []byte("\n") ) func main() { flag.Parse() pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) { var w skate.MinimalCitations if err := json.Unmarshal(p, &w); err != nil { return nil, err } idl := w.ParseIDList() if idl.DOI == "" { return nil, nil } s := fmt.Sprintf("%s\t%s\t%s", strings.TrimSpace(w.PageTitle), idl.DOI, string(p)) return []byte(s), nil }) pp.NumWorkers = *numWorkers pp.BatchSize = *batchSize if err := pp.Run(); err != nil { log.Fatal(err) } }