diff options
author | Martin Czygan <martin.czygan@gmail.com> | 2021-03-31 23:54:28 +0200 |
---|---|---|
committer | Martin Czygan <martin.czygan@gmail.com> | 2021-03-31 23:54:28 +0200 |
commit | d60dff7db926cc40a288584ac3f9970bb85c30c0 (patch) | |
tree | e79829df3f2229354b4dc079ee434274eb09aae0 /skate/cmd/skate-biblioref-from-wikipedia/main.go | |
parent | a7e0e8fe3a78fa6a9872cefcb7a9eb261128f6f2 (diff) | |
download | refcat-d60dff7db926cc40a288584ac3f9970bb85c30c0.tar.gz refcat-d60dff7db926cc40a288584ac3f9970bb85c30c0.zip |
rename to: skate-wikipedia-doi
Diffstat (limited to 'skate/cmd/skate-biblioref-from-wikipedia/main.go')
-rw-r--r-- | skate/cmd/skate-biblioref-from-wikipedia/main.go | 43 |
1 files changed, 0 insertions, 43 deletions
diff --git a/skate/cmd/skate-biblioref-from-wikipedia/main.go b/skate/cmd/skate-biblioref-from-wikipedia/main.go deleted file mode 100644 index 15b0d0a..0000000 --- a/skate/cmd/skate-biblioref-from-wikipedia/main.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "runtime" - "strings" - - "git.archive.org/martin/cgraph/skate" - "git.archive.org/martin/cgraph/skate/parallel" - jsoniter "github.com/json-iterator/go" -) - -var ( - numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") - batchSize = flag.Int("b", 100000, "batch size") - json = jsoniter.ConfigCompatibleWithStandardLibrary - bytesNewline = []byte("\n") -) - -func main() { - flag.Parse() - pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) { - var w skate.MinimalCitations - if err := json.Unmarshal(p, &w); err != nil { - return nil, err - } - idl := w.ParseIDList() - if idl.DOI == "" { - return nil, nil - } - s := fmt.Sprintf("%s\t%s\t%s", strings.TrimSpace(w.PageTitle), idl.DOI, string(p)) - return []byte(s), nil - - }) - pp.NumWorkers = *numWorkers - pp.BatchSize = *batchSize - if err := pp.Run(); err != nil { - log.Fatal(err) - } -} |