diff options
| -rw-r--r-- | skate/Makefile | 2 | ||||
| -rw-r--r-- | skate/cmd/skate-bref-id/main.go | 41 |
2 files changed, 1 insertions, 42 deletions
diff --git a/skate/Makefile b/skate/Makefile index 94b64fb..35cf2a9 100644 --- a/skate/Makefile +++ b/skate/Makefile @@ -1,5 +1,5 @@ SHELL := /bin/bash -TARGETS := skate-conv skate-cluster skate-cleanup skate-bref-id skate-from-unstructured skate-wikipedia-doi skate-dot skate-map skate-reduce skate-cdx-lookup skate-resolve-journal-name +TARGETS := skate-conv skate-cluster skate-cleanup skate-from-unstructured skate-wikipedia-doi skate-dot skate-map skate-reduce skate-cdx-lookup skate-resolve-journal-name PKGNAME := skate .PHONY: test diff --git a/skate/cmd/skate-bref-id/main.go b/skate/cmd/skate-bref-id/main.go deleted file mode 100644 index ba52219..0000000 --- a/skate/cmd/skate-bref-id/main.go +++ /dev/null @@ -1,41 +0,0 @@ -// skate-bref-id is a temporary helper to generate an id for a bref doc. -package main - -import ( - "flag" - "fmt" - "log" - "os" - "runtime" - "time" - - "git.archive.org/martin/cgraph/skate" - "git.archive.org/martin/cgraph/skate/parallel" - "github.com/segmentio/encoding/json" -) - -var ( - numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") - batchSize = flag.Int("b", 100000, "batch size") - - newlineB = []byte("\n") -) - -func main() { - pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) { - var bref skate.BiblioRef - if err := json.Unmarshal(p, &bref); err != nil { - return nil, err - } - bref.Key = fmt.Sprintf("%s_%d", bref.SourceReleaseIdent, bref.RefIndex) - bref.IndexedTs = time.Now().Format(time.RFC3339) - b, err := json.Marshal(bref) - b = append(b, newlineB...) - return b, err - }) - pp.NumWorkers = *numWorkers - pp.BatchSize = *batchSize - if err := pp.Run(); err != nil { - log.Fatal(err) - } -} |