aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--skate/cmd/skate-map/main.go26
-rw-r--r--skate/cmd/skate-reduce/main.go68
2 files changed, 45 insertions, 49 deletions
diff --git a/skate/cmd/skate-map/main.go b/skate/cmd/skate-map/main.go
index f87c02f..a437705 100644
--- a/skate/cmd/skate-map/main.go
+++ b/skate/cmd/skate-map/main.go
@@ -2,9 +2,9 @@
// extract a key from a json document. For simple cases, you can use `jq` and
// other tools. Some key derivations require a bit more, hence a dedicated program.
//
-// An example with mostly unix tools. We want to extract (DOI, doc) tuples from
-// newline delimited JSON and sort by it; we also want to do this fast, hence
-// parallel, LC_ALL, etc.
+// An example with mostly unix tools. We want to extract (DOI, doc) tuples
+// (sorted by DOI) from newline delimited JSON; we also want to do this fast,
+// hence GNU parallel, LC_ALL, etc.
//
// $ zstdcat -T0 file.zst | (1)
// LC_ALL=C tr -d '\t' | (2) *
@@ -20,7 +20,7 @@
// (1) zstd is fast! "~4x faster than zlib" (https://is.gd/HT1DUs)
// (2) we use tab as column separator and we want clean this up before (could
// be skipped, if we limit number of splits)
-// (3) we pass the data to jq, with a bit larger buffer (default is 1MB)
+// (3) we pass the data to jq, with a bit larger buffer for GNU parallel (default is 1MB, currently)
// (4) we want no "null" output
// (5) tostring prints the input as string, because we need to carry the document forward ...
// (6) ... but we'll need some cleanup, too
@@ -28,9 +28,9 @@
// (8) a custom filter to normalize a DOI in a specific column
// (9) sorting by DOI
//
-// This is reasonably fast, but some cleanup is ugly. We also want more complex
-// keys, e.g. more normalizations, etc; in short: we'd like to encapsulate (2)
-// to (8) with `skate-map`.
+// This is reasonably fast, but some data cleanup code is ugly. We also want
+// more complex keys, e.g. more normalizations, etc; in short: we'd like to
+// encapsulate (2) to (8) with `skate-map`.
package main
import (
@@ -92,19 +92,19 @@ func main() {
}
switch {
case *mapperName != "":
- if mapf, ok := availableMappers[*mapperName]; !ok {
+ if mapper, ok := availableMappers[*mapperName]; !ok {
log.Fatalf("unknown mapper name: %v", *mapperName)
} else {
if *skipOnEmpty > 0 {
- mapf = skate.WithSkipOnEmpty(mapf, *skipOnEmpty-1)
+ mapper = skate.WithSkipOnEmpty(mapper, *skipOnEmpty-1)
}
if *keyPrefix != "" {
- mapf = skate.WithPrefix(mapf, *keyPrefix)
+ mapper = skate.WithPrefix(mapper, *keyPrefix)
}
if *bestEffort {
- mapf = skate.WithBestEffort(mapf)
+ mapper = skate.WithBestEffort(mapper)
}
- pp := parallel.NewProcessor(os.Stdin, os.Stdout, mapf.AsTSV)
+ pp := parallel.NewProcessor(os.Stdin, os.Stdout, mapper.AsTSV)
pp.NumWorkers = *numWorkers
pp.BatchSize = *batchSize
pp.Verbose = *verbose
@@ -115,9 +115,9 @@ func main() {
default:
fmt.Println(help)
w := tabwriter.NewWriter(os.Stdout, 0, 0, 4, ' ', 0)
+ defer w.Flush()
for k, v := range availableMappers {
fmt.Fprintf(w, "%s\t%s\n", k, skate.NameOf(v))
}
- w.Flush()
}
}
diff --git a/skate/cmd/skate-reduce/main.go b/skate/cmd/skate-reduce/main.go
index 7918a28..6d743b7 100644
--- a/skate/cmd/skate-reduce/main.go
+++ b/skate/cmd/skate-reduce/main.go
@@ -1,6 +1,6 @@
-// skate-reduce takes prepared inputs (e.g. from skate-map or skate-cluster)
-// and applies various verification and conversion functions. The output will
-// often be a stream of biblioref schema docs.
+// skate-reduce takes prepared inputs (e.g. from skate-map) and applies various
+// verification and conversion functions. The output will often be a stream of
+// biblioref schema docs.
//
// Support various "modes", e.g. exact, verify, ref, bref, wiki. Each mode may
// work on one or two files, and may need extra args.
@@ -9,38 +9,38 @@
// | will emit biblioref docs relating *one* element from releases with *all*
// | elements from ref; this is for "doi", "pmid" and other id matches, where no
// | further checks are necessary. The match reason, e.g. "doi" needs to be
-// | supplied.
+// | supplied
// |
// | $ skate-reduce -m exact -r doi -F a.tsv -L b.tsv
// |
// |
// * fuzzy | takes two (key, doc) TSV files (one for release, one for refs),
-// | runs verification within a group and will emit biblioref.
+// | runs verification within a group and will emit biblioref
// |
// | $ skate-reduce -m fuzzy -F a.tsv -L b.tsv
// |
// |
-// * wiki | zippy mode for releases and wikipedia inputs.
+// * wiki | zippy mode for releases and wikipedia inputs
// |
// | $ skate-reduce -m wiki -L a.ndj -W b.ndj
// |
// |
-// * oledt | zippy mode for releases and OL inputs, dumps table for debugging.
+// * oledt | zippy mode for releases and OL inputs, dumps table for debugging
// |
// | $ skate-reduce -m oled -F a.ndj -O b.ndj
// |
// |
-// * oled | zippy mode for releases and OL inputs, emit biblioref.
+// * oled | zippy mode for releases and OL inputs, emit biblioref
// |
// | $ skate-reduce -m oled -F a.ndj -O b.ndj
// |
// |
-// * rere | zippy mode for OL release and ref (as release) inputs, emit biblioref.
+// * rere | zippy mode for OL release and ref (as release) inputs, emit biblioref
// |
// | $ skate-reduce -m rere -O a.ndj -F b.ndj
// |
// |
-// * unmatched | join matched and unmatched reference data; do deduplicate on the fly
+// * unmatched | join matched and unmatched reference data; deduplicate on the fly
// |
// | $ skate-reduce -m unmatched -B a.ndj -F b.ndj
// |
@@ -81,8 +81,8 @@ var (
cdxAdhoc = flag.String("C", "", "path to cdx adhoc schema")
// Extra args.
- reason = flag.String("r", "", "reason for match: doi, pmid, pmcid, arxiv, unknown")
- reasonMap = map[string]skate.MatchResult{
+ matchResult = flag.String("r", "", "reason for match: doi, pmid, pmcid, arxiv, unknown")
+ matchResultMap = map[string]skate.MatchResult{
"doi": skate.MatchResult{Status: skate.StatusExact, Reason: skate.ReasonDOI},
"pmid": skate.MatchResult{Status: skate.StatusExact, Reason: skate.ReasonPMID},
"pmcid": skate.MatchResult{Status: skate.StatusExact, Reason: skate.ReasonPMCID},
@@ -118,76 +118,72 @@ func main() {
if *quite {
log.SetOutput(ioutil.Discard)
}
- bw := bufio.NewWriter(os.Stdout)
+ var (
+ bw = bufio.NewWriter(os.Stdout)
+ l, f, w, o, b, c *os.File
+ r skate.MatchResult
+ ok bool
+ err error
+ )
defer bw.Flush()
switch *mode {
case "exact":
- l, f, err := xio.OpenTwo(*releases, *refs)
- if err != nil {
+ if l, f, err = xio.OpenTwo(*releases, *refs); err != nil {
log.Fatal(err)
}
- r, ok := reasonMap[*reason]
- if !ok {
- log.Fatalf("unknown reason: %v", *reason)
+ if r, ok = matchResultMap[*matchResult]; !ok {
+ log.Fatalf("unknown reason: %v", *matchResult)
}
if err := skate.ZippyExact(l, f, r, bw); err != nil {
log.Fatal(err)
}
case "fuzzy":
- l, f, err := xio.OpenTwo(*releases, *refs)
- if err != nil {
+ if l, f, err = xio.OpenTwo(*releases, *refs); err != nil {
log.Fatal(err)
}
if err := skate.ZippyVerifyRefs(l, f, bw); err != nil {
log.Fatal(err)
}
case "wiki":
- l, w, err := xio.OpenTwo(*releases, *wiki)
- if err != nil {
+ if l, w, err = xio.OpenTwo(*releases, *wiki); err != nil {
log.Fatal(err)
}
- if err := skate.ZippyExactWiki(l, w, reasonMap["doi"], bw); err != nil {
+ if err := skate.ZippyExactWiki(l, w, matchResultMap["doi"], bw); err != nil {
log.Fatal(err)
}
case "oledt":
- o, f, err := xio.OpenTwo(*openLibrary, *refs)
- if err != nil {
+ if o, f, err = xio.OpenTwo(*openLibrary, *refs); err != nil {
log.Fatal(err)
}
if err := skate.ZippyVerifyRefsOpenLibraryTable(o, f, bw); err != nil {
log.Fatal(err)
}
case "oled":
- o, f, err := xio.OpenTwo(*openLibrary, *refs)
- if err != nil {
+ if o, f, err = xio.OpenTwo(*openLibrary, *refs); err != nil {
log.Fatal(err)
}
if err := skate.ZippyVerifyRefsOpenLibrary(o, f, bw); err != nil {
log.Fatal(err)
}
case "rere":
- o, f, err := xio.OpenTwo(*releases, *refs)
- if err != nil {
+ if o, f, err = xio.OpenTwo(*releases, *refs); err != nil {
log.Fatal(err)
}
- r, ok := reasonMap[*reason]
- if !ok {
- log.Fatalf("unknown reason: %v", *reason)
+ if r, ok = matchResultMap[*matchResult]; !ok {
+ log.Fatalf("unknown reason: %v", *matchResult)
}
if err := skate.ZippyExactReleases(o, f, r, bw); err != nil {
log.Fatal(err)
}
case "unmatched":
- b, f, err := xio.OpenTwo(*bref, *refs)
- if err != nil {
+ if b, f, err = xio.OpenTwo(*bref, *refs); err != nil {
log.Fatal(err)
}
if err := skate.ZippyBrefAugment(b, f, bw); err != nil {
log.Fatal(err)
}
case "wb":
- f, c, err := xio.OpenTwo(*refs, *cdxAdhoc)
- if err != nil {
+ if f, c, err = xio.OpenTwo(*refs, *cdxAdhoc); err != nil {
log.Fatal(err)
}
if err := skate.ZippyWayback(f, c, bw); err != nil {
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-23 09:57:49 +0000