1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
// Generate pairs and run verification on larger number of records. Mimick
// fuzzycat.verify, but make it faster (e.g. fuzzycat took about 50h for the
// complete set).
//
// Currently: about 2h for 40M clusters (in "ref" mode).
//
// XXX: Cleanup inconsistent "modes".
package main
import (
"bufio"
"flag"
"log"
"os"
"runtime"
"runtime/pprof"
"strings"
"git.archive.org/martin/cgraph/skate"
"git.archive.org/martin/cgraph/skate/parallel"
jsoniter "github.com/json-iterator/go"
)
var (
numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers")
batchSize = flag.Int("b", 10000, "batch size")
mode = flag.String("m", "ref", "mode: exact, ref, bref, zip, bzip, wiki")
exactReason = flag.String("r", "", "doi, pmid, pmcid, arxiv")
provenance = flag.String("p", "join", "provenance info")
wikiFile = flag.String("W", "", "wiki citation file")
releasesFile = flag.String("R", "", "releases, tsv, sorted by key (zip mode only)")
refsFile = flag.String("F", "", "refs, tsv, sorted by key (zip mode only)")
cpuProfile = flag.String("cpuprofile", "", "write cpu profile to file")
memProfile = flag.String("memprofile", "", "write heap profile to file (go tool pprof -png --alloc_objects program mem.pprof > mem.png)")
json = jsoniter.ConfigCompatibleWithStandardLibrary
// XXX: This should be cleanup up soon.
matchResults = map[string]skate.MatchResult{
"doi": skate.MatchResult{skate.StatusExact, skate.ReasonDOI},
"pmid": skate.MatchResult{skate.StatusExact, skate.ReasonPMID},
"pmcid": skate.MatchResult{skate.StatusExact, skate.ReasonPMCID},
"arxiv": skate.MatchResult{skate.StatusExact, skate.ReasonArxiv},
"unknown": skate.MatchResult{skate.StatusUnknown, skate.ReasonUnknown},
}
)
func main() {
flag.Parse()
if *cpuProfile != "" {
file, err := os.Create(*cpuProfile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(file)
defer pprof.StopCPUProfile()
}
switch *mode {
case "exact":
// Fixed zip mode for DOI.
if *refsFile == "" || *releasesFile == "" {
log.Fatal("mode requires -R and -F to be set")
}
if *exactReason == "" {
var keys []string
for k := range matchResults {
keys = append(keys, k)
}
log.Fatalf("need a reason for the record, one of: %s", strings.Join(keys, ", "))
}
f, err := os.Open(*releasesFile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
g, err := os.Open(*refsFile)
if err != nil {
log.Fatal(err)
}
defer g.Close()
bw := bufio.NewWriter(os.Stdout)
defer bw.Flush()
mr, ok := matchResults[*exactReason]
if !ok {
mr = matchResults["unknown"]
}
if err := skate.ZipUnverified(f, g, mr, *provenance, bw); err != nil {
log.Fatal(err)
}
case "zip":
// Take two "sorted key files" (one refs, one releases) and run
// verification across groups, generate biblioref file.
if *refsFile == "" || *releasesFile == "" {
log.Fatal("zip mode requires -F and -R to be set")
}
f, err := os.Open(*releasesFile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
g, err := os.Open(*refsFile)
if err != nil {
log.Fatal(err)
}
defer g.Close()
bw := bufio.NewWriter(os.Stdout)
defer bw.Flush()
if err := skate.ZipVerifyRefs(f, g, bw); err != nil {
log.Fatal(err)
}
case "ref":
// https://git.io/JtACz
pp := parallel.NewProcessor(os.Stdin, os.Stdout, skate.RefClusterVerify)
pp.NumWorkers = *numWorkers
pp.BatchSize = *batchSize
if err := pp.Run(); err != nil {
log.Fatal(err)
}
case "bref":
// generate biblioref
pp := parallel.NewProcessor(os.Stdin, os.Stdout, skate.RefClusterToBiblioRef)
pp.NumWorkers = *numWorkers
pp.BatchSize = *batchSize
if err := pp.Run(); err != nil {
log.Fatal(err)
}
case "wiki":
// Fixed zip mode for DOI from wikipedia.
if *wikiFile == "" || *releasesFile == "" {
log.Fatal("mode requires -W and -F to be set")
}
f, err := os.Open(*releasesFile)
if err != nil {
log.Fatal(err)
}
defer f.Close()
g, err := os.Open(*wikiFile)
if err != nil {
log.Fatal(err)
}
defer g.Close()
bw := bufio.NewWriter(os.Stdout)
defer bw.Flush()
if err := skate.ZipWikiUnverified(f, g, skate.MatchResult{skate.StatusExact, skate.ReasonDOI}, "wiki", bw); err != nil {
log.Fatal(err)
}
default:
log.Fatal("not implemented, only: zip, ref, bref")
}
if *memProfile != "" {
f, err := os.Create(*memProfile)
if err != nil {
log.Fatal("could not create memory profile: ", err)
}
defer f.Close()
runtime.GC()
if err := pprof.WriteHeapProfile(f); err != nil {
log.Fatal(err)
}
}
}
|
