1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
|
package skate
import (
"bytes"
"errors"
"reflect"
"runtime"
"strings"
json "github.com/segmentio/encoding/json"
"github.com/tidwall/gjson"
)
var (
bTab = []byte("\t")
bNewline = []byte("\n")
ErrZeroFields = errors.New("zero fields")
ErrMissingFieldName = errors.New("missing field name")
)
// Title is a document with a title.
type TitleDoc struct {
Title string `json:"title"`
}
// PartialDoc for docs, that do not have DOI or title. E.g. we found 49701699
// (NCVY), 36401044 (NCVYU), 29668363 (NCUY), and so on. Some examples: XXX
type PartialDoc struct {
ContainerName string `json:"container_name"`
Contribs []struct {
// XXX: Need a way to sensibly compare sets of author names.
RawName string `json:"raw_name"`
} `json:"contribs"`
Volume string `json:"volume"`
Unstructured string `json:"unstructured"`
Year string `json:"release_year"`
}
// Mapper maps a blob to an arbitrary number of fields, e.g. for (key,
// doc). We want fields, but we do not want to bake in TSV into each function.
type Mapper func([]byte) ([][]byte, error)
// AsTSV serializes the result of a field mapper as TSV. This is a slim
// adapter, e.g. to parallel.Processor, which expects this function signature.
// A newline will be appended, if not there already.
func (f Mapper) AsTSV(p []byte) ([]byte, error) {
var (
fields [][]byte
err error
b []byte
)
if fields, err = f(p); err != nil {
return nil, err
}
if len(fields) == 0 {
return nil, nil
}
b = bytes.Join(fields, bTab)
if len(b) > 0 && !bytes.HasSuffix(b, bNewline) {
b = append(b, bNewline...)
}
return b, nil
}
// WithPrefix is a "mapper middleware", adding a given prefix to the first field.
func WithPrefix(f Mapper, prefix string) Mapper {
return func(p []byte) ([][]byte, error) {
fields, err := f(p)
if err != nil {
return fields, err
}
if len(fields) == 0 {
return nil, ErrZeroFields
}
fields[0] = append([]byte(prefix+":"), fields[0]...)
return fields, err
}
}
// WithBestEffort will not fail on an error.
func WithBestEffort(f Mapper) Mapper {
return func(p []byte) ([][]byte, error) {
if fields, err := f(p); err != nil {
return nil, nil
} else {
return fields, err
}
}
}
// WithSkipOnEmpty ignores results where the value at a given field is empty.
// One indexed.
func WithSkipOnEmpty(f Mapper, index int) Mapper {
return func(p []byte) ([][]byte, error) {
fields, err := f(p)
if err != nil {
return nil, err
}
if index < len(fields) && len(fields[index]) == 0 {
return nil, nil
}
return fields, err
}
}
// NameOf returns name of value, e.g. the name of a function.
func NameOf(f interface{}) string {
v := reflect.ValueOf(f)
if v.Kind() == reflect.Func {
if rf := runtime.FuncForPC(v.Pointer()); rf != nil {
return rf.Name()
}
}
return v.String()
}
// Identifier returns just the input again.
func Identity(p []byte) ([][]byte, error) {
return [][]byte{p}, nil
}
// CreateFixedMapper extract the value from a given fixed json key, e.g.
// "biblio.doi" and the like. Returns a function that maps doc to (value, doc).
func CreateFixedMapper(field string) Mapper {
if len(field) > 0 && field[0] == "." {
// gjson is not jq, we do not use a leading dot, so remove it, if
// accidentally used here
field = field[1:]
}
f := func(p []byte) ([][]byte, error) {
result := gjson.GetBytes(p, field)
key := []byte(result.String())
return [][]byte{key, p}, nil
}
return f
}
// MapperTitle extracts (title, doc).
func MapperTitle(p []byte) ([][]byte, error) {
var (
doc TitleDoc
key []byte
)
if err := json.Unmarshal(p, &doc); err != nil {
return nil, err
} else {
key = []byte(wsReplacer.Replace(strings.TrimSpace(doc.Title)))
}
return [][]byte{key, p}, nil
}
// MapperTitleNormalized extracts (title normalized, doc).
func MapperTitleNormalized(p []byte) (fields [][]byte, err error) {
if fields, err = MapperTitle(p); err != nil {
return nil, err
}
key := string(fields[0])
key = wsReplacer.Replace(strings.TrimSpace(key))
key = strings.ToLower(key)
key = repeatedWs.ReplaceAllString(key, " ")
key = nonWord.ReplaceAllString(key, "")
fields[0] = []byte(key)
return fields, nil
}
// MapperTitleNormalized extracts (title nysiis, doc).
func MapperTitleNysiis(p []byte) (fields [][]byte, err error) {
if fields, err = MapperTitle(p); err != nil {
return nil, err
}
key := string(fields[0])
key = wsReplacer.Replace(strings.TrimSpace(key))
key = NYSIIS(key)
fields[0] = []byte(key)
return fields, nil
}
// MapperTitleSandcrawler extracts (title sandcrawler, doc).
func MapperTitleSandcrawler(p []byte) (fields [][]byte, err error) {
if fields, err = MapperTitle(p); err != nil {
return nil, err
}
key := string(fields[0])
key = sandcrawlerSlugify(wsReplacer.Replace(strings.TrimSpace(key)))
fields[0] = []byte(key)
return fields, nil
}
// MapperURLFromRef extracts the (work ident, release ident, url, doc).
// Previously: parallel -j 16 --block 100M --pipe "jq -rc '[.work_ident,
// .release_ident, .biblio.url?] | @tsv'" ...
// This implementation seems slightly faster that jq and parallel.
func MapperURLFromRef(p []byte) (fields [][]byte, err error) {
var ref Ref
if err = json.Unmarshal(p, &ref); err != nil {
return nil, err
}
fields = [][]byte{
[]byte(ref.WorkIdent),
[]byte(ref.ReleaseIdent),
[]byte(ref.Biblio.Url),
p,
}
return fields, nil
}
// MapperPartial works on partial documents.
func MapperPartial(p []byte) (fields [][]byte, err error) {
// XXX: slugify authors, how to compare two author strings? How do these
// things look like?
return nil, nil
}
|
