1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
package skate
import (
"bytes"
"errors"
"reflect"
"runtime"
"strings"
json "github.com/segmentio/encoding/json"
"github.com/tidwall/gjson"
)
var (
bTab = []byte("\t")
bNewline = []byte("\n")
ErrZeroFields = errors.New("zero fields")
ErrMissingFieldName = errors.New("missing field name")
)
// Title is a document with a title.
type TitleDoc struct {
Title string `json:"title"`
}
// PartialDoc for docs, that do not have DOI or title. E.g. we found 49701699
// (NCVY), 36401044 (NCVYU), 29668363 (NCUY), and so on.
type PartialDoc struct {
ContainerName string `json:"container_name"`
Contribs []struct {
RawName string `json:"raw_name"`
} `json:"contribs"`
Volume string `json:"volume"`
Unstructured string `json:"unstructured"`
Year string `json:"release_year"`
}
// Mapper maps a blob to an arbitrary number of fields, e.g. for (key,
// doc). We want fields, but we do not want to bake in TSV into each function.
type Mapper func([]byte) ([][]byte, error)
// AsTSV serializes the result of a field mapper as TSV. This is a slim adapter,
// e.g. to parallel.Processor, which expects this function signature.
func (f Mapper) AsTSV(p []byte) ([]byte, error) {
fields, err := f(p)
if err != nil {
return nil, err
}
return bytes.Join(fields, bTab), nil
}
// WithPrefix adds a given prefix to the first element.
func WithPrefix(f Mapper, prefix string) Mapper {
return func(p []byte) ([][]byte, error) {
fields, err := f(p)
if err != nil {
return fields, err
}
if len(fields) == 0 {
return nil, ErrZeroFields
}
fields[0] = append([]byte(prefix+":"), fields[0]...)
return fields, err
}
}
// NameOf returns name of value, e.g. the name of a function.
func NameOf(f interface{}) string {
v := reflect.ValueOf(f)
if v.Kind() == reflect.Func {
if rf := runtime.FuncForPC(v.Pointer()); rf != nil {
return rf.Name()
}
}
return v.String()
}
// Identifier returns just the input again.
func Identity(p []byte) ([][]byte, error) {
return [][]byte{p}, nil
}
// CreateFixedMapper extract the value from a given fixed top level json key.
// Returns a function that maps doc to (v, doc).
func CreateFixedMapper(field string) Mapper {
f := func(p []byte) ([][]byte, error) {
result := gjson.GetBytes(p, field)
key := []byte(result.String())
return [][]byte{key, p}, nil
}
return f
}
// MapperTitle extracts (title, doc).
func MapperTitle(p []byte) ([][]byte, error) {
var (
doc TitleDoc
key []byte
)
if err := json.Unmarshal(p, &doc); err != nil {
return nil, err
} else {
key = []byte(wsReplacer.Replace(strings.TrimSpace(doc.Title)))
}
return [][]byte{key, p}, nil
}
// MapperTitleNormalized extracts (title normalized, doc).
func MapperTitleNormalized(p []byte) (fields [][]byte, err error) {
if fields, err = MapperTitle(p); err != nil {
return nil, err
}
key := string(fields[0])
key = wsReplacer.Replace(strings.TrimSpace(key))
key = strings.ToLower(key)
key = repeatedWs.ReplaceAllString(key, " ")
key = nonWord.ReplaceAllString(key, "")
fields[0] = []byte(key)
return fields, nil
}
// MapperTitleNormalized extracts (title nysiis, doc).
func MapperTitleNysiis(p []byte) (fields [][]byte, err error) {
if fields, err = MapperTitle(p); err != nil {
return nil, err
}
key := string(fields[0])
key = wsReplacer.Replace(strings.TrimSpace(key))
key = NYSIIS(key)
fields[0] = []byte(key)
return fields, nil
}
// MapperTitleSandcrawler extracts (title sandcrawler, doc).
func MapperTitleSandcrawler(p []byte) (fields [][]byte, err error) {
if fields, err = MapperTitle(p); err != nil {
return nil, err
}
key := string(fields[0])
key = sandcrawlerSlugify(wsReplacer.Replace(strings.TrimSpace(key)))
fields[0] = []byte(key)
return fields, nil
}
// MapperPartial works on partial documents.
func MapperPartial(p []byte) (fields [][]byte, err error) {
// XXX: slugify authors, how to compare two author strings? How do these
// things look like?
return nil, nil
}
|
