aboutsummaryrefslogtreecommitdiffstats
path: root/skate/schema.go
blob: d0c6833a082abfd18a6226422bbd1366f4ae4966 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
package skate

import (
	"fmt"
	"strconv"
)

// RefToRelease converts a ref to a release. Set a extra.skate.status flag to
// be able to distinguish converted entities later.
func RefToRelease(ref *Ref) (*Release, error) {
	var (
		release  Release
		b        = ref.Biblio
		contribs = make([]struct {
			Index   int    `json:"index,omitempty"`
			RawName string `json:"raw_name,omitempty"`
			Role    string `json:"role,omitempty"`
		}, len(b.ContribRawNames))
	)
	release.Ident = ref.ReleaseIdent
	release.WorkID = ref.WorkIdent
	release.ExtIDs.Arxiv = b.ArxivId
	release.ExtIDs.DOI = b.DOI
	release.ExtIDs.PMID = b.PMID
	release.ExtIDs.PMCID = b.PMCID
	release.Title = b.Title
	release.Publisher = b.Publisher
	release.ContainerName = b.ContainerName
	release.Volume = b.Volume
	release.Issue = b.Issue
	release.Pages = b.Pages
	if ref.ReleaseYear > 1000 {
		release.ReleaseYearValue = fmt.Sprintf("%d", ref.ReleaseYear)
	}
	for i, name := range b.ContribRawNames {
		contribs[i].Index = i
		contribs[i].RawName = name
	}
	release.Contribs = contribs
	return &release, nil
}

// Ref is a reference document, can be very partial.
type Ref struct {
	Biblio struct {
		ArxivId         string   `json:"arxiv_id,omitempty"`
		ContainerName   string   `json:"container_name,omitempty"`
		ContribRawNames []string `json:"contrib_raw_names,omitempty"`
		DOI             string   `json:"doi,omitempty"`
		Issue           string   `json:"issue,omitempty"`
		PMCID           string   `json:"pmcid,omitempty"`
		PMID            string   `json:"pmid,omitempty"`
		Pages           string   `json:"pages,omitempty"`
		Publisher       string   `json:"publisher,omitempty"`
		Title           string   `json:"title,omitempty"`
		Unstructured    string   `json:"unstructured,omitempty"`
		Url             string   `json:"url,omitempty"`
		Volume          string   `json:"volume,omitempty"`
		Year            int64    `json:"year,omitempty"`
	} `json:"biblio"`
	Index        int64  `json:"index,omitempty"`
	Key          string `json:"key,omitempty"`
	RefSource    string `json:"ref_source,omitempty"`
	ReleaseYear  int    `json:"release_year,omitempty"`
	ReleaseIdent string `json:"release_ident,omitempty"`
	ReleaseStage string `json:"release_stage,omitempty"`
	WorkIdent    string `json:"work_ident,omitempty"`
}

// Release document. Note that we may have varying types for some fields.
// Mitigation for now is to make the field an interface{}, name the field
// "...Value" and to add a method with the field name, doing type assertion.
// Example: ReleaseYearValue interface{}, ReleaseYear() int, etc.
type Release struct {
	ContainerID   string `json:"container_id,omitempty"`
	ContainerName string `json:"container_name,omitempty"`
	Contribs      []struct {
		Index   int    `json:"index,omitempty"`
		RawName string `json:"raw_name,omitempty"`
		Role    string `json:"role,omitempty"`
	} `json:"contribs,omitempty"`
	ExtIDs struct {
		DOI         string `json:"doi,omitempty"`
		PMID        string `json:"pmid,omitempty"`
		PMCID       string `json:"pmcid,omitempty"`
		Arxiv       string `json:"arxiv,omitempty"`
		Core        string `json:"core,omitempty"`
		WikidataQID string `json:"wikidata_qid,omitempty"`
		Jstor       string `json:"jstor,omitempty"`
	} `json:"ext_ids,omitempty"`
	Ident     string `json:"ident,omitempty"`
	Publisher string `json:"publisher,omitempty"`
	Refs      []struct {
		ContainerName string `json:"container_name,omitempty"`
		Extra         struct {
			DOI     string   `json:"doi,omitempty"`
			Authors []string `json:"authors,omitempty"`
			Key     string   `json:"key,omitempty"`
			Year    string   `json:"year,omitempty"`
			Locator string   `json:"locator,omitempty"`
			Volume  string   `json:"volume,omitempty"`
		} `json:"extra"`
		Index   int64  `json:"index,omitempty"`
		Key     string `json:"key,omitempty"`
		Locator string `json:"locator,omitempty"`
		Year    int64  `json:"year,omitempty"`
	} `json:"refs,omitempty"`
	ReleaseDate      string      `json:"release_date,omitempty"`
	ReleaseYearValue interface{} `json:"release_year,omitempty"` // might be int or str
	ReleaseStage     string      `json:"release_stage,omitempty"`
	ReleaseType      string      `json:"release_type,omitempty"`
	Issue            string      `json:"issue,omitempty"`
	Volume           string      `json:"volume,omitempty"`
	Pages            string      `json:"pages,omitempty"`
	Title            string      `json:"title,omitempty"`
	WorkID           string      `json:"work_id,omitempty"`
	Extra            struct {
		ContainerName string      `json:"container_name,omitempty"`
		SubtitleValue interface{} `json:"subtitle,omitempty"` // []str or str
		Crossref      struct {
			Type string `json:"type,omitempty"`
		} `json:"crossref,omitempty"`
		DataCite struct {
			MetadataVersion int                `json:"metadataVersion,omitempty"`
			Relations       []DataCiteRelation `json:"relations,omitempty"`
		} `json:"datacite,omitempty"`
		Skate struct {
			// Mark as converted (e.g. by setting status to "ref")
			Status string `json:"status,omitempty"`
			// Carry the ref index and key around.
			Ref struct {
				Index   int64  `json:"index,omitempty"`
				Key     string `json:"key,omitempty"`
				Locator string `json:"locator,omitempty"`
			} `json:"ref,omitempty"`
			ResearchGate struct {
				URL string `json:"url,omitempty"`
			} `json:"rg,omitempty"`
		} `json:"skate,omitempty"`
	} `json:"extra,omitempty"`
}

// Subtitle returns a slice of subtitle strings.
func (r *Release) Subtitle() (result []string) {
	switch v := r.Extra.SubtitleValue.(type) {
	case []interface{}:
		for _, e := range v {
			result = append(result, fmt.Sprintf("%v", e))
		}
		return result
	case []string:
		return v
	case string:
		return []string{v}
	}
	return []string{}
}

// ReleaseYearString returns release year as string.
func (r *Release) ReleaseYearString() string {
	return fmt.Sprintf("%d", r.ReleaseYear())
}

// ReleaseYear returns year as int, no further validity checks.
func (r *Release) ReleaseYear() int {
	switch v := r.ReleaseYearValue.(type) {
	case int:
		return v
	case float64:
		return int(v)
	case string:
		w, err := strconv.Atoi(v)
		if err != nil {
			return 0
		}
		return w
	default:
		return 0
	}
}

// DataCiteRelation as it appears in the release extra field.
type DataCiteRelation struct {
	RelatedIdentifierType  string      `json:"relatedIdentifierType,omitempty"`
	RelatedIdentifierValue interface{} `json:"relatedIdentifier,omitempty"`
}

// RelatedIdentifier returns the identifier as string.
func (r *DataCiteRelation) RelatedIdentifier() string {
	switch v := r.RelatedIdentifierValue.(type) {
	case string:
		return v
	default:
		return fmt.Sprintf("%v", v)
	}
}

// Sitemap basic JSON style, e.g. for https://archive.org/details/rg_sitemap_2021_02_23.
type Sitemap struct {
	Lastmod string `json:"lastmod,omitempty"`
	Title   string `json:"title,omitempty"`
	URL     string `json:"url,omitempty"`
}

// BiblioRef as a prototype for indexing, https://is.gd/yicTom.
type BiblioRef struct {
	Key                    string `json:"_id,omitempty"`
	UpdateTs               int64  `json:"update_ts,omitempty"` // XXX: maybe: epoch_millis, https://www.elastic.co/guide/en/elasticsearch/reference/current/date.html
	SourceReleaseIdent     string `json:"source_release_ident,omitempty"`
	SourceWorkIdent        string `json:"source_work_ident,omitempty"`
	SourceWikipediaArticle string `json:"source_wikipedia_article,omitempty"`
	SourceReleaseStage     string `json:"source_release_stage,omitempty"`
	SourceYear             string `json:"source_year,omitempty"`
	RefIndex               int64  `json:"ref_index,omitempty"`
	RefKey                 string `json:"ref_key,omitempty"`
	RefLocator             string `json:"ref_locator,omitempty"`
	TargetReleaseIdent     string `json:"target_release_ident,omitempty"`
	TargetWorkIdent        string `json:"target_work_ident,omitempty"`
	TargetOpenLibraryWork  string `json:"target_openlibrary_work,omitempty"`
	TargetURLSurt          string `json:"target_url_surt,omitempty"`
	TargetURL              string `json:"target_url,omitempty"`
	MatchProvenance        string `json:"match_provenance,omitempty"`
	MatchStatus            string `json:"match_status,omitempty"`
	MatchReason            string `json:"match_reason,omitempty"`
	TargetUnstructured     string `json:"target_unstructured,omitempty"`
	TargetCSL              string `json:"target_csl,omitempty"`
}

// ClusterResult results.
type ClusterResult struct {
	Key    string     `json:"k"`
	Values []*Release `json:"v"`
}

// NonRef returns the first non-reference release found in a cluster, or an
// error, if none has been found.
func (cr *ClusterResult) OneNonRef() (*Release, error) {
	for _, re := range cr.Values {
		if re.Extra.Skate.Status != "ref" {
			return re, nil
		}
	}
	return nil, fmt.Errorf("no reference/release found")
}

// Group is a cluster with explicit groups (e.g. store the json lines in A, B).
type Group struct {
	Key string
	A   []string
	B   []string
}

func (g *Group) String() string {
	return fmt.Sprintf("<Group A/B %d/%d>", len(g.A), len(g.B))
}