diff options
| -rw-r--r-- | skate/cluster.go | 9 | ||||
| -rw-r--r-- | skate/cmd/skate-derive-key/main.go | 3 |
2 files changed, 11 insertions, 1 deletions
diff --git a/skate/cluster.go b/skate/cluster.go index bec8154..954d971 100644 --- a/skate/cluster.go +++ b/skate/cluster.go @@ -110,6 +110,15 @@ func KeyTitleSandcrawler(p []byte) (ident string, key string, err error) { return ident, sandcrawlerSlugify(key), nil } +// KeySourceIdent extracts the source ident. +func KeySourceIdent(p []byte) (ident string, key string, err error) { + var doc IdentTitleDoc + if err = json.Unmarshal(p, &doc); err != nil { + return doc.Ident, doc.Ident, err + } + return doc.Ident, doc.Ident, nil +} + // sandcrawlerSlugify normalizes a string. func sandcrawlerSlugify(s string) string { slug := strings.ToLower(strings.TrimSpace(s)) diff --git a/skate/cmd/skate-derive-key/main.go b/skate/cmd/skate-derive-key/main.go index 2375a73..653e258 100644 --- a/skate/cmd/skate-derive-key/main.go +++ b/skate/cmd/skate-derive-key/main.go @@ -33,7 +33,7 @@ import ( ) var ( - keyFuncName = flag.String("f", "tsand", "key function name, other: title, tnorm, tnysi, tsand") + keyFuncName = flag.String("f", "tsand", "key function name, other: title, tnorm, tnysi, tsand, ident") numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers") batchSize = flag.Int("b", 50000, "batch size") verbose = flag.Bool("verbose", false, "show progress") @@ -47,6 +47,7 @@ var ( "tnorm": skate.KeyTitleNormalized, "tnysi": skate.KeyTitleNysiis, "tsand": skate.KeyTitleSandcrawler, + "ident": skate.KeySourceIdent, } keyFunc skate.IdentifierKeyFunc ok bool |