diff options
| author | Martin Czygan <martin.czygan@gmail.com> | 2021-07-20 00:33:16 +0200 |
|---|---|---|
| committer | Martin Czygan <martin.czygan@gmail.com> | 2021-07-20 00:33:16 +0200 |
| commit | bffd1b13b72136dbfc6c1539f480d32c2f62a7aa (patch) | |
| tree | 3184583aef6ecff0002a9d638ffc757568804280 /skate | |
| parent | f5bed5dddc9e43e6d5623e03be985197d7244f8b (diff) | |
| download | refcat-bffd1b13b72136dbfc6c1539f480d32c2f62a7aa.tar.gz refcat-bffd1b13b72136dbfc6c1539f480d32c2f62a7aa.zip | |
reduce: a preliminary id for wb links
Diffstat (limited to 'skate')
| -rw-r--r-- | skate/reduce.go | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/skate/reduce.go b/skate/reduce.go index 8913cae..9620be3 100644 --- a/skate/reduce.go +++ b/skate/reduce.go @@ -364,6 +364,7 @@ func ZippyVerifyRefsOpenLibrary(olr, refs io.Reader, w io.Writer) error { func ZippyWayback(refs, cdx io.Reader, w io.Writer) error { var ( enc = json.NewEncoder(xio.NewSingleWriter(w)) + b32enc = base32.StdEncoding.WithPadding(base32.NoPadding) keyer = makeKeyFunc("\t", 1) grouper = func(g *zipkey.Group) error { var ( @@ -379,6 +380,10 @@ func ZippyWayback(refs, cdx io.Reader, w io.Writer) error { return err } var bref BiblioRef + // TODO: this is a temporary way to generate an id. + encodedURL = strings.ToLower(b32enc.EncodeToString([]byte(lang + ":" + cdx.Line))) + bref.Key = fmt.Sprintf("web_%s_%s", ref.ReleaseIdent, encodedURL) + bref.IndexedTs = timeNow().UTC().Format(time.RFC3339) bref.SourceReleaseIdent = ref.ReleaseIdent bref.SourceWorkIdent = ref.WorkIdent bref.SourceReleaseStage = ref.ReleaseStage |