From bffd1b13b72136dbfc6c1539f480d32c2f62a7aa Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Tue, 20 Jul 2021 00:33:16 +0200 Subject: reduce: a preliminary id for wb links --- skate/reduce.go | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'skate') diff --git a/skate/reduce.go b/skate/reduce.go index 8913cae..9620be3 100644 --- a/skate/reduce.go +++ b/skate/reduce.go @@ -364,6 +364,7 @@ func ZippyVerifyRefsOpenLibrary(olr, refs io.Reader, w io.Writer) error { func ZippyWayback(refs, cdx io.Reader, w io.Writer) error { var ( enc = json.NewEncoder(xio.NewSingleWriter(w)) + b32enc = base32.StdEncoding.WithPadding(base32.NoPadding) keyer = makeKeyFunc("\t", 1) grouper = func(g *zipkey.Group) error { var ( @@ -379,6 +380,10 @@ func ZippyWayback(refs, cdx io.Reader, w io.Writer) error { return err } var bref BiblioRef + // TODO: this is a temporary way to generate an id. + encodedURL = strings.ToLower(b32enc.EncodeToString([]byte(lang + ":" + cdx.Line))) + bref.Key = fmt.Sprintf("web_%s_%s", ref.ReleaseIdent, encodedURL) + bref.IndexedTs = timeNow().UTC().Format(time.RFC3339) bref.SourceReleaseIdent = ref.ReleaseIdent bref.SourceWorkIdent = ref.WorkIdent bref.SourceReleaseStage = ref.ReleaseStage -- cgit v1.2.3