aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--skate/cleanup.go78
-rw-r--r--skate/cmd/skate-cleanup/main.go75
-rw-r--r--skate/go.mod3
-rw-r--r--skate/go.sum7
4 files changed, 101 insertions, 62 deletions
diff --git a/skate/cleanup.go b/skate/cleanup.go
new file mode 100644
index 0000000..b50e048
--- /dev/null
+++ b/skate/cleanup.go
@@ -0,0 +1,78 @@
+package skate
+
+import (
+ "fmt"
+ "log"
+ "strings"
+
+ "mvdan.cc/xurls/v2"
+)
+
+var rxRelaxed = xurls.Relaxed()
+
+// URLFilter is a line oriented URL filter.
+type FilterURL struct {
+ Delimiter string
+ Index int
+ BestEffort bool
+ Aggressive bool
+ SkipNonMatches bool
+ AllowedSchemas []string
+}
+
+func (f *FilterURL) Run(p []byte) ([]byte, error) {
+ parts := strings.Split(string(p), f.Delimiter)
+ if len(parts) < f.Index {
+ msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
+ if f.BestEffort {
+ log.Println(msg)
+ return nil, nil
+ } else {
+ return nil, fmt.Errorf(msg)
+ }
+ }
+ url := rxRelaxed.FindString(parts[f.Index-1])
+ if f.Aggressive {
+ url = SanitizeURL(url)
+ }
+ if url == "" && f.SkipNonMatches {
+ return nil, nil
+ }
+ if len(f.AllowedSchemas) > 0 && !HasAnyPrefix(url, f.AllowedSchemas) {
+ return nil, nil
+ }
+ if len(parts) == 1 || f.Index == len(parts) {
+ url = url + "\n"
+ }
+ parts[f.Index-1] = url
+ return []byte(strings.Join(parts, f.Delimiter)), nil
+}
+
+// FilterDOI is a line oriented DOI filter.
+type FilterDOI struct {
+ Delimiter string
+ Index int
+ BestEffort bool
+ Aggressive bool
+ SkipNonMatches bool
+ AllowedSchema []string
+}
+
+func (f *FilterDOI) Run(p []byte) ([]byte, error) {
+ parts := strings.Split(string(p), f.Delimiter)
+ if len(parts) < f.Index {
+ msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
+ if f.BestEffort {
+ log.Println(msg)
+ return nil, nil
+ } else {
+ return nil, fmt.Errorf(msg)
+ }
+ }
+ doi := PatDOI.FindString(parts[f.Index-1])
+ if doi == "" && f.SkipNonMatches {
+ return nil, nil
+ }
+ parts[f.Index-1] = strings.ToLower(doi)
+ return []byte(strings.Join(parts, f.Delimiter)), nil
+}
diff --git a/skate/cmd/skate-cleanup/main.go b/skate/cmd/skate-cleanup/main.go
index d4d5f5a..4dd367f 100644
--- a/skate/cmd/skate-cleanup/main.go
+++ b/skate/cmd/skate-cleanup/main.go
@@ -9,16 +9,13 @@ package main
import (
"flag"
- "fmt"
"log"
"os"
- "regexp"
"runtime"
"strings"
"git.archive.org/martin/cgraph/skate"
"git.archive.org/martin/cgraph/skate/parallel"
- "mvdan.cc/xurls/v2"
)
var (
@@ -32,8 +29,6 @@ var (
extendedCleanup = flag.Bool("X", false, "extended (and slower) cleanup for urls")
allow = flag.String("allow", "http,https", "comma separted list of schemas to allow for urls")
- PatDOI = regexp.MustCompile(`10[.][0-9]{1,8}/[^ ]*[\w]`)
- rxRelaxed = xurls.Relaxed()
allowedSchemas []string // parsed from allow flag
)
@@ -44,12 +39,25 @@ func main() {
}
var f func([]byte) ([]byte, error)
switch *what {
- case "doi":
- f = doiFilter
case "url":
- f = urlFilter
+ filter := skate.FilterURL{
+ Delimiter: *delimiter,
+ Index: *index,
+ BestEffort: *bestEffort,
+ Aggressive: *extendedCleanup,
+ SkipNonMatches: *skipNonMatches,
+ AllowedSchemas: allowedSchemas,
+ }
+ f = filter.Run
default:
- f = doiFilter
+ filter := skate.FilterDOI{
+ Delimiter: *delimiter,
+ Index: *index,
+ BestEffort: *bestEffort,
+ Aggressive: *extendedCleanup,
+ SkipNonMatches: *skipNonMatches,
+ }
+ f = filter.Run
}
pp := parallel.NewProcessor(os.Stdin, os.Stdout, f)
pp.NumWorkers = *numWorkers
@@ -58,52 +66,3 @@ func main() {
log.Fatal(err)
}
}
-
-// urlFilter parses finds the first URL.
-func urlFilter(p []byte) ([]byte, error) {
- parts := strings.Split(string(p), *delimiter)
- if len(parts) < *index {
- msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
- if *bestEffort {
- log.Println(msg)
- return nil, nil
- } else {
- return nil, fmt.Errorf(msg)
- }
- }
- url := rxRelaxed.FindString(parts[*index-1])
- if *extendedCleanup {
- url = skate.SanitizeURL(url)
- }
- if url == "" && *skipNonMatches {
- return nil, nil
- }
- if len(allowedSchemas) > 0 && !skate.HasAnyPrefix(url, allowedSchemas) {
- return nil, nil
- }
- if len(parts) == 1 || *index == len(parts) {
- url = url + "\n"
- }
- parts[*index-1] = url
- return []byte(strings.Join(parts, *delimiter)), nil
-}
-
-// doiFilter finds a DOI, normalizes to lowercase.
-func doiFilter(p []byte) ([]byte, error) {
- parts := strings.Split(string(p), *delimiter)
- if len(parts) < *index {
- msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
- if *bestEffort {
- log.Println(msg)
- return nil, nil
- } else {
- return nil, fmt.Errorf(msg)
- }
- }
- result := PatDOI.FindString(parts[*index-1])
- if result == "" && *skipNonMatches {
- return nil, nil
- }
- parts[*index-1] = strings.ToLower(result)
- return []byte(strings.Join(parts, *delimiter)), nil
-}
diff --git a/skate/go.mod b/skate/go.mod
index ed7ca25..13804b3 100644
--- a/skate/go.mod
+++ b/skate/go.mod
@@ -13,5 +13,6 @@ require (
github.com/sethgrid/pester v1.1.0
github.com/tidwall/gjson v1.7.5
golang.org/x/text v0.3.6
- mvdan.cc/xurls/v2 v2.2.0
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+ mvdan.cc/xurls/v2 v2.3.0
)
diff --git a/skate/go.sum b/skate/go.sum
index c4a3dcf..0e176b0 100644
--- a/skate/go.sum
+++ b/skate/go.sum
@@ -17,9 +17,10 @@ github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e h1:S+/ptYdZtpK/MDstwCyt+ZHdXEpz86RJZ5gyZU4txJY=
github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/segmentio/encoding v0.2.17 h1:cgfmPc44u1po1lz5bSgF00gLCROBjDNc7h+H7I20zpc=
github.com/segmentio/encoding v0.2.17/go.mod h1:7E68jTSWMnNoYhHi1JbLd7NBSB6XfE4vzqhR88hDBQc=
github.com/sethgrid/pester v1.1.0 h1:IyEAVvwSUPjs2ACFZkBe5N59BBUpSIkQ71Hr6cM5A+w=
@@ -43,5 +44,5 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A=
-mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=
+mvdan.cc/xurls/v2 v2.3.0 h1:59Olnbt67UKpxF1EwVBopJvkSUBmgtb468E4GVWIZ1I=
+mvdan.cc/xurls/v2 v2.3.0/go.mod h1:AjuTy7gEiUArFMjgBBDU4SMxlfUYsRokpJQgNWOt3e4=