diff options
| -rw-r--r-- | skate/cleanup.go | 78 | ||||
| -rw-r--r-- | skate/cmd/skate-cleanup/main.go | 75 | ||||
| -rw-r--r-- | skate/go.mod | 3 | ||||
| -rw-r--r-- | skate/go.sum | 7 |
4 files changed, 101 insertions, 62 deletions
diff --git a/skate/cleanup.go b/skate/cleanup.go new file mode 100644 index 0000000..b50e048 --- /dev/null +++ b/skate/cleanup.go @@ -0,0 +1,78 @@ +package skate + +import ( + "fmt" + "log" + "strings" + + "mvdan.cc/xurls/v2" +) + +var rxRelaxed = xurls.Relaxed() + +// URLFilter is a line oriented URL filter. +type FilterURL struct { + Delimiter string + Index int + BestEffort bool + Aggressive bool + SkipNonMatches bool + AllowedSchemas []string +} + +func (f *FilterURL) Run(p []byte) ([]byte, error) { + parts := strings.Split(string(p), f.Delimiter) + if len(parts) < f.Index { + msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p)) + if f.BestEffort { + log.Println(msg) + return nil, nil + } else { + return nil, fmt.Errorf(msg) + } + } + url := rxRelaxed.FindString(parts[f.Index-1]) + if f.Aggressive { + url = SanitizeURL(url) + } + if url == "" && f.SkipNonMatches { + return nil, nil + } + if len(f.AllowedSchemas) > 0 && !HasAnyPrefix(url, f.AllowedSchemas) { + return nil, nil + } + if len(parts) == 1 || f.Index == len(parts) { + url = url + "\n" + } + parts[f.Index-1] = url + return []byte(strings.Join(parts, f.Delimiter)), nil +} + +// FilterDOI is a line oriented DOI filter. +type FilterDOI struct { + Delimiter string + Index int + BestEffort bool + Aggressive bool + SkipNonMatches bool + AllowedSchema []string +} + +func (f *FilterDOI) Run(p []byte) ([]byte, error) { + parts := strings.Split(string(p), f.Delimiter) + if len(parts) < f.Index { + msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p)) + if f.BestEffort { + log.Println(msg) + return nil, nil + } else { + return nil, fmt.Errorf(msg) + } + } + doi := PatDOI.FindString(parts[f.Index-1]) + if doi == "" && f.SkipNonMatches { + return nil, nil + } + parts[f.Index-1] = strings.ToLower(doi) + return []byte(strings.Join(parts, f.Delimiter)), nil +} diff --git a/skate/cmd/skate-cleanup/main.go b/skate/cmd/skate-cleanup/main.go index d4d5f5a..4dd367f 100644 --- a/skate/cmd/skate-cleanup/main.go +++ b/skate/cmd/skate-cleanup/main.go @@ -9,16 +9,13 @@ package main import ( "flag" - "fmt" "log" "os" - "regexp" "runtime" "strings" "git.archive.org/martin/cgraph/skate" "git.archive.org/martin/cgraph/skate/parallel" - "mvdan.cc/xurls/v2" ) var ( @@ -32,8 +29,6 @@ var ( extendedCleanup = flag.Bool("X", false, "extended (and slower) cleanup for urls") allow = flag.String("allow", "http,https", "comma separted list of schemas to allow for urls") - PatDOI = regexp.MustCompile(`10[.][0-9]{1,8}/[^ ]*[\w]`) - rxRelaxed = xurls.Relaxed() allowedSchemas []string // parsed from allow flag ) @@ -44,12 +39,25 @@ func main() { } var f func([]byte) ([]byte, error) switch *what { - case "doi": - f = doiFilter case "url": - f = urlFilter + filter := skate.FilterURL{ + Delimiter: *delimiter, + Index: *index, + BestEffort: *bestEffort, + Aggressive: *extendedCleanup, + SkipNonMatches: *skipNonMatches, + AllowedSchemas: allowedSchemas, + } + f = filter.Run default: - f = doiFilter + filter := skate.FilterDOI{ + Delimiter: *delimiter, + Index: *index, + BestEffort: *bestEffort, + Aggressive: *extendedCleanup, + SkipNonMatches: *skipNonMatches, + } + f = filter.Run } pp := parallel.NewProcessor(os.Stdin, os.Stdout, f) pp.NumWorkers = *numWorkers @@ -58,52 +66,3 @@ func main() { log.Fatal(err) } } - -// urlFilter parses finds the first URL. -func urlFilter(p []byte) ([]byte, error) { - parts := strings.Split(string(p), *delimiter) - if len(parts) < *index { - msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p)) - if *bestEffort { - log.Println(msg) - return nil, nil - } else { - return nil, fmt.Errorf(msg) - } - } - url := rxRelaxed.FindString(parts[*index-1]) - if *extendedCleanup { - url = skate.SanitizeURL(url) - } - if url == "" && *skipNonMatches { - return nil, nil - } - if len(allowedSchemas) > 0 && !skate.HasAnyPrefix(url, allowedSchemas) { - return nil, nil - } - if len(parts) == 1 || *index == len(parts) { - url = url + "\n" - } - parts[*index-1] = url - return []byte(strings.Join(parts, *delimiter)), nil -} - -// doiFilter finds a DOI, normalizes to lowercase. -func doiFilter(p []byte) ([]byte, error) { - parts := strings.Split(string(p), *delimiter) - if len(parts) < *index { - msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p)) - if *bestEffort { - log.Println(msg) - return nil, nil - } else { - return nil, fmt.Errorf(msg) - } - } - result := PatDOI.FindString(parts[*index-1]) - if result == "" && *skipNonMatches { - return nil, nil - } - parts[*index-1] = strings.ToLower(result) - return []byte(strings.Join(parts, *delimiter)), nil -} diff --git a/skate/go.mod b/skate/go.mod index ed7ca25..13804b3 100644 --- a/skate/go.mod +++ b/skate/go.mod @@ -13,5 +13,6 @@ require ( github.com/sethgrid/pester v1.1.0 github.com/tidwall/gjson v1.7.5 golang.org/x/text v0.3.6 - mvdan.cc/xurls/v2 v2.2.0 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + mvdan.cc/xurls/v2 v2.3.0 ) diff --git a/skate/go.sum b/skate/go.sum index c4a3dcf..0e176b0 100644 --- a/skate/go.sum +++ b/skate/go.sum @@ -17,9 +17,10 @@ github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e h1:S+/ptYdZtpK/MDstwCyt+ZHdXEpz86RJZ5gyZU4txJY= github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/segmentio/encoding v0.2.17 h1:cgfmPc44u1po1lz5bSgF00gLCROBjDNc7h+H7I20zpc= github.com/segmentio/encoding v0.2.17/go.mod h1:7E68jTSWMnNoYhHi1JbLd7NBSB6XfE4vzqhR88hDBQc= github.com/sethgrid/pester v1.1.0 h1:IyEAVvwSUPjs2ACFZkBe5N59BBUpSIkQ71Hr6cM5A+w= @@ -43,5 +44,5 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A= -mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8= +mvdan.cc/xurls/v2 v2.3.0 h1:59Olnbt67UKpxF1EwVBopJvkSUBmgtb468E4GVWIZ1I= +mvdan.cc/xurls/v2 v2.3.0/go.mod h1:AjuTy7gEiUArFMjgBBDU4SMxlfUYsRokpJQgNWOt3e4= |