skate: move cleanup code out

4 files changed, 101 insertions, 62 deletions

diff --git a/skate/cleanup.go b/skate/cleanup.go
new file mode 100644
index 0000000..b50e048
--- /dev/null
+++ b/skate/cleanup.go
@@ -0,0 +1,78 @@
+package skate
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"mvdan.cc/xurls/v2"
+)
+
+var rxRelaxed = xurls.Relaxed()
+
+// URLFilter is a line oriented URL filter.
+type FilterURL struct {
+	Delimiter      string
+	Index          int
+	BestEffort     bool
+	Aggressive     bool
+	SkipNonMatches bool
+	AllowedSchemas []string
+}
+
+func (f *FilterURL) Run(p []byte) ([]byte, error) {
+	parts := strings.Split(string(p), f.Delimiter)
+	if len(parts) < f.Index {
+		msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
+		if f.BestEffort {
+			log.Println(msg)
+			return nil, nil
+		} else {
+			return nil, fmt.Errorf(msg)
+		}
+	}
+	url := rxRelaxed.FindString(parts[f.Index-1])
+	if f.Aggressive {
+		url = SanitizeURL(url)
+	}
+	if url == "" && f.SkipNonMatches {
+		return nil, nil
+	}
+	if len(f.AllowedSchemas) > 0 && !HasAnyPrefix(url, f.AllowedSchemas) {
+		return nil, nil
+	}
+	if len(parts) == 1 || f.Index == len(parts) {
+		url = url + "\n"
+	}
+	parts[f.Index-1] = url
+	return []byte(strings.Join(parts, f.Delimiter)), nil
+}
+
+// FilterDOI is a line oriented DOI filter.
+type FilterDOI struct {
+	Delimiter      string
+	Index          int
+	BestEffort     bool
+	Aggressive     bool
+	SkipNonMatches bool
+	AllowedSchema  []string
+}
+
+func (f *FilterDOI) Run(p []byte) ([]byte, error) {
+	parts := strings.Split(string(p), f.Delimiter)
+	if len(parts) < f.Index {
+		msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
+		if f.BestEffort {
+			log.Println(msg)
+			return nil, nil
+		} else {
+			return nil, fmt.Errorf(msg)
+		}
+	}
+	doi := PatDOI.FindString(parts[f.Index-1])
+	if doi == "" && f.SkipNonMatches {
+		return nil, nil
+	}
+	parts[f.Index-1] = strings.ToLower(doi)
+	return []byte(strings.Join(parts, f.Delimiter)), nil
+}
diff --git a/skate/cmd/skate-cleanup/main.go b/skate/cmd/skate-cleanup/main.go
index d4d5f5a..4dd367f 100644
--- a/skate/cmd/skate-cleanup/main.go
+++ b/skate/cmd/skate-cleanup/main.go
@@ -9,16 +9,13 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"log"
 	"os"
-	"regexp"
 	"runtime"
 	"strings"
 
 	"git.archive.org/martin/cgraph/skate"
 	"git.archive.org/martin/cgraph/skate/parallel"
-	"mvdan.cc/xurls/v2"
 )
 
 var (
@@ -32,8 +29,6 @@ var (
 	extendedCleanup = flag.Bool("X", false, "extended (and slower) cleanup for urls")
 	allow           = flag.String("allow", "http,https", "comma separted list of schemas to allow for urls")
 
-	PatDOI         = regexp.MustCompile(`10[.][0-9]{1,8}/[^ ]*[\w]`)
-	rxRelaxed      = xurls.Relaxed()
 	allowedSchemas []string // parsed from allow flag
 )
 
@@ -44,12 +39,25 @@ func main() {
 	}
 	var f func([]byte) ([]byte, error)
 	switch *what {
-	case "doi":
-		f = doiFilter
 	case "url":
-		f = urlFilter
+		filter := skate.FilterURL{
+			Delimiter:      *delimiter,
+			Index:          *index,
+			BestEffort:     *bestEffort,
+			Aggressive:     *extendedCleanup,
+			SkipNonMatches: *skipNonMatches,
+			AllowedSchemas: allowedSchemas,
+		}
+		f = filter.Run
 	default:
-		f = doiFilter
+		filter := skate.FilterDOI{
+			Delimiter:      *delimiter,
+			Index:          *index,
+			BestEffort:     *bestEffort,
+			Aggressive:     *extendedCleanup,
+			SkipNonMatches: *skipNonMatches,
+		}
+		f = filter.Run
 	}
 	pp := parallel.NewProcessor(os.Stdin, os.Stdout, f)
 	pp.NumWorkers = *numWorkers
@@ -58,52 +66,3 @@ func main() {
 		log.Fatal(err)
 	}
 }
-
-// urlFilter parses finds the first URL.
-func urlFilter(p []byte) ([]byte, error) {
-	parts := strings.Split(string(p), *delimiter)
-	if len(parts) < *index {
-		msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
-		if *bestEffort {
-			log.Println(msg)
-			return nil, nil
-		} else {
-			return nil, fmt.Errorf(msg)
-		}
-	}
-	url := rxRelaxed.FindString(parts[*index-1])
-	if *extendedCleanup {
-		url = skate.SanitizeURL(url)
-	}
-	if url == "" && *skipNonMatches {
-		return nil, nil
-	}
-	if len(allowedSchemas) > 0 && !skate.HasAnyPrefix(url, allowedSchemas) {
-		return nil, nil
-	}
-	if len(parts) == 1 || *index == len(parts) {
-		url = url + "\n"
-	}
-	parts[*index-1] = url
-	return []byte(strings.Join(parts, *delimiter)), nil
-}
-
-// doiFilter finds a DOI, normalizes to lowercase.
-func doiFilter(p []byte) ([]byte, error) {
-	parts := strings.Split(string(p), *delimiter)
-	if len(parts) < *index {
-		msg := fmt.Sprintf("warn: line has too few fields (%d): %s", len(parts), string(p))
-		if *bestEffort {
-			log.Println(msg)
-			return nil, nil
-		} else {
-			return nil, fmt.Errorf(msg)
-		}
-	}
-	result := PatDOI.FindString(parts[*index-1])
-	if result == "" && *skipNonMatches {
-		return nil, nil
-	}
-	parts[*index-1] = strings.ToLower(result)
-	return []byte(strings.Join(parts, *delimiter)), nil
-}
diff --git a/skate/go.mod b/skate/go.mod
index ed7ca25..13804b3 100644
--- a/skate/go.mod
+++ b/skate/go.mod
@@ -13,5 +13,6 @@ require (
 	github.com/sethgrid/pester v1.1.0
 	github.com/tidwall/gjson v1.7.5
 	golang.org/x/text v0.3.6
-	mvdan.cc/xurls/v2 v2.2.0
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	mvdan.cc/xurls/v2 v2.3.0
 )
diff --git a/skate/go.sum b/skate/go.sum
index c4a3dcf..0e176b0 100644
--- a/skate/go.sum
+++ b/skate/go.sum
@@ -17,9 +17,10 @@ github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
 github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
 github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e h1:S+/ptYdZtpK/MDstwCyt+ZHdXEpz86RJZ5gyZU4txJY=
 github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/segmentio/encoding v0.2.17 h1:cgfmPc44u1po1lz5bSgF00gLCROBjDNc7h+H7I20zpc=
 github.com/segmentio/encoding v0.2.17/go.mod h1:7E68jTSWMnNoYhHi1JbLd7NBSB6XfE4vzqhR88hDBQc=
 github.com/sethgrid/pester v1.1.0 h1:IyEAVvwSUPjs2ACFZkBe5N59BBUpSIkQ71Hr6cM5A+w=
@@ -43,5 +44,5 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-mvdan.cc/xurls/v2 v2.2.0 h1:NSZPykBXJFCetGZykLAxaL6SIpvbVy/UFEniIfHAa8A=
-mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8=
+mvdan.cc/xurls/v2 v2.3.0 h1:59Olnbt67UKpxF1EwVBopJvkSUBmgtb468E4GVWIZ1I=
+mvdan.cc/xurls/v2 v2.3.0/go.mod h1:AjuTy7gEiUArFMjgBBDU4SMxlfUYsRokpJQgNWOt3e4=