add best effort flag

1 files changed, 21 insertions, 4 deletions

diff --git a/skate/cmd/skate-conv/main.go b/skate/cmd/skate-conv/main.go
index a2c85a3..850d0d5 100644
--- a/skate/cmd/skate-conv/main.go
+++ b/skate/cmd/skate-conv/main.go
@@ -22,6 +22,7 @@ import (
 var (
 	numWorkers                    = flag.Int("w", runtime.NumCPU(), "number of workers")
 	batchSize                     = flag.Int("b", 100000, "batch size")
+	bestEffort                    = flag.Bool("B", false, "only log errors, do not halt")
 	fromFormat                    = flag.String("f", "ref", "import schema: ref, rg, ol, oled")
 	extraOpenLibraryAuthorMapping = flag.String("Xa", "", "TSV file, mapping OL author keys (e.g. to plain text names")
 
@@ -78,7 +79,11 @@ func refToRelease(p []byte) ([]byte, error) {
 		err     error
 	)
 	if err = json.Unmarshal(p, &ref); err != nil {
-		return nil, err
+		if *bestEffort {
+			log.Printf("failed to unmarshal: %v", string(p))
+		} else {
+			return nil, err
+		}
 	}
 	if release, err = skate.RefToRelease(&ref); err != nil {
 		return nil, err
@@ -98,7 +103,11 @@ func rgSitemapToRelease(p []byte) ([]byte, error) {
 		err     error
 	)
 	if err = json.Unmarshal(p, &s); err != nil {
-		return nil, err
+		if *bestEffort {
+			log.Printf("failed to unmarshal: %v", string(p))
+		} else {
+			return nil, err
+		}
 	}
 	release.Title = s.Title
 	if len(s.URL) > 41 {
@@ -119,7 +128,11 @@ func openLibraryToRelease(p []byte) ([]byte, error) {
 		err     error
 	)
 	if err = json.Unmarshal(p, &w); err != nil {
-		return nil, err
+		if *bestEffort {
+			log.Printf("failed to unmarshal: %v", string(p))
+		} else {
+			return nil, err
+		}
 	}
 	if release, err = skate.OpenLibraryWorkToRelease(&w); err != nil {
 		return nil, err
@@ -137,7 +150,11 @@ func openLibraryEditionToRelease(p []byte) ([]byte, error) {
 		err     error
 	)
 	if err = json.Unmarshal(p, &w); err != nil {
-		return nil, err
+		if *bestEffort {
+			log.Printf("failed to unmarshal: %v", string(p))
+		} else {
+			return nil, err
+		}
 	}
 	if release, err = skate.OpenLibraryEditionToRelease(&w, openLibraryAuthorMap); err != nil {
 		return nil, err