package main import ( "bufio" "encoding/json" "flag" "fmt" "io" "io/ioutil" "log" "os" "path" "strings" "time" "git.archive.org/martin/cgraph/skate" "github.com/adrg/xdg" ) var ( sleep = flag.Duration("s", 200*time.Millisecond, "sleep between requests") cacheDir = flag.String("c", path.Join(xdg.CacheHome, "skate-cdx"), "cache dir") bestEffort = flag.Bool("B", false, "best effort") writeJson = flag.Bool("j", false, "json output") quiet = flag.Bool("q", false, "no logging") ) func main() { flag.Parse() var r io.Reader = os.Stdin if flag.NArg() > 0 { f, err := os.Open(flag.Arg(0)) if err != nil { log.Fatal(err) } defer f.Close() r = f } if *quiet { log.SetOutput(ioutil.Discard) } var ( cache = skate.Cache{Dir: *cacheDir} br = bufio.NewReader(r) i int ) for { line, err := br.ReadString('\n') if err == io.EOF { break } if err != nil { log.Fatal(err) } line = strings.TrimSpace(line) if !cache.Has(line) { _, b, err := skate.LookupCDX(line) if err != nil { if *bestEffort { log.Println(err) continue } else { log.Fatal(err) } } if err := cache.Set(line, b); err != nil { log.Fatal(err) } time.Sleep(*sleep) } b, err := cache.Get(line) if err != nil { log.Fatal(err) } rows, err := skate.ParseCDX(b) if err != nil { if *bestEffort { log.Println(err) continue } else { log.Fatal(err) } } if *writeJson { cdxSummary := rows.Summary() var dummy = struct { Summary *skate.CDXSummary `json:"summary"` NumRows int `json:"numRows"` Line string `json:"line"` }{ Summary: cdxSummary, NumRows: len(rows), Line: line, } b, err := json.Marshal(dummy) if err != nil { log.Fatal(err) } fmt.Println(string(b)) } else { fmt.Printf("[%05d] % 10d %s %s\n", i, len(rows), rows.Summary(), line) } i++ } }