package main import ( "bufio" "flag" "fmt" "io" "io/ioutil" "log" "os" "path" "strings" "time" "git.archive.org/martin/cgraph/skate" "github.com/adrg/xdg" ) var ( sleep = flag.Duration("s", 200*time.Millisecond, "sleep between requests") cacheDir = flag.String("c", path.Join(xdg.CacheHome, "skate-cdx"), "cache dir") bestEffort = flag.Bool("B", false, "best effort") quiet = flag.Bool("q", false, "no logging") ) func main() { flag.Parse() var r io.Reader = os.Stdin if flag.NArg() > 0 { f, err := os.Open(flag.Arg(0)) if err != nil { log.Fatal(err) } defer f.Close() r = f } if *quiet { log.SetOutput(ioutil.Discard) } var ( cache = skate.Cache{Dir: *cacheDir} br = bufio.NewReader(r) i int ) for { line, err := br.ReadString('\n') if err == io.EOF { break } if err != nil { log.Fatal(err) } line = strings.TrimSpace(line) if !cache.Has(line) { _, b, err := skate.LookupCDX(line) if err != nil { if *bestEffort { log.Println(err) continue } else { log.Fatal(err) } } if err := cache.Set(line, b); err != nil { log.Fatal(err) } time.Sleep(*sleep) } b, err := cache.Get(line) if err != nil { log.Fatal(err) } rows, err := skate.ParseCDX(b) if err != nil { if *bestEffort { log.Println(err) continue } else { log.Fatal(err) } } fmt.Printf("[%05d] % 10d %s %s\n", i, len(rows), rows.Summary(), line) i++ } }