#!/usr/bin/env python # http://web.archive.org/cdx/search/cdx?url=http://www.cell.com/current-biology/abstract/S0960-9822x # com,cell)/current-biology/abstract/s0960-9822 20170602053046 http://www.cell.com/current-biology/abstract/S0960-9822 text/html 404 7WYMCJSIU4CDPQAQ7Y7547EA6FXPRVBV 9747 import fileinput import collections import requests import sys Cdx = collections.namedtuple("Cdx", "surt date url mime status hash size") def parse_cdx_lines(blob): result = [] lines = blob.split("\n") cutoff = ["access", "Access", "abgeruf", "aufgeruf"] for line in lines: for c in cutoff: if c in line: line = line[:line.index(c)] break fields = line.strip().split() if len(fields) == 0: continue cdx = Cdx(*fields) result.append(cdx) result = sorted(result, key=lambda cdx: cdx.date, reverse=True) result = list(filter(lambda cdx: cdx.status in ("200", "301", "302", "303"), result)) return result def main(): stats = collections.Counter() for line in fileinput.input(): line = line.strip() r = requests.get("http://web.archive.org/cdx/search/cdx?url={}".format(line)) if not r.ok: continue try: cdx_lines = parse_cdx_lines(r.text) except TypeError: stats["parse_failed"] += 1 print("failed to parse cdx: {}".format(line), file=sys.stderr) continue if len(cdx_lines) == 0: stats["missing"] += 1 print("MISS\tNA\t{}".format(line)) else: stats["ok"] += 1 print("OK\t{}\t{}".format(cdx_lines[0].date, line)) return stats if __name__ == '__main__': stats = main() print(stats)