1 files changed, 57 insertions, 0 deletions
diff --git a/extra/cdx/cdx_query.py b/extra/cdx/cdx_query.py
new file mode 100644
index 0000000..9b7a1f0
--- /dev/null
+++ b/extra/cdx/cdx_query.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+# http://web.archive.org/cdx/search/cdx?url=http://www.cell.com/current-biology/abstract/S0960-9822x
+# com,cell)/current-biology/abstract/s0960-9822 20170602053046 http://www.cell.com/current-biology/abstract/S0960-9822 text/html 404 7WYMCJSIU4CDPQAQ7Y7547EA6FXPRVBV 9747
+
+import fileinput
+import collections
+import requests
+import sys
+
+Cdx = collections.namedtuple("Cdx", "surt date url mime status hash size")
+
+
+def parse_cdx_lines(blob):
+    result = []
+    lines = blob.split("\n")
+    cutoff = ["access", "Access", "abgeruf", "aufgeruf"]
+    for line in lines:
+        for c in cutoff:
+            if c in line:
+                line = line[:line.index(c)]
+                break
+        fields = line.strip().split()
+        if len(fields) == 0:
+            continue
+        cdx = Cdx(*fields)
+        result.append(cdx)
+    result = sorted(result, key=lambda cdx: cdx.date, reverse=True)
+    result = list(filter(lambda cdx: cdx.status in ("200", "301", "302", "303"), result))
+    return result
+
+def main():
+    stats = collections.Counter()
+    for line in fileinput.input():
+        line = line.strip()
+        r = requests.get("http://web.archive.org/cdx/search/cdx?url={}".format(line))
+        if not r.ok:
+            continue
+        try:
+            cdx_lines = parse_cdx_lines(r.text)
+        except TypeError:
+            stats["parse_failed"] += 1
+            print("failed to parse cdx: {}".format(line), file=sys.stderr)
+            continue
+        if len(cdx_lines) == 0:
+            stats["missing"] += 1
+            print("MISS\tNA\t{}".format(line))
+        else:
+            stats["ok"] += 1
+            print("OK\t{}\t{}".format(cdx_lines[0].date, line))
+
+
+    return stats
+
+if __name__ == '__main__':
+    stats = main()
+    print(stats)