blob: 9b7a1f040776738cdf8efae08510aab0b9aa176c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
#!/usr/bin/env python
# http://web.archive.org/cdx/search/cdx?url=http://www.cell.com/current-biology/abstract/S0960-9822x
# com,cell)/current-biology/abstract/s0960-9822 20170602053046 http://www.cell.com/current-biology/abstract/S0960-9822 text/html 404 7WYMCJSIU4CDPQAQ7Y7547EA6FXPRVBV 9747
import fileinput
import collections
import requests
import sys
Cdx = collections.namedtuple("Cdx", "surt date url mime status hash size")
def parse_cdx_lines(blob):
result = []
lines = blob.split("\n")
cutoff = ["access", "Access", "abgeruf", "aufgeruf"]
for line in lines:
for c in cutoff:
if c in line:
line = line[:line.index(c)]
break
fields = line.strip().split()
if len(fields) == 0:
continue
cdx = Cdx(*fields)
result.append(cdx)
result = sorted(result, key=lambda cdx: cdx.date, reverse=True)
result = list(filter(lambda cdx: cdx.status in ("200", "301", "302", "303"), result))
return result
def main():
stats = collections.Counter()
for line in fileinput.input():
line = line.strip()
r = requests.get("http://web.archive.org/cdx/search/cdx?url={}".format(line))
if not r.ok:
continue
try:
cdx_lines = parse_cdx_lines(r.text)
except TypeError:
stats["parse_failed"] += 1
print("failed to parse cdx: {}".format(line), file=sys.stderr)
continue
if len(cdx_lines) == 0:
stats["missing"] += 1
print("MISS\tNA\t{}".format(line))
else:
stats["ok"] += 1
print("OK\t{}\t{}".format(cdx_lines[0].date, line))
return stats
if __name__ == '__main__':
stats = main()
print(stats)
|
