blob: b41ae5385d2e9734891fbc046aa2720a37d72285 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
#!/usr/bin/env python
# http://web.archive.org/cdx/search/cdx?url=http://www.cell.com/current-biology/abstract/S0960-9822x
# com,cell)/current-biology/abstract/s0960-9822 20170602053046 http://www.cell.com/current-biology/abstract/S0960-9822 text/html 404 7WYMCJSIU4CDPQAQ7Y7547EA6FXPRVBV 9747
import fileinput
import collections
import requests
import sys
Cdx = collections.namedtuple("Cdx", "surt date url mime status hash size")
CdxApi = "http://web.archive.org/cdx/search/cdx"
okStatus = ('200', '301', '302', '303')
def parse_cdx_lines(blob):
result = []
lines = blob.split("\n")
for line in lines:
fields = line.strip().split()
if len(fields) == 0:
continue
cdx = Cdx(*fields)
result.append(cdx)
result = sorted(result, key=lambda cdx: cdx.date, reverse=True)
result = list(filter(lambda cdx: cdx.status in okStatus, result))
return result
def main():
stats = collections.Counter()
for line in fileinput.input():
line = line.strip()
r = requests.get("{}?url={}".format(CdxApi, line))
if not r.ok:
continue
try:
cdx_lines = parse_cdx_lines(r.text)
except TypeError:
stats["parse_failed"] += 1
print("failed to parse cdx: {}".format(line), file=sys.stderr)
continue
if len(cdx_lines) == 0:
stats["missing"] += 1
print("MISS\tNA\t{}".format(line))
else:
stats["ok"] += 1
print("OK\t{}\t{}".format(cdx_lines[0].date, line))
return stats
if __name__ == "__main__":
stats = main()
print(stats)
|
