From b362abd38ad4a6624bc056c58eb90ae235c63f00 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 23 Sep 2019 23:00:23 -0700 Subject: rename postgrest directory sql --- sql/backfill/petabox_transform.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 sql/backfill/petabox_transform.py (limited to 'sql/backfill/petabox_transform.py') diff --git a/sql/backfill/petabox_transform.py b/sql/backfill/petabox_transform.py new file mode 100755 index 0000000..b638911 --- /dev/null +++ b/sql/backfill/petabox_transform.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import json, sys, os + +for l in sys.stdin.readlines(): + l = l.strip() + if not l: + continue + r = json.loads(l) + if not r['sha1']: + continue + sha1hex = r['sha1'] + for url in r['urls']: + u = url['url'] + if not '//archive.org/' in u: + continue + u = u.split('/') + if u[2] == 'web.archive.org': + continue + #print(u) + assert u[2] == 'archive.org' and u[3] in ('download', 'serve') + item = u[4] + path = '/'.join(u[5:]) + print("\t".join([item, path, sha1hex])) -- cgit v1.2.3