diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/refcat/tasks.py | 28 |
1 files changed, 28 insertions, 0 deletions
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index 9a4aa40..4a4a357 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -1467,6 +1467,7 @@ class BrefZipWikiDOI(Refcat): # Wayback related + class RefsURL(Refcat): """ Extract (url, doc), sort by url. @@ -1490,3 +1491,30 @@ class RefsURL(Refcat): def output(self): return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) + +class CDXURL(Refcat): + """ + Stub implementation of ad-hoc CDX. We only consider a subset of documents. + """ + cache = luigi.Parameter(default="/magna/.cache/skate/cdx", significant=False) + limit = luigi.IntParameter(default=10000, significant=False) + + def requires(self): + return URLList() + + def run(self): + output = shellout(""" + zstdcat -T0 {input} | + head -n {limit} | + skate-cdx-lookup -q -s 50ms -c {cache} -j -B | + skate-map -m cdxu | + LC_ALL=C sort -T {tmpdir} -k1,1 -S25% | zstd -c -T0 > {output} + """, + limit=self.limit, + input=self.input().path, + cache=self.cache) + + luigi.LocalTarget(output).move(self.output().path) + + def output(self): + return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd) |