tasks: add CDXURL

1 files changed, 28 insertions, 0 deletions

diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 9a4aa40..4a4a357 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -1467,6 +1467,7 @@ class BrefZipWikiDOI(Refcat):
 
 # Wayback related
 
+
 class RefsURL(Refcat):
     """
     Extract (url, doc), sort by url.
@@ -1490,3 +1491,30 @@ class RefsURL(Refcat):
     def output(self):
         return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
 
+
+class CDXURL(Refcat):
+    """
+    Stub implementation of ad-hoc CDX. We only consider a subset of documents.
+    """
+    cache = luigi.Parameter(default="/magna/.cache/skate/cdx", significant=False)
+    limit = luigi.IntParameter(default=10000, significant=False)
+
+    def requires(self):
+        return URLList()
+
+    def run(self):
+        output = shellout("""
+                 zstdcat -T0 {input} |
+                 head -n {limit} |
+                 skate-cdx-lookup -q -s 50ms -c {cache} -j -B |
+                 skate-map -m cdxu |
+                 LC_ALL=C sort -T {tmpdir} -k1,1 -S25% | zstd -c -T0 > {output}
+                 """,
+                          limit=self.limit,
+                          input=self.input().path,
+                          cache=self.cache)
+
+        luigi.LocalTarget(output).move(self.output().path)
+
+    def output(self):
+        return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)