aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Czygan <martin.czygan@gmail.com>2021-07-15 03:47:26 +0200
committerMartin Czygan <martin.czygan@gmail.com>2021-07-15 03:47:26 +0200
commit8b2558c60eb8bef5a8d96371422e23b6df15fe0b (patch)
treea71aa1695659d03a3bfae818c9477279363f8136
parent83eea98c43f3ac6076b8aeb985a14f5b168b991e (diff)
downloadrefcat-8b2558c60eb8bef5a8d96371422e23b6df15fe0b.tar.gz
refcat-8b2558c60eb8bef5a8d96371422e23b6df15fe0b.zip
tasks: add RefsURL
-rw-r--r--python/refcat/tasks.py26
1 files changed, 26 insertions, 0 deletions
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 969d8dd..c04ec3c 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -1463,3 +1463,29 @@ class BrefZipWikiDOI(Refcat):
def output(self):
return luigi.LocalTarget(path=self.path(ext="json.zst"), format=Zstd)
+
+
+# Wayback related
+
+class RefsURL(Refcat):
+ """
+ Extract (url, doc), sort by url.
+ """
+ def requires(self):
+ return RefsWithUnstructured()
+
+ def run(self):
+ output = shellout("""
+ zstdcat -T0 {input} |
+ skate-map -m ur -skip-on-empty 1 |
+ LC_ALL=C sort -T {tmpdir} -k1,1 -S25% |
+ zstd -T0 -c > {output}
+ """,
+ n=self.n,
+ tmpdir=self.tmpdir,
+ input=self.input().path)
+ luigi.LocalTarget(output).move(self.output().path)
+
+ def output(self):
+ return luigi.LocalTarget(path=self.path(ext="tsv.zst"), format=Zstd)
+