From ef4127637c91cbf75ca4be711b0825306887d95b Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Wed, 12 May 2021 23:51:58 +0200 Subject: tasks: update URLList --- python/refcat/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'python') diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index 5553b23..0339932 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -298,7 +298,7 @@ class URLTabs(Refcat): class URLList(Refcat): """ - List of cleaned URLs from refs. + List of cleaned, unique URLs from refs. """ def requires(self): return URLTabs() @@ -307,8 +307,8 @@ class URLList(Refcat): output = shellout(""" zstdcat -T0 {input} | cut -f 3 | - skate-cleanup -c url -B -S -f 1 | - LC_ALL=C sort -T {tmpdir} -k1,1 -S25% --parallel 4 | + skate-cleanup -X -c url -B -S -f 1 | + LC_ALL=C sort -u -T {tmpdir} -k1,1 -S25% --parallel 4 | zstd -T0 -c > {output} """, n=self.n, -- cgit v1.2.3