diff options
| -rw-r--r-- | python/refcat/tasks.py | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index 5553b23..0339932 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -298,7 +298,7 @@ class URLTabs(Refcat): class URLList(Refcat): """ - List of cleaned URLs from refs. + List of cleaned, unique URLs from refs. """ def requires(self): return URLTabs() @@ -307,8 +307,8 @@ class URLList(Refcat): output = shellout(""" zstdcat -T0 {input} | cut -f 3 | - skate-cleanup -c url -B -S -f 1 | - LC_ALL=C sort -T {tmpdir} -k1,1 -S25% --parallel 4 | + skate-cleanup -X -c url -B -S -f 1 | + LC_ALL=C sort -u -T {tmpdir} -k1,1 -S25% --parallel 4 | zstd -T0 -c > {output} """, n=self.n, |