diff options
| author | Martin Czygan <martin.czygan@gmail.com> | 2021-05-12 23:51:58 +0200 |
|---|---|---|
| committer | Martin Czygan <martin.czygan@gmail.com> | 2021-05-12 23:51:58 +0200 |
| commit | ef4127637c91cbf75ca4be711b0825306887d95b (patch) | |
| tree | 3bb17a5ae8aac34d46b60c5ec7f9158d36758089 /python | |
| parent | 497473c203f84cc26da96a51e0e19ab94a1a6e3a (diff) | |
| download | refcat-ef4127637c91cbf75ca4be711b0825306887d95b.tar.gz refcat-ef4127637c91cbf75ca4be711b0825306887d95b.zip | |
tasks: update URLList
Diffstat (limited to 'python')
| -rw-r--r-- | python/refcat/tasks.py | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py index 5553b23..0339932 100644 --- a/python/refcat/tasks.py +++ b/python/refcat/tasks.py @@ -298,7 +298,7 @@ class URLTabs(Refcat): class URLList(Refcat): """ - List of cleaned URLs from refs. + List of cleaned, unique URLs from refs. """ def requires(self): return URLTabs() @@ -307,8 +307,8 @@ class URLList(Refcat): output = shellout(""" zstdcat -T0 {input} | cut -f 3 | - skate-cleanup -c url -B -S -f 1 | - LC_ALL=C sort -T {tmpdir} -k1,1 -S25% --parallel 4 | + skate-cleanup -X -c url -B -S -f 1 | + LC_ALL=C sort -u -T {tmpdir} -k1,1 -S25% --parallel 4 | zstd -T0 -c > {output} """, n=self.n, |