From ea784a826b6d0160575121f245675086b3dfc130 Mon Sep 17 00:00:00 2001
From: Martin Czygan <martin.czygan@gmail.com>
Date: Thu, 10 Jun 2021 02:10:24 +0200
Subject: task: get rid of grep

---
 python/refcat/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'python')

diff --git a/python/refcat/tasks.py b/python/refcat/tasks.py
index 50e13c7..09e88c4 100644
--- a/python/refcat/tasks.py
+++ b/python/refcat/tasks.py
@@ -470,8 +470,8 @@ class URLTabsCleaned(Refcat):
     def run(self):
         output = shellout("""
                           zstdcat -T0 {input} |
-                          skate-cleanup -X -c url -B -S -f 3 |
-                          LC_ALL=C grep -E '^https?://' |
+                          skate-cleanup -c url -allow http,https -X -B -S -f 3 |
+                          LC_ALL=C sort -T {tmpdir} -k3,3 -S25% --parallel 4 |
                           zstd -T0 -c > {output}
                           """,
                           n=self.n,
-- 
cgit v1.2.3