From 8f5c2ed46b809faad22c8f911ad77be4f8a156f8 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 17 Jan 2020 12:24:42 -0800 Subject: add SKIP log line for skip-url-blocklist path --- python/sandcrawler/ingest.py | 1 + 1 file changed, 1 insertion(+) (limited to 'python') diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 4732509..38ba55c 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -223,6 +223,7 @@ class IngestFileWorker(SandcrawlerWorker): for block in self.base_url_blocklist: if block in base_url: + print("[SKIP {}\t] {}".format(ingest_type, base_url), file=sys.stderr) return dict(request=request, hit=False, status="skip-url-blocklist") print("[INGEST {}\t] {}".format(ingest_type, base_url), file=sys.stderr) -- cgit v1.2.3