aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-01-17 12:24:42 -0800
committerBryan Newbold <bnewbold@archive.org>2020-01-17 12:24:42 -0800
commit8f5c2ed46b809faad22c8f911ad77be4f8a156f8 (patch)
tree083cefaadf9b2253edb140b943bca3e22bd47b14
parent58f744e97c8f3f1a3472aa821f4518d7d139e850 (diff)
downloadsandcrawler-8f5c2ed46b809faad22c8f911ad77be4f8a156f8.tar.gz
sandcrawler-8f5c2ed46b809faad22c8f911ad77be4f8a156f8.zip
add SKIP log line for skip-url-blocklist path
-rw-r--r--python/sandcrawler/ingest.py1
1 files changed, 1 insertions, 0 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 4732509..38ba55c 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -223,6 +223,7 @@ class IngestFileWorker(SandcrawlerWorker):
for block in self.base_url_blocklist:
if block in base_url:
+ print("[SKIP {}\t] {}".format(ingest_type, base_url), file=sys.stderr)
return dict(request=request, hit=False, status="skip-url-blocklist")
print("[INGEST {}\t] {}".format(ingest_type, base_url), file=sys.stderr)