aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/ingest.py3
1 files changed, 3 insertions, 0 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index d63baff..58f3783 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -70,6 +70,9 @@ class IngestFileWorker(SandcrawlerWorker):
self.try_spn2 = kwargs.get('try_spn2', True)
self.base_url_blocklist = [
+ # robot blocking
+ "://hkvalidate.perfdrive.com/",
+
# temporary, until we implement specific fetch and 'petabox' output
"://archive.org/",
"://web.archive.org/web/",