From d5f0602e80847adf3d359a7fd06cc131c07cb6dd Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 11 Aug 2020 17:16:39 -0700 Subject: ingest: check for URL blocklist and cookie URL patterns on every hop --- python/sandcrawler/ingest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'python') diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 263b9d5..1f693dc 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -323,6 +323,19 @@ class IngestFileWorker(SandcrawlerWorker): while len(hops) <= self.max_hops: result['hops'] = hops + + # check against blocklist again on each hop + for block in self.base_url_blocklist: + if block in next_url: + result['status'] = "skip-url-blocklist" + return result + + # check for popular cookie blocking URL patterns. On successful SPN + # crawls, shouldn't see these redirect URLs + if '/cookieAbsent' in next_url or 'cookieSet=1' in next_url: + result['status'] = 'blocked-cookie' + return result + try: resource = self.find_resource(next_url, best_mimetype, force_recrawl=force_recrawl) except SavePageNowError as e: -- cgit v1.2.3