aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-10-21 12:22:30 -0700
committerBryan Newbold <bnewbold@archive.org>2020-10-21 12:22:30 -0700
commit86cc15d9c2e1f2e857d0dcf141dd5ea4d720dff5 (patch)
treef2eccc61f14b9159f7656e873b288ef2bbf38db7
parent200bf734bd459dd3c7a147b3dfe127dbf0ed7f70 (diff)
downloadsandcrawler-86cc15d9c2e1f2e857d0dcf141dd5ea4d720dff5.tar.gz
sandcrawler-86cc15d9c2e1f2e857d0dcf141dd5ea4d720dff5.zip
ingest: add a check for blocked-cookie before trying PDF url extraction
-rw-r--r--python/sandcrawler/ingest.py11
1 files changed, 11 insertions, 0 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 57988e8..ef4ca25 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -414,6 +414,17 @@ class IngestFileWorker(SandcrawlerWorker):
return result
file_meta = gen_file_metadata(resource.body)
+ if resource.terminal_url and ('/cookieAbsent' in next_url or 'cookieSet=1' in resource.terminal_url):
+ result['status'] = 'blocked-cookie'
+ result['terminal'] = {
+ "terminal_url": resource.terminal_url,
+ "terminal_dt": resource.terminal_dt,
+ "terminal_status_code": resource.terminal_status_code,
+ }
+ if resource.terminal_url not in result['hops']:
+ result['hops'].append(resource.terminal_url)
+ return result
+
# crude handling of content-encoding; wayback fetch library usually
# (and should always?) handle this
if file_meta['mimetype'] == 'application/gzip' and resource.cdx and resource.cdx.mimetype != 'application/gzip':