diff options
author | Bryan Newbold <bnewbold@archive.org> | 2020-10-12 13:06:31 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2020-10-12 13:06:31 -0700 |
commit | f5286affa62c0caf95ff4848626e85aff476737e (patch) | |
tree | 93e2dcb550fa8a057bc02b38e34d32ce0ee0e561 /python/sandcrawler/ingest.py | |
parent | 2469d483326e5e81e774e46fd100888710b9bbc3 (diff) | |
download | sandcrawler-f5286affa62c0caf95ff4848626e85aff476737e.tar.gz sandcrawler-f5286affa62c0caf95ff4848626e85aff476737e.zip |
store no-capture URLs in terminal_url
Diffstat (limited to 'python/sandcrawler/ingest.py')
-rw-r--r-- | python/sandcrawler/ingest.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index e8e517a..5ab7e13 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -387,7 +387,7 @@ class IngestFileWorker(SandcrawlerWorker): if not resource.hit: result['status'] = resource.status - if resource.terminal_dt and resource.terminal_status_code: + if resource.terminal_url: result['terminal'] = { "terminal_url": resource.terminal_url, "terminal_dt": resource.terminal_dt, @@ -465,7 +465,7 @@ class IngestFileWorker(SandcrawlerWorker): result['status'] = "max-hops-exceeded" return result - if resource.terminal_dt: + if resource.terminal_url: result['terminal'] = { "terminal_url": resource.terminal_url, "terminal_dt": resource.terminal_dt, |