diff options
author | Bryan Newbold <bnewbold@archive.org> | 2020-01-15 13:52:42 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2020-01-15 13:52:42 -0800 |
commit | 4d0224f3e73315ef4db39643e6d4851e4a466658 (patch) | |
tree | 43f22e098e565ae67a515d1a17bca5d6f632ae76 /python/sandcrawler/ingest.py | |
parent | 2d052b610ed02341aebab865f174671f8381146e (diff) | |
download | sandcrawler-4d0224f3e73315ef4db39643e6d4851e4a466658.tar.gz sandcrawler-4d0224f3e73315ef4db39643e6d4851e4a466658.zip |
pass through revisit_cdx
Diffstat (limited to 'python/sandcrawler/ingest.py')
-rw-r--r-- | python/sandcrawler/ingest.py | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index e5eb6e8..de5e957 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -239,6 +239,7 @@ class IngestFileWorker(SandcrawlerWorker): if not resource.hit: result['status'] = resource.status return result + file_meta = gen_file_metadata(resource.body) if "html" in file_meta['mimetype'] or "xml" in file_meta['mimetype']: @@ -289,6 +290,8 @@ class IngestFileWorker(SandcrawlerWorker): result['file_meta'] = file_meta result['cdx'] = cdx_to_dict(resource.cdx) + if resource.revisit_cdx: + result['revisit_cdx'] = cdx_to_dict(resource.revisit_cdx) # other failure cases if not resource.body or file_meta['size_bytes'] == 0: |