diff options
| -rw-r--r-- | python/sandcrawler/ingest.py | 73 | ||||
| -rw-r--r-- | python/tests/test_ingest.py | 11 |
2 files changed, 57 insertions, 27 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 4b6c587..591c971 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -151,47 +151,70 @@ class IngestFileWorker(SandcrawlerWorker): result = dict(request=request, hit=False) - try: - # first hop - resource = self.find_resource(base_url, best_mimetype) + next_url = base_url + hops = [base_url] + self.max_hops = 4 + + + while len(hops) <= self.max_hops: + + result['hops'] = hops + try: + resource = self.find_resource(next_url, best_mimetype) + except SavePageNowError as e: + result['status'] = 'spn-error' + result['error_message'] = str(e) + return result + except PetaboxError as e: + result['status'] = 'petabox-error' + result['error_message'] = str(e) + return result + except CdxApiError as e: + result['status'] = 'cdx-error' + result['error_message'] = str(e) + return result + except WaybackError as e: + result['status'] = 'wayback-error' + result['error_message'] = str(e) + return result + if not resource.hit: result['status'] = resource.status return result file_meta = gen_file_metadata(resource.body) if "html" in file_meta['mimetype']: - # got landing page, try another hop + # got landing page or similar fulltext_url = extract_fulltext_url(resource.terminal_url, resource.body) result['html'] = fulltext_url if not fulltext_url or not 'pdf_url' in fulltext_url: result['status'] = 'no-pdf-link' + if resource.terminal_dt: + result['terminal'] = { + "terminal_url": resource.terminal_url, + "terminal_dt": resource.terminal_dt, + "terminal_status_code": resource.terminal_status_code, + } return result print("\tlanding page URL extracted ({}): {}".format( fulltext_url.get('technique'), fulltext_url['pdf_url'], ), file=sys.stderr) - resource = self.find_resource(fulltext_url['pdf_url'], best_mimetype) - if not resource.hit: - result['status'] = resource.status + next_url = fulltext_url['pdf_url'] + if next_url in hops: + result['status'] = 'link-loop' + result['error_message'] = "repeated: {}".format(next_url) return result - file_meta = gen_file_metadata(resource.body) - except SavePageNowError as e: - result['status'] = 'spn-error' - result['error_message'] = str(e) - return result - except PetaboxError as e: - result['status'] = 'petabox-error' - result['error_message'] = str(e) - return result - except CdxApiError as e: - result['status'] = 'cdx-error' - result['error_message'] = str(e) - return result - except WaybackError as e: - result['status'] = 'wayback-error' - result['error_message'] = str(e) + hops.append(next_url) + continue + + # default is to NOT keep hopping + break + + if len(hops) >= self.max_hops: + result['status'] = "max-hops-exceeded" return result if resource.terminal_dt: @@ -201,11 +224,12 @@ class IngestFileWorker(SandcrawlerWorker): "terminal_status_code": resource.terminal_status_code, } - # must be a hit if we got this far + # fetch must be a hit if we got this far (though not necessarily an ingest hit!) assert resource.hit == True assert resource.terminal_status_code == 200 result['file_meta'] = file_meta + result['cdx'] = cdx_to_dict(resource.cdx) # other failure cases if not resource.body or file_meta['size_bytes'] == 0: @@ -221,7 +245,6 @@ class IngestFileWorker(SandcrawlerWorker): result['status'] = "success" result['hit'] = True - result['cdx'] = cdx_to_dict(resource.cdx) return result diff --git a/python/tests/test_ingest.py b/python/tests/test_ingest.py index 8692b21..f5599e9 100644 --- a/python/tests/test_ingest.py +++ b/python/tests/test_ingest.py @@ -109,12 +109,19 @@ def test_ingest_landing(ingest_worker): headers={"X-Archive-Src": "liveweb-whatever.warc.gz"}, body=WARC_BODY) + # this is for second time around; don't want to fetch same landing page + # HTML again and result in a loop + responses.add(responses.GET, + 'https://web.archive.org/web/{}id_/{}'.format("20180326070330", TARGET + "/redirect"), + status=200, + headers={"X-Archive-Src": "liveweb-whatever.warc.gz"}, + body="<html></html>") + resp = ingest_worker.process(request) print(resp) assert resp['hit'] == False - assert resp['status'] == "wrong-mimetype" + assert resp['status'] == "no-pdf-link" assert resp['request'] == request assert 'grobid' not in resp - assert resp['terminal'] |