diff options
| -rw-r--r-- | python/sandcrawler/ia.py | 23 | ||||
| -rw-r--r-- | python/sandcrawler/ingest.py | 34 |
2 files changed, 42 insertions, 15 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index 27bbc34..cbf901a 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -266,9 +266,8 @@ class WaybackClient: """ if not self.petabox_webdata_secret: raise Exception("WaybackClient needs petabox secret to do direct WARC fetches") - # TODO: - #if not "/" in warc_path: - # raise ValueError("what looks like a liveweb/SPN temporary warc path: {}".format(warc_path)) + if not "/" in warc_path: + raise ValueError("what looks like a liveweb/SPN temporary warc path: {}".format(warc_path)) warc_uri = self.warc_uri_prefix + warc_path if not self.rstore: self.rstore = ResourceStore(loaderfactory=CDXLoaderFactory( @@ -278,6 +277,7 @@ class WaybackClient: #print("offset: {} csize: {} uri: {}".format(offset, csize, warc_uri), file=sys.stderr) gwb_record = self.rstore.load_resource(warc_uri, offset, csize) except wayback.exception.ResourceUnavailable: + print("Failed to fetch from warc_path:{}".format(warc_path), file=sys.stderr) raise PetaboxError("failed to load file contents from wayback/petabox (ResourceUnavailable)") except ValueError as ve: raise PetaboxError("failed to load file contents from wayback/petabox (ValueError: {})".format(ve)) @@ -347,8 +347,10 @@ class WaybackClient: assert datetime.isdigit() try: - # TODO: don't follow redirects? - resp = requests.get(self.wayback_endpoint + datetime + "id_/" + url) + resp = requests.get( + self.wayback_endpoint + datetime + "id_/" + url, + allow_redirects=False, + ) except requests.exceptions.TooManyRedirects: raise WaybackError("redirect loop (wayback replay fetch)") try: @@ -543,6 +545,7 @@ class SavePageNowClient: data={ 'url': request_url, 'capture_all': 1, + 'capture_screenshot': 0, 'if_not_archived_within': '1d', }, ) @@ -619,6 +622,16 @@ class SavePageNowClient: spn_result = self.save_url_now_v2(start_url) if not spn_result.success: + status = spn_result.status + if status in ("error:invalid-url", "error:not-found", + "error:invalid-host-resolution", "error:gateway-timeout"): + status = status.replace("error:", "") + elif status == "error:no-access": + status = "forbidden" + elif status == "error:user-session-limit": + raise Exception("SPNv2 user-session-limit, need to backoff") + elif status.startswith("error:"): + status = "spn2-" + status return ResourceResult( start_url=start_url, hit=False, diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 5dde205..f618f1b 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -85,13 +85,20 @@ class IngestFileWorker(SandcrawlerWorker): if self.try_wayback: via = "wayback" resource = self.wayback_client.lookup_resource(url, best_mimetype) - if self.try_spn2 and (not resource or not resource.hit): + + # check for "soft 404" conditions, where we should retry with live SPNv2 + # TODO: could refactor these into the resource fetch things themselves? + soft404 = False + if resource and resource.hit and resource.terminal_url.endswith('/cookieAbsent'): + soft404 = True + + if self.try_spn2 and (not resource or not resource.hit or soft404): via = "spn2" resource = self.spn_client.crawl_resource(url, self.wayback_client) - print("[FETCH {}\t] {}\turl:{}".format( + print("[FETCH {}\t] {}\t{}".format( via, resource.status, - url), + resource.terminal_url or url), file=sys.stderr) return resource @@ -141,10 +148,14 @@ class IngestFileWorker(SandcrawlerWorker): reqeust['ingest_type'] = 'pdf' # for now, only pdf ingest is implemented + if not 'ingest_type' in request: + request['ingest_type'] = "pdf" assert request.get('ingest_type') == "pdf" ingest_type = request.get('ingest_type') base_url = request['base_url'] + print("[INGEST {}\t] {}".format(ingest_type, base_url), file=sys.stderr) + best_mimetype = None if ingest_type == "pdf": best_mimetype = "application/pdf" @@ -157,7 +168,7 @@ class IngestFileWorker(SandcrawlerWorker): next_url = base_url hops = [base_url] - self.max_hops = 4 + self.max_hops = 6 while len(hops) <= self.max_hops: @@ -166,7 +177,7 @@ class IngestFileWorker(SandcrawlerWorker): try: resource = self.find_resource(next_url, best_mimetype) except SavePageNowError as e: - result['status'] = 'spn-error' + result['status'] = 'spn2-error' result['error_message'] = str(e) return result except PetaboxError as e: @@ -187,16 +198,14 @@ class IngestFileWorker(SandcrawlerWorker): return result file_meta = gen_file_metadata(resource.body) - if "html" in file_meta['mimetype']: - - # got landing page or similar + if "html" in file_meta['mimetype'] or "xml" in file_meta['mimetype']: + # Got landing page or similar. Some XHTML detected as "application/xml" if resource.terminal_dt: result['terminal'] = { "terminal_url": resource.terminal_url, "terminal_dt": resource.terminal_dt, "terminal_status_code": resource.terminal_status_code, } - fulltext_url = extract_fulltext_url(resource.terminal_url, resource.body) result['html'] = fulltext_url @@ -205,7 +214,7 @@ class IngestFileWorker(SandcrawlerWorker): return result next_url = fulltext_url.get('pdf_url') or fulltext_url.get('next_url') assert next_url - print("\tnext hop extracted ({}): {}".format( + print("[EXTRACT\t] {}\t{}".format( fulltext_url.get('technique'), next_url, ), @@ -252,6 +261,11 @@ class IngestFileWorker(SandcrawlerWorker): result['status'] = "success" result['hit'] = True + print("[SUCCESS\t] sha1:{} grobid:{}".format( + result.get('file_meta', {}).get('sha1hex'), + result.get('grobid', {}).get('status_code'), + ), + file=sys.stderr) return result |