aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2019-11-13 18:22:18 -0800
committerBryan Newbold <bnewbold@archive.org>2019-11-13 18:47:57 -0800
commit70d232ff43d34557ef73b117c4d4eb0dd9edc9d2 (patch)
tree2e6e69ac2580da874d1839f8f4e125a54a0dc168
parentbbb8c03f9059d8c498d6a2f5933703c145931aeb (diff)
downloadsandcrawler-70d232ff43d34557ef73b117c4d4eb0dd9edc9d2.tar.gz
sandcrawler-70d232ff43d34557ef73b117c4d4eb0dd9edc9d2.zip
have SPN client differentiate between SPN and remote errors
This is only a partial implementation. The requests client will still make way too many SPN requests trying to figure out if this is a real error or not (eg, if remote was a 502, we'll retry many times). We may just want to switch to SPNv2 for everything.
-rw-r--r--python/sandcrawler/__init__.py2
-rw-r--r--python/sandcrawler/ia.py12
2 files changed, 11 insertions, 3 deletions
diff --git a/python/sandcrawler/__init__.py b/python/sandcrawler/__init__.py
index e8fbcdf..c9cc0c9 100644
--- a/python/sandcrawler/__init__.py
+++ b/python/sandcrawler/__init__.py
@@ -2,6 +2,6 @@
from .grobid import GrobidClient, GrobidWorker, GrobidBlobWorker
from .misc import gen_file_metadata, b32_hex, parse_cdx_line, parse_cdx_datetime
from .workers import KafkaSink, KafkaGrobidSink, JsonLinePusher, CdxLinePusher, CdxLinePusher, KafkaJsonPusher, BlackholeSink, ZipfilePusher, MultiprocessWrapper
-from .ia import WaybackClient, WaybackError, CdxApiClient, CdxApiError, SavePageNowClient, SavePageNowError
+from .ia import WaybackClient, WaybackError, CdxApiClient, CdxApiError, SavePageNowClient, SavePageNowError, SavePageNowRemoteError
from .ingest import IngestFileWorker
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 455c9f6..489736e 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -126,6 +126,9 @@ class WaybackClient:
class SavePageNowError(Exception):
pass
+class SavePageNowRemoteError(Exception):
+ pass
+
class SavePageNowClient:
def __init__(self, cdx_client=None,
@@ -156,13 +159,18 @@ class SavePageNowClient:
error on non-success.
"""
resp = self.http_session.get(self.v1endpoint + url)
- if resp.status_code != 200:
+ if resp.status_code != 200 and not resp.headers.get('X-Archive-Orig-Location'):
+ # looks like an error which was *not* a remote server error. Some
+ # problem with wayback, might need to short-circuit
raise SavePageNowError("HTTP status: {}, url: {}".format(resp.status_code, url))
+ if resp.headers.get('X-Archive-Wayback-Runtime-Error'):
+ # looks like a weird remote error; would not expect a CDX reply so bailing here
+ raise SavePageNowRemoteError(resp.headers['X-Archive-Wayback-Runtime-Error'])
terminal_url = '/'.join(resp.url.split('/')[5:])
body = resp.content
cdx = self.cdx_client.lookup_latest(terminal_url)
if not cdx:
- raise SavePageNowError("SPN was successful, but CDX lookup then failed")
+ raise SavePageNowError("SPN was successful, but CDX lookup then failed. URL: {}".format(terminal_url))
return (cdx, body)
def save_url_now_v2(self, url):