diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-11-13 18:22:18 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2019-11-13 18:47:57 -0800 |
| commit | 70d232ff43d34557ef73b117c4d4eb0dd9edc9d2 (patch) | |
| tree | 2e6e69ac2580da874d1839f8f4e125a54a0dc168 /python/sandcrawler/__init__.py | |
| parent | bbb8c03f9059d8c498d6a2f5933703c145931aeb (diff) | |
| download | sandcrawler-70d232ff43d34557ef73b117c4d4eb0dd9edc9d2.tar.gz sandcrawler-70d232ff43d34557ef73b117c4d4eb0dd9edc9d2.zip | |
have SPN client differentiate between SPN and remote errors
This is only a partial implementation. The requests client will still
make way too many SPN requests trying to figure out if this is a real
error or not (eg, if remote was a 502, we'll retry many times).
We may just want to switch to SPNv2 for everything.
Diffstat (limited to 'python/sandcrawler/__init__.py')
| -rw-r--r-- | python/sandcrawler/__init__.py | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/python/sandcrawler/__init__.py b/python/sandcrawler/__init__.py index e8fbcdf..c9cc0c9 100644 --- a/python/sandcrawler/__init__.py +++ b/python/sandcrawler/__init__.py @@ -2,6 +2,6 @@ from .grobid import GrobidClient, GrobidWorker, GrobidBlobWorker from .misc import gen_file_metadata, b32_hex, parse_cdx_line, parse_cdx_datetime from .workers import KafkaSink, KafkaGrobidSink, JsonLinePusher, CdxLinePusher, CdxLinePusher, KafkaJsonPusher, BlackholeSink, ZipfilePusher, MultiprocessWrapper -from .ia import WaybackClient, WaybackError, CdxApiClient, CdxApiError, SavePageNowClient, SavePageNowError +from .ia import WaybackClient, WaybackError, CdxApiClient, CdxApiError, SavePageNowClient, SavePageNowError, SavePageNowRemoteError from .ingest import IngestFileWorker |