aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-11-08 13:20:00 -0800
committerBryan Newbold <bnewbold@archive.org>2020-11-08 13:20:00 -0800
commite87f28d62e7e92de5502cd04d8c3effb19da19f8 (patch)
tree787d61b379e36c3c86793eed50871166cfcc9128
parent2fdd892f8c4a56247840fcb2e64c07f518b4bfbb (diff)
downloadsandcrawler-e87f28d62e7e92de5502cd04d8c3effb19da19f8.tar.gz
sandcrawler-e87f28d62e7e92de5502cd04d8c3effb19da19f8.zip
html: missing fetch is wayback-content-error, not wayback-error
-rw-r--r--python/sandcrawler/html_ingest.py4
1 files changed, 2 insertions, 2 deletions
diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py
index 6b71115..6b9497b 100644
--- a/python/sandcrawler/html_ingest.py
+++ b/python/sandcrawler/html_ingest.py
@@ -11,7 +11,7 @@ import trafilatura
import pydantic
from selectolax.parser import HTMLParser
-from sandcrawler.ia import WaybackClient, CdxApiClient, ResourceResult, cdx_to_dict, fix_transfer_encoding, NoCaptureError
+from sandcrawler.ia import WaybackClient, CdxApiClient, ResourceResult, cdx_to_dict, fix_transfer_encoding, NoCaptureError, WaybackContentError
from sandcrawler.misc import gen_file_metadata, parse_cdx_datetime, datetime_to_cdx
from sandcrawler.html_metadata import BiblioMetadata, html_extract_resources, html_extract_biblio, load_adblock_rules
@@ -158,7 +158,7 @@ def fetch_html_resources(resources: List[dict], wayback_client: WaybackClient, w
raise NoCaptureError(f"HTML sub-resource not found: {resource['url']}")
file_meta = gen_file_metadata(wayback_resp.body)
if file_meta['sha1hex'] != wayback_resp.cdx.sha1hex:
- raise WaybackError("wayback payload sha1hex mismatch: {wayback_resp.cdx.url}")
+ raise WaybackContentError("wayback payload sha1hex mismatch: {wayback_resp.cdx.url}")
full.append(WebResource(
surt=wayback_resp.cdx.surt,
timestamp=parse_cdx_datetime(wayback_resp.cdx.datetime),