aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/ingest.py40
1 files changed, 31 insertions, 9 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index d95b8bf..fb442d9 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -343,19 +343,40 @@ class IngestFileWorker(SandcrawlerWorker):
def process_html(self, resource: ResourceResult, file_meta: dict) -> dict:
- html_doc = HTMLParser(resource.body)
+ assert resource.body
+ try:
+ html_doc = HTMLParser(resource.body)
+ except ValueError as ve:
+ return dict(
+ status="html-selectolax-error",
+ )
html_biblio = html_extract_biblio(resource.terminal_url, html_doc)
assert html_biblio
html_body = html_extract_body_teixml(resource.body)
html_scope = html_guess_scope(resource.terminal_url, html_doc, html_biblio, html_body.get('word_count'))
html_biblio_dict = json.loads(html_biblio.json(exclude_none=True))
- if html_scope not in ('article-fulltext', 'unknown'):
+ if html_scope in ('blocked-captcha','blocked-cookie','blocked-forbidden'):
+ return dict(
+ status=html_scope,
+ html_biblio=html_biblio_dict,
+ html_scope=html_scope,
+ )
+ elif html_scope == 'unknown':
+ html_body.pop("tei_xml", None)
+ return dict(
+ status="unknown-scope",
+ html_biblio=html_biblio_dict,
+ html_scope=html_scope,
+ html_body=html_body,
+ )
+ elif html_scope not in ('article-fulltext',):
html_body.pop("tei_xml", None)
return dict(
status="wrong-scope",
html_biblio=html_biblio_dict,
html_scope=html_scope,
+ html_body=html_body,
)
raw_resources = html_extract_resources(resource.terminal_url, html_doc, self.adblock_rules)
@@ -365,17 +386,23 @@ class IngestFileWorker(SandcrawlerWorker):
status="too-many-resources",
html_biblio=html_biblio_dict,
html_scope=html_scope,
+ html_body=html_body,
)
- when = parse_cdx_datetime(resource.cdx.datetime)
+ if self.htmlteixml_sink and html_body['status'] == "success":
+ self.htmlteixml_sink.push_record(html_body, key=file_meta['sha1hex'])
- full_resources: List[WebResource] = []
+ html_body.pop("tei_xml", None)
partial_result = dict(
html_biblio=html_biblio_dict,
html_scope=html_scope,
+ html_body=html_body,
)
+ when = parse_cdx_datetime(resource.cdx.datetime)
+ full_resources: List[WebResource] = []
+
try:
if self.html_quick_mode:
print(" WARN: running quick CDX-only fetches", file=sys.stderr)
@@ -403,11 +430,6 @@ class IngestFileWorker(SandcrawlerWorker):
partial_result['error_message'] = str(e)[:1600]
return partial_result
- if self.htmlteixml_sink and html_body['status'] == "success":
- self.htmlteixml_sink.push_record(html_body, key=file_meta['sha1hex'])
-
- html_body.pop("tei_xml", None)
-
return dict(
html_body=html_body,
html_biblio=html_biblio_dict,