aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/html_ingest.py29
-rw-r--r--python/sandcrawler/html_metadata.py5
-rw-r--r--python/sandcrawler/ingest.py55
-rw-r--r--python/sandcrawler/persist.py20
4 files changed, 94 insertions, 15 deletions
diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py
index fe883ba..11909e6 100644
--- a/python/sandcrawler/html_ingest.py
+++ b/python/sandcrawler/html_ingest.py
@@ -76,21 +76,36 @@ class IngestWebResult(pydantic.BaseModel):
datetime.datetime: lambda dt: dt.isoformat(),
}
+class HtmlMetaRow(pydantic.BaseModel):
+ sha1hex: str
+ status: str
+ scope: Optional[str]
+ has_teixml: bool
+ has_thumbnail: bool
+ word_count: Optional[int]
+ biblio: Optional[dict]
+ resources: Optional[List[dict]]
+
+ class Config:
+ arbitrary_types_allowed = True
+ json_encoders = {
+ datetime.datetime: lambda dt: dt.isoformat(),
+ }
+
def to_sql_tuple(self) -> Tuple:
"""
This is for the html_meta SQL table.
"""
- assert self.file_meta and "sha1hex" in self.file_meta
return (
- self.file_meta["sha1hex"],
+ self.sha1hex,
datetime.datetime.now(), # updated
self.status,
self.scope,
- bool(self.html_body and self.html_body['status'] == 'success' and self.html_body['tei_xml']),
- False, # has_thumbnail
- (self.html_body and self.html_body.get('word_count')) or None,
- self.html_biblio,
- self.html_resources,
+ self.has_teixml,
+ self.has_thumbnail,
+ self.word_count,
+ self.biblio and json.dumps(self.biblio, sort_keys=True),
+ self.resources and json.dumps(self.resources, sort_keys=True),
)
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index 3ebba57..8928978 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -248,6 +248,11 @@ class BiblioMetadata(pydantic.BaseModel):
html_fulltext_url: Optional[str]
xml_fulltext_url: Optional[str]
+ class Config:
+ json_encoders = {
+ datetime.date: lambda dt: dt.isoformat()
+ }
+
def html_extract_fulltext_url(doc_url: str, doc: HTMLParser, patterns: List[dict]) -> Optional[Tuple[str, str]]:
"""
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 2e227bf..cc64fa5 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -5,7 +5,7 @@ import gzip
import time
import base64
import requests
-from typing import Optional, Tuple, Any, Dict
+from typing import Optional, Tuple, Any, Dict, List
from http.server import BaseHTTPRequestHandler, HTTPServer
from collections import namedtuple
from selectolax.parser import HTMLParser
@@ -13,9 +13,14 @@ from selectolax.parser import HTMLParser
from sandcrawler.ia import SavePageNowClient, CdxApiClient, WaybackClient, WaybackError, WaybackContentError, SavePageNowError, CdxApiError, PetaboxError, cdx_to_dict, ResourceResult, fix_transfer_encoding
from sandcrawler.grobid import GrobidClient
from sandcrawler.pdfextract import process_pdf, PdfExtractResult
-from sandcrawler.misc import gen_file_metadata, clean_url
+from sandcrawler.misc import gen_file_metadata, clean_url, parse_cdx_datetime
from sandcrawler.html import extract_fulltext_url
-from sandcrawler.html_metadata import html_extract_fulltext_url, XML_FULLTEXT_PATTERNS
+from sandcrawler.html_ingest import fetch_html_resources, \
+ quick_fetch_html_resources, html_guess_scope, html_extract_body_teixml, \
+ WebResource
+from sandcrawler.html_metadata import html_extract_fulltext_url, \
+ XML_FULLTEXT_PATTERNS, BiblioMetadata, html_extract_resources, \
+ html_extract_biblio, load_adblock_rules
from sandcrawler.workers import SandcrawlerWorker
from sandcrawler.db import SandcrawlerPostgrestClient
from sandcrawler.xml import xml_reserialize
@@ -75,6 +80,8 @@ class IngestFileWorker(SandcrawlerWorker):
self.try_existing_pdfextract = kwargs.get('try_existing_pdfextract', True)
self.try_wayback = kwargs.get('try_wayback', True)
self.try_spn2 = kwargs.get('try_spn2', True)
+ self.html_quick_mode = False
+ self.adblock_rules = load_adblock_rules()
self.base_url_blocklist = [
# robot blocking
@@ -247,6 +254,8 @@ class IngestFileWorker(SandcrawlerWorker):
return {
'xml_meta': self.process_xml(resource, file_meta),
}
+ elif ingest_type == "html":
+ return self.process_html(resource, file_meta)
else:
raise NotImplementedError(f"process {ingest_type} hit")
@@ -326,6 +335,33 @@ class IngestFileWorker(SandcrawlerWorker):
self.xmldoc_sink.push_record(msg, key=file_meta['sha1hex'])
return dict(status="success")
+ def process_html(self, resource: ResourceResult, file_meta: dict) -> dict:
+
+ html_doc = HTMLParser(resource.body)
+ html_biblio = html_extract_biblio(resource.terminal_url, html_doc)
+ html_body = html_extract_body_teixml(resource.body)
+ html_scope = html_guess_scope(resource.terminal_url, html_doc, html_biblio, html_body.get('tei_xml'))
+
+ assert html_biblio
+
+ raw_resources = html_extract_resources(resource.terminal_url, html_doc, self.adblock_rules)
+ assert len(raw_resources) <= 200
+
+ when = parse_cdx_datetime(resource.cdx.datetime)
+
+ full_resources: List[WebResource] = []
+ if self.html_quick_mode:
+ full_resources = quick_fetch_html_resources(raw_resources, self.wayback_client.cdx_client, when)
+ else:
+ full_resources = fetch_html_resources(raw_resources, self.wayback_client, when)
+
+ return dict(
+ html_body=html_body,
+ html_biblio=json.loads(html_biblio.json(exclude_none=True)),
+ scope=html_scope,
+ html_resources=[json.loads(r.json(exclude_none=True)) for r in full_resources],
+ )
+
def timeout_response(self, task: dict) -> dict:
print("[TIMEOUT]", file=sys.stderr)
return dict(
@@ -336,7 +372,7 @@ class IngestFileWorker(SandcrawlerWorker):
)
def want(self, request: dict) -> bool:
- if not request.get('ingest_type') in ('file', 'pdf', 'xml'):
+ if not request.get('ingest_type') in ('file', 'pdf', 'xml', 'html'):
return False
return True
@@ -347,7 +383,7 @@ class IngestFileWorker(SandcrawlerWorker):
request['ingest_type'] = 'pdf'
ingest_type = request.get('ingest_type')
- if ingest_type not in ("pdf", "xml"):
+ if ingest_type not in ("pdf", "xml", "html"):
raise NotImplementedError(f"can't handle ingest_type={ingest_type}")
# parse/clean URL
@@ -541,12 +577,21 @@ class IngestFileWorker(SandcrawlerWorker):
if file_meta['mimetype'] not in ("application/xml", "text/xml", "application/jats+xml"):
result['status'] = "wrong-mimetype"
return result
+ elif ingest_type == "html":
+ if file_meta['mimetype'] not in ("text/html",):
+ result['status'] = "wrong-mimetype"
+ return result
else:
raise NotImplementedError()
info = self.process_hit(ingest_type, resource, file_meta)
result.update(info)
+ # scope is getting calculated in process_hit()
+ if result.get('scope') and result['scope'] not in ('article-fulltext', 'unknown'):
+ result['status'] = "wrong-scope"
+ return result
+
result['status'] = "success"
result['hit'] = True
if ingest_type == "pdf":
diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py
index c225d5a..033bc91 100644
--- a/python/sandcrawler/persist.py
+++ b/python/sandcrawler/persist.py
@@ -28,6 +28,7 @@ from sandcrawler.db import SandcrawlerPostgresClient
from sandcrawler.minio import SandcrawlerMinioClient
from sandcrawler.grobid import GrobidClient
from sandcrawler.pdfextract import PdfExtractResult
+from sandcrawler.html_ingest import HtmlMetaRow
class PersistCdxWorker(SandcrawlerWorker):
@@ -159,8 +160,21 @@ class PersistIngestFileResultWorker(SandcrawlerWorker):
result['terminal_sha1hex'] = terminal.get('terminal_sha1hex')
return result
- def result_to_html_meta(self, record: dict) -> Optional[dict]:
- raise NotImplementedError()
+ def result_to_html_meta(self, record: dict) -> Optional[HtmlMetaRow]:
+ html_body = record.get('html_body')
+ file_meta = record.get('file_meta')
+ if not (file_meta and html_body):
+ return None
+ return HtmlMetaRow(
+ sha1hex=file_meta["sha1hex"],
+ status=record.get('status'),
+ scope=record.get('scope'),
+ has_teixml=bool(html_body and html_body['status'] == 'success' and html_body.get('tei_xml')),
+ has_thumbnail=False, # TODO
+ word_count=(html_body and html_body.get('word_count')) or None,
+ biblio=record.get('html_biblio'),
+ resources=record.get('html_resources'),
+ )
def push_batch(self, batch):
self.counts['total'] += len(batch)
@@ -200,7 +214,7 @@ class PersistIngestFileResultWorker(SandcrawlerWorker):
self.counts['insert-file_meta'] += resp[0]
self.counts['update-file_meta'] += resp[1]
- html_meta_batch = [self.result_to_html_meta(r) for r in batch if r.get('hit') and r.get('html_meta')]
+ html_meta_batch = [self.result_to_html_meta(r) for r in batch if r.get('hit') and r.get('html_body')]
if html_meta_batch:
resp = self.db.insert_html_meta(self.cur, html_meta_batch, on_conflict="nothing")
self.counts['insert-html_meta'] += resp[0]