aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/ingest.py50
1 files changed, 35 insertions, 15 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index fb442d9..2c60b22 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -17,10 +17,11 @@ from sandcrawler.misc import gen_file_metadata, clean_url, parse_cdx_datetime
from sandcrawler.html import extract_fulltext_url
from sandcrawler.html_ingest import fetch_html_resources, \
quick_fetch_html_resources, html_guess_scope, html_extract_body_teixml, \
- WebResource
+ WebResource, html_guess_platform
from sandcrawler.html_metadata import html_extract_fulltext_url, \
- XML_FULLTEXT_PATTERNS, HTML_FULLTEXT_PATTERNS, BiblioMetadata, \
- html_extract_resources, html_extract_biblio, load_adblock_rules
+ XML_FULLTEXT_PATTERNS, HTML_FULLTEXT_PATTERNS, PDF_FULLTEXT_PATTERNS, \
+ BiblioMetadata, html_extract_resources, html_extract_biblio, \
+ load_adblock_rules
from sandcrawler.workers import SandcrawlerWorker
from sandcrawler.db import SandcrawlerPostgrestClient
from sandcrawler.xml import xml_reserialize
@@ -353,6 +354,7 @@ class IngestFileWorker(SandcrawlerWorker):
html_biblio = html_extract_biblio(resource.terminal_url, html_doc)
assert html_biblio
html_body = html_extract_body_teixml(resource.body)
+ html_platform = html_guess_platform(resource.terminal_url, html_doc, html_biblio)
html_scope = html_guess_scope(resource.terminal_url, html_doc, html_biblio, html_body.get('word_count'))
html_biblio_dict = json.loads(html_biblio.json(exclude_none=True))
@@ -360,14 +362,16 @@ class IngestFileWorker(SandcrawlerWorker):
return dict(
status=html_scope,
html_biblio=html_biblio_dict,
- html_scope=html_scope,
+ scope=html_scope,
+ platform=html_platform,
)
elif html_scope == 'unknown':
html_body.pop("tei_xml", None)
return dict(
status="unknown-scope",
html_biblio=html_biblio_dict,
- html_scope=html_scope,
+ scope=html_scope,
+ platform=html_platform,
html_body=html_body,
)
elif html_scope not in ('article-fulltext',):
@@ -375,7 +379,8 @@ class IngestFileWorker(SandcrawlerWorker):
return dict(
status="wrong-scope",
html_biblio=html_biblio_dict,
- html_scope=html_scope,
+ scope=html_scope,
+ platform=html_platform,
html_body=html_body,
)
@@ -385,7 +390,8 @@ class IngestFileWorker(SandcrawlerWorker):
return dict(
status="too-many-resources",
html_biblio=html_biblio_dict,
- html_scope=html_scope,
+ scope=html_scope,
+ platform=html_platform,
html_body=html_body,
)
@@ -396,7 +402,8 @@ class IngestFileWorker(SandcrawlerWorker):
partial_result = dict(
html_biblio=html_biblio_dict,
- html_scope=html_scope,
+ scope=html_scope,
+ platform=html_platform,
html_body=html_body,
)
@@ -434,6 +441,7 @@ class IngestFileWorker(SandcrawlerWorker):
html_body=html_body,
html_biblio=html_biblio_dict,
scope=html_scope,
+ platform=html_platform,
html_resources=[json.loads(r.json(exclude_none=True)) for r in full_resources],
)
@@ -586,18 +594,30 @@ class IngestFileWorker(SandcrawlerWorker):
or "text/xml" in file_meta['mimetype']
)
html_biblio = None
+ html_doc = None
if html_ish_resource and resource.body:
- html_doc = HTMLParser(resource.body)
- html_biblio = html_extract_biblio(resource.terminal_url, html_doc)
- if html_biblio and html_biblio.title:
- result['html_biblio'] = json.loads(html_biblio.json(exclude_none=True))
- #print(f" setting html_biblio: {result['html_biblio']}", file=sys.stderr)
+ try:
+ html_doc = HTMLParser(resource.body)
+ html_biblio = html_extract_biblio(resource.terminal_url, html_doc)
+ if html_biblio:
+ if not 'html_biblio' in result or html_biblio.title:
+ result['html_biblio'] = json.loads(html_biblio.json(exclude_none=True))
+ #print(f" setting html_biblio: {result['html_biblio']}", file=sys.stderr)
+ except ValueError:
+ pass
if ingest_type == "pdf" and html_ish_resource:
- # Got landing page or similar. Some XHTML detected as "application/xml"
+ # Got landing page or similar
fulltext_url = extract_fulltext_url(resource.terminal_url, resource.body)
- result['extract_next_hop'] = fulltext_url
+ # this is the new style of URL extraction
+ if not fulltext_url and html_biblio and html_biblio.pdf_fulltext_url:
+ fulltext_url = dict(
+ pdf_url=html_biblio.pdf_fulltext_url,
+ technique="html_biblio",
+ )
+
+ result['extract_next_hop'] = fulltext_url
if not fulltext_url:
result['status'] = 'no-pdf-link'
return result