diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/sandcrawler/ingest.py | 41 |
1 files changed, 17 insertions, 24 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 2c60b22..2f6be05 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -18,10 +18,7 @@ from sandcrawler.html import extract_fulltext_url from sandcrawler.html_ingest import fetch_html_resources, \ quick_fetch_html_resources, html_guess_scope, html_extract_body_teixml, \ WebResource, html_guess_platform -from sandcrawler.html_metadata import html_extract_fulltext_url, \ - XML_FULLTEXT_PATTERNS, HTML_FULLTEXT_PATTERNS, PDF_FULLTEXT_PATTERNS, \ - BiblioMetadata, html_extract_resources, html_extract_biblio, \ - load_adblock_rules +from sandcrawler.html_metadata import BiblioMetadata, html_extract_resources, html_extract_biblio, load_adblock_rules from sandcrawler.workers import SandcrawlerWorker from sandcrawler.db import SandcrawlerPostgrestClient from sandcrawler.xml import xml_reserialize @@ -607,21 +604,23 @@ class IngestFileWorker(SandcrawlerWorker): pass if ingest_type == "pdf" and html_ish_resource: - # Got landing page or similar + fulltext_url = extract_fulltext_url(resource.terminal_url, resource.body) - # this is the new style of URL extraction - if not fulltext_url and html_biblio and html_biblio.pdf_fulltext_url: - fulltext_url = dict( - pdf_url=html_biblio.pdf_fulltext_url, - technique="html_biblio", - ) + # the new style of URL extraction (already computed) + # we aren't quite ready to adopt this for the PDF path (which + # has more complex logic to avoid loops, etc) + #if not fulltext_url and html_biblio and html_biblio.pdf_fulltext_url: + # fulltext_url = dict( + # pdf_url=html_biblio.pdf_fulltext_url, + # technique="html_biblio", + # ) result['extract_next_hop'] = fulltext_url if not fulltext_url: result['status'] = 'no-pdf-link' return result - next_url = fulltext_url.get('pdf_url') or fulltext_url.get('next_url') + next_url = fulltext_url.get('pdf_url') or fulltext_url.get('next_url') or "" assert next_url next_url = clean_url(next_url) print("[PARSE {:>6}] {} {}".format( @@ -637,12 +636,9 @@ class IngestFileWorker(SandcrawlerWorker): hops.append(next_url) continue elif ingest_type == "xml" and html_ish_resource: - # parse with selectolax, extract XML fulltext URL - html_doc = HTMLParser(resource.body) - extract_next_hop = html_extract_fulltext_url(resource.terminal_url, html_doc, XML_FULLTEXT_PATTERNS) - if extract_next_hop: - next_url = extract_next_hop[0] - technique = extract_next_hop[1] + if html_biblio and html_biblio.xml_fulltext_url: + next_url = html_biblio.xml_fulltext_url + technique = "html_biblio" print("[PARSE {:>6}] {} {}".format( ingest_type, technique, @@ -656,12 +652,9 @@ class IngestFileWorker(SandcrawlerWorker): hops.append(next_url) continue elif ingest_type == "html" and html_ish_resource: - # parse with selectolax, extract XML fulltext URL - html_doc = HTMLParser(resource.body) - extract_next_hop = html_extract_fulltext_url(resource.terminal_url, html_doc, HTML_FULLTEXT_PATTERNS) - if extract_next_hop: - next_url = extract_next_hop[0] - technique = extract_next_hop[1] + if html_biblio and html_biblio.html_fulltext_url: + next_url = html_biblio.html_fulltext_url + technique = "html_bibli" if next_url in hops: # for HTML ingest, we don't count this as a link-loop break |