aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-11-08 14:28:37 -0800
committerBryan Newbold <bnewbold@archive.org>2020-11-08 14:28:37 -0800
commit6431b2f6b4bd9bd4dea4b373b89eb3f89648cc4c (patch)
treea97bdff5e392854fbb779ee179d851319426166f
parent5d525e9744303bf5ddcf673623483d4a6a787326 (diff)
downloadsandcrawler-6431b2f6b4bd9bd4dea4b373b89eb3f89648cc4c.tar.gz
sandcrawler-6431b2f6b4bd9bd4dea4b373b89eb3f89648cc4c.zip
html: small ingest improvements
-rw-r--r--python/sandcrawler/html_ingest.py4
-rw-r--r--python/sandcrawler/html_metadata.py15
2 files changed, 19 insertions, 0 deletions
diff --git a/python/sandcrawler/html_ingest.py b/python/sandcrawler/html_ingest.py
index 42bd946..a8ba0d6 100644
--- a/python/sandcrawler/html_ingest.py
+++ b/python/sandcrawler/html_ingest.py
@@ -127,6 +127,10 @@ def quick_fetch_html_resources(resources: List[dict], cdx_client: CdxApiClient,
raise NoCaptureError(f"HTML sub-resource not found: {resource['url']}")
if cdx_row.url != resource['url']:
print(f" WARN: CDX fuzzy match: {cdx_row.url} != {resource['url']}", file=sys.stderr)
+ if not cdx_row.status_code:
+ # TODO: fall back to a full fetch?
+ print(f" WARN: skipping revisit record", file=sys.stderr)
+ continue
full.append(WebResource(
surt=cdx_row.surt,
timestamp=cdx_row.datetime,
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index 0d14166..cd49a05 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -195,6 +195,13 @@ XML_FULLTEXT_PATTERNS: List[dict] = [
"attr": "href",
"technique": "SciElo XML link",
},
+ {
+ "in_doc_url": "/article/view/",
+ "in_fulltext_url": "viewXML",
+ "selector": "a[class='obj_galley_link']",
+ "attr": "href",
+ "technique": "OJS Gallery XML link",
+ },
]
HTML_FULLTEXT_PATTERNS: List[dict] = [
@@ -203,6 +210,13 @@ HTML_FULLTEXT_PATTERNS: List[dict] = [
"attr": "content",
"technique": "citation_fulltext_html_url",
},
+ {
+ "in_doc_url": "/article/view/",
+ "in_fulltext_url": "inline=1",
+ "selector": "iframe[name='htmlFrame']",
+ "attr": "src",
+ "technique": "OJS HTML iframe",
+ },
]
PDF_FULLTEXT_PATTERNS: List[dict] = [
@@ -393,6 +407,7 @@ def load_adblock_rules() -> braveblock.Adblocker:
# badges, "share" buttons, etc
"apis.google.com/js/plusone",
+ "www.google.com/recaptcha/",
# PLOS images
"/resource/img/icon.*.16.png^",