From 855153ae4fe03656adde16c56a4347f4b3d26487 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 24 Oct 2022 10:19:46 -0700 Subject: html: worldscientific PDF URL extraction --- python/sandcrawler/html_metadata.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'python') diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py index 3d9e8ca..04a2f4a 100644 --- a/python/sandcrawler/html_metadata.py +++ b/python/sandcrawler/html_metadata.py @@ -709,6 +709,22 @@ PDF_FULLTEXT_PATTERNS: List[Dict[str, str]] = [ "technique": "OJS remote_pdf link", "example_page": "https://www.mediterranea-comunicacion.org/article/view/22240", }, + { + "in_doc_url": "worldscientific.com/doi/abs/", + "in_fulltext_url": "/reader/", + "selector": "article.container .single__download a", + "attr": "href", + "technique": "worldscientific landing pages", + "example_page": "https://www.worldscientific.com/doi/abs/10.1142/S0116110521500098", + }, + { + "in_doc_url": "worldscientific.com/doi/", + "in_fulltext_url": "/pdf/", + "selector": "noscript a[target='_blank']", + "attr": "href", + "technique": "worldscientific reader", + "example_page": "https://www.worldscientific.com/doi/epdf/10.1142/S0116110521500098", + }, ] FULLTEXT_URL_PATTERNS_SKIP: List[str] = [ -- cgit v1.2.3