aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-03-25 16:29:12 -0700
committerBryan Newbold <bnewbold@archive.org>2020-03-25 16:29:12 -0700
commitb91987978330b6da1c6bb5650418c2c6068340a1 (patch)
treef95415a7355724988d9bfd37b0abe33c8636faa2
parent11f7c22e1de104918d32274feaea310c11476cc7 (diff)
downloadsandcrawler-b91987978330b6da1c6bb5650418c2c6068340a1.tar.gz
sandcrawler-b91987978330b6da1c6bb5650418c2c6068340a1.zip
ingest: eurosurveillance PDF parser
-rw-r--r--python/sandcrawler/html.py11
1 files changed, 11 insertions, 0 deletions
diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py
index c76d7a2..04b3afe 100644
--- a/python/sandcrawler/html.py
+++ b/python/sandcrawler/html.py
@@ -310,4 +310,15 @@ def extract_fulltext_url(html_url, html_body):
if url and url.startswith('http'):
return dict(pdf_url=url, technique='figshare-json')
+ # eurosurveillance
+ # https://www.eurosurveillance.org/content/10.2807/1560-7917.ES.2020.25.11.2000230
+ if "://www.eurosurveillance.org/content/" in html_url:
+ # <a href="/deliver/fulltext/eurosurveillance/25/11/eurosurv-25-11-3.pdf?itemId=/content/10.2807/1560-7917.ES.2020.25.11.2000230&mimeType=pdf&containerItemId=content/eurosurveillance" class="pdf " title="Download" rel="http://instance.metastore.ingenta.com/content/10.2807/1560-7917.ES.2020.25.11.2000230" target="/content/10.2807/1560-7917.ES.2020.25.11.2000230-pdf" >
+ href = soup.find('a', attrs={"class":"pdf", "title": "Download"})
+ if href:
+ url = href['href'].strip()
+ if not url.startswith('http'):
+ url = host_prefix + url
+ return dict(pdf_url=url, technique='eurosurveillance-href')
+
return dict()