aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/html.py11
1 files changed, 10 insertions, 1 deletions
diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py
index acf1522..75e111e 100644
--- a/python/sandcrawler/html.py
+++ b/python/sandcrawler/html.py
@@ -95,13 +95,22 @@ def extract_fulltext_url(html_url, html_body):
# https://pubs.acs.org/doi/10.1021/acs.estlett.9b00379
# <a href="/doi/pdf/10.1021/acs.estlett.9b00379" title="PDF" target="_blank" class="button_primary"><i class="icon-file-pdf-o"></i><span>PDF (1 MB)</span></a>
href = soup.find('a', attrs={"title":"PDF"})
- if href and 'href' in href:
+ if href and href.get('href'):
url = href['href'].strip()
if url.startswith('http'):
return dict(pdf_url=url, technique='href_title')
elif url.startswith('/'):
return dict(pdf_url=host_prefix+url, technique='href_title')
+ # http://www.revistas.unam.mx/index.php/rep/article/view/35503/32336
+ href = soup.find('a', attrs={"id":"pdfDownloadLink"})
+ if href and href.get('href'):
+ url = href['href'].strip()
+ if url.startswith('http'):
+ return dict(pdf_url=url, technique='href_pdfDownloadLink')
+ elif url.startswith('/'):
+ return dict(pdf_url=host_prefix+url, technique='href_pdfDownloadLink')
+
# http://www.jasstudies.com/DergiTamDetay.aspx?ID=3401
# <embed src="/files/jass_makaleler/1359848334_33-Okt.%20Yasemin%20KARADEM%C4%B0R.pdf" type="application/pdf" />
embed = soup.find('embed', attrs={"type": "application/pdf"})