diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/sandcrawler/html.py | 100 |
1 files changed, 96 insertions, 4 deletions
diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py index 25587e6..812335f 100644 --- a/python/sandcrawler/html.py +++ b/python/sandcrawler/html.py @@ -7,6 +7,15 @@ from bs4 import BeautifulSoup RESEARCHSQUARE_REGEX = re.compile(r'"url":"(https://assets.researchsquare.com/files/.{1,50}/v\d+/Manuscript.pdf)"') IEEEXPLORE_REGEX = re.compile(r'"pdfPath":"(/.*?\.pdf)"') +OVID_JOURNAL_URL_REGEX = re.compile(r'journalURL = "(http.*)";') + +def test_regex(): + lines = """ + blah + var journalURL = "https://journals.lww.com/co-urology/fulltext/10.1097/MOU.0000000000000689"; + asdf""" + m = OVID_JOURNAL_URL_REGEX.search(lines) + assert m.group(1) == "https://journals.lww.com/co-urology/fulltext/10.1097/MOU.0000000000000689" def extract_fulltext_url(html_url, html_body): """ @@ -23,7 +32,8 @@ def extract_fulltext_url(html_url, html_body): meta = soup.find('meta', attrs={"name":"citation_pdf_url"}) if not meta: meta = soup.find('meta', attrs={"name":"bepress_citation_pdf_url"}) - if meta: + # wiley has a weird almost-blank page we don't want to loop on + if meta and not "://onlinelibrary.wiley.com/doi/pdf/" in html_url: url = meta['content'].strip() if url.startswith('/'): return dict(pdf_url=host_prefix+url, technique='citation_pdf_url') @@ -32,6 +42,16 @@ def extract_fulltext_url(html_url, html_body): else: print("malformed citation_pdf_url? {}".format(url), file=sys.stderr) + # sage, and also utpjournals (see below) + # https://journals.sagepub.com/doi/10.1177/2309499019888836 + # <a href="http://journals.sagepub.com/doi/pdf/10.1177/2309499019888836" class="show-pdf" target="_self"> + # <a href="http://utpjournals.press/doi/pdf/10.3138/cjh.ach.54.1-2.05" class="show-pdf" target="_blank"> + href = soup.find('a', attrs={"class":"show-pdf"}) + if href: + url = href['href'].strip() + if url.startswith('http'): + return dict(pdf_url=url, technique='href_show-pdf') + # ACS (and probably others) like: # https://pubs.acs.org/doi/10.1021/acs.estlett.9b00379 # <a href="/doi/pdf/10.1021/acs.estlett.9b00379" title="PDF" target="_blank" class="button_primary"><i class="icon-file-pdf-o"></i><span>PDF (1 MB)</span></a> @@ -43,6 +63,16 @@ def extract_fulltext_url(html_url, html_body): elif url.startswith('/'): return dict(pdf_url=host_prefix+url, technique='href_title') + # http://www.jasstudies.com/DergiTamDetay.aspx?ID=3401 + # <embed src="/files/jass_makaleler/1359848334_33-Okt.%20Yasemin%20KARADEM%C4%B0R.pdf" type="application/pdf" /> + embed = soup.find('embed', attrs={"type": "application/pdf"}) + if embed: + url = embed['src'].strip() + if url.startswith('/'): + url = host_prefix+url + if url.startswith('http'): + return dict(pdf_url=url, technique='embed_type') + ### Publisher/Platform Specific ### # eLife (elifesciences.org) @@ -64,13 +94,17 @@ def extract_fulltext_url(html_url, html_body): return dict(release_stage="manuscript", pdf_url=url, technique='publisher') # elseiver linking hub + # https://linkinghub.elsevier.com/retrieve/pii/S1569199319308975 if '://linkinghub.elsevier.com/retrieve/pii/' in html_url: + # <input type="hidden" name="redirectURL" value="http%3A%2F%2Fcysticfibrosisjournal.com%2Fretrieve%2Fpii%2FS1569199319308975" id="redirectURL"/> redirect = soup.find("input", attrs={"name": "redirectURL"}) if redirect: url = redirect['value'].strip() - if 'sciencedirect.com' in url: + if 'http' in url: url = urllib.parse.unquote(url) - return dict(next_url=url, technique="publisher") + # drop any the query parameter + url = url.split('?via')[0] + return dict(next_url=url, technique="elsevier-linkinghub") # ieeexplore.ieee.org # https://ieeexplore.ieee.org/document/8730316 @@ -90,6 +124,64 @@ def extract_fulltext_url(html_url, html_body): if iframe and '.pdf' in iframe['src']: return dict(pdf_url=iframe['src'], technique="iframe") - # TODO: hrmars.com. anchor with .pdf href, and anchor text is "PDF" + # utpjournals.press + # https://utpjournals.press/doi/10.3138/cjh.ach.54.1-2.05 + if '://utpjournals.press/doi/10.' in html_url: + # <a href="http://utpjournals.press/doi/pdf/10.3138/cjh.ach.54.1-2.05" class="show-pdf" target="_blank"> + href = soup.find('a', attrs={"class":"show-pdf"}) + if href: + url = href['href'].strip() + if url.startswith('http'): + return dict(pdf_url=url, technique='publisher-href') + + # https://www.jcancer.org/v10p4038.htm + # simple journal-specific href + if '://www.jcancer.org/' in html_url and html_url.endswith(".htm"): + # <a href='v10p4038.pdf' class='textbutton'>PDF</a> + href = soup.find('a', attrs={"class":"textbutton"}) + if href: + url = href['href'].strip() + if url.endswith(".pdf") and not "http" in url: + return dict(pdf_url=host_prefix+"/"+url, technique='journal-href') + + # https://insights.ovid.com/crossref?an=00042307-202001000-00013 + # Ovid is some kind of landing page bounce portal tracking run-around. + # Can extract actual journal URL from javascript blob in the HTML + if '://insights.ovid.com/crossref' in html_url: + # var journalURL = "https://journals.lww.com/co-urology/fulltext/10.1097/MOU.0000000000000689"; + m = OVID_JOURNAL_URL_REGEX.search(html_body.decode('utf-8')) + if m: + url = m.group(1) + assert len(url) < 1024 + return dict(next_url=url, technique='ovid') + + # osf.io + # https://osf.io/8phvx/ + # https://osf.io/preprints/socarxiv/8phvx/ + # wow, they ship total javascript crud! going to just guess download URL + # based on URL for now. Maybe content type header would help? + if '://osf.io/' in html_url and not '/download' in html_url: + if not html_url.endswith("/"): + next_url = html_url+"/download" + else: + next_url = html_url+"download" + return dict(next_url=next_url, technique='osf-by-url') + + # wiley + # https://onlinelibrary.wiley.com/doi/pdf/10.1111/1467-923X.12787 + if "://onlinelibrary.wiley.com/doi/pdf/" in html_url: + if "/doi/pdfdirect/" in html_body: + next_url = html_url.replace('/doi/pdf/', '/doi/pdfdirect/') + return dict(next_url=next_url, technique='wiley-pdfdirect') + + # taylor and frances + # https://www.tandfonline.com/doi/full/10.1080/19491247.2019.1682234 + # <a href="/doi/pdf/10.1080/19491247.2019.1682234?needAccess=true" class="show-pdf" target="_blank"> + if "://www.tandfonline.com/doi/full/10." in html_url: + href = soup.find('a', attrs={"class":"show-pdf"}) + if href: + url = href['href'].strip() + if "/pdf/" in url: + return dict(pdf_url=host_prefix+url, technique='publisher') return dict() |