more ingest HTML extraction hacks

1 files changed, 46 insertions, 6 deletions

diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py
index 812335f..93eb28e 100644
--- a/python/sandcrawler/html.py
+++ b/python/sandcrawler/html.py
@@ -8,6 +8,7 @@ from bs4 import BeautifulSoup
 RESEARCHSQUARE_REGEX = re.compile(r'"url":"(https://assets.researchsquare.com/files/.{1,50}/v\d+/Manuscript.pdf)"')
 IEEEXPLORE_REGEX = re.compile(r'"pdfPath":"(/.*?\.pdf)"')
 OVID_JOURNAL_URL_REGEX = re.compile(r'journalURL = "(http.*)";')
+SCIENCEDIRECT_BOUNCE_URL_REGEX = re.compile(r"window.location = '(http.*)';")
 
 def test_regex():
     lines = """
@@ -17,6 +18,17 @@ def test_regex():
     m = OVID_JOURNAL_URL_REGEX.search(lines)
     assert m.group(1) == "https://journals.lww.com/co-urology/fulltext/10.1097/MOU.0000000000000689"
 
+    lines = """
+            window.onload = function () {
+                window.location = 'https://pdf.sciencedirectassets.com/320270/AIP/1-s2.0-S2590109519300424/main.pdf?X-Amz-Security-Token=IQoJb3JpZ2luX2VjEH0aCXVzLWVhc3QtMSJGMEQCICBF0dnrtKfpcs3T1kOjMS9w9gedqiLBrcbp4aKQSP8fAiAT9G426t6FWXHO2zPSXRFLq2eiqgbew2vkNKbcn87teyq9Awj1%2F%2F%2F%2F%2F%2F%2F%2F%2F%2F8BEAIaDDA1OTAwMzU0Njg2NSIMnZcTRhbvMwF%2F5PA5KpEDdN%2FDI4V%2BNMDWQDFeAdUc99Lyxak%2B6vhAsfCBCf8hhvrRpalz75e74%2FXMAQwMN9m6i98o0Ljv9od7cuQEy8t%2B0DLzjzX5n3%2FxmpttowhMUm1jc8tBniLKBjwhTyiSHwhdeaVZf6x2zCJ0EIOWMNJHp3iFEqpaFvkRZbC1KWK4XPNNKo72HCvXuG7xmGrdHByz91AP7UgIYCy4hT10fnM43gbOE4wW8fqpgnvwCId%2F2u8k4rQoCLBqLYZzqshCRm1DBbsXCQhTwDXiMC2Ek3f63yKgw7rRCAxvs0vqirG%2B4mJ6LADaztAFMtKDPfnd4e%2B7%2FvnKU2NeotrqrkRgOkIAoFumbQXf20ky6mKWyHBk%2FxirVp60vUcLQpUm2Pcp6ythYxUi9IJxRGX8EF6aV4UHuCpUDUE7o8N84KUXIedUpytUZx7Xoxfk9w%2BR3%2FgX4LEHfkrWgiFAS3bVxNGOeV7GTwcXdcAggbdCaiAe46dfv7DDedx0KhVKOPH7obfvShqd6TYc0BjrV4sx61594ZJ3%2FO0ws7Lj8AU67AF17%2B1NZ3Ugu%2BwG9Ys9s7OxG8E4kBJ58vEY1yuBOQK9y2we4%2FTGPuqSxCuezqA%2BseslXYP%2FRc%2FZL9xx%2FUYaSjZhk1p1mhojxgBrckJYU7d8c4ELMPmtVy6R1yd2VDUoawEU8SB7nbNnMKzqQ3RgGgqGJiELys6dt%2FIr%2BVhpqM%2FZT4zadvzs8P%2FLoGzUHJKNZt0f99wLvZilphV92E%2BOUnwC4wbg3i3af3zozULwgEr7T%2FX2VsyREgexlzk76qMALPn0lgnciUyyQXxyUWAilXYQ0mQdXefh9lFfycczvt0UEuarX9p1sMwl8Ve5aw%3D%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20200110T210936Z&X-Amz-SignedHeaders=host&X-Amz-Expires=300&X-Amz-Credential=ASIAQ3PHCVTY23CMDBNC%2F20200110%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=b43525576e1a0fdbab581481a3fe6db2862cbb2c69f2860b70cc8d444ccd73d5&hash=ccd128dfe597e704224bdfb4b3358de29b2be5d95887c71076bdab1236ba9e42&host=68042c943591013ac2b2430a89b270f6af2c76d8dfd086a07176afe7c76c2c61&pii=S2590109519300424&tid=spdf-74468ebd-6be6-43ac-b294-ced86e8eea58&sid=f9676d658285a749c46b6d081d965bb12aa8gxrqa&type=client';
+                refreshOriginalWindow();
+            }
+    """
+    url = "https://pdf.sciencedirectassets.com/320270/AIP/1-s2.0-S2590109519300424/main.pdf?X-Amz-Security-Token=IQoJb3JpZ2luX2VjEH0aCXVzLWVhc3QtMSJGMEQCICBF0dnrtKfpcs3T1kOjMS9w9gedqiLBrcbp4aKQSP8fAiAT9G426t6FWXHO2zPSXRFLq2eiqgbew2vkNKbcn87teyq9Awj1%2F%2F%2F%2F%2F%2F%2F%2F%2F%2F8BEAIaDDA1OTAwMzU0Njg2NSIMnZcTRhbvMwF%2F5PA5KpEDdN%2FDI4V%2BNMDWQDFeAdUc99Lyxak%2B6vhAsfCBCf8hhvrRpalz75e74%2FXMAQwMN9m6i98o0Ljv9od7cuQEy8t%2B0DLzjzX5n3%2FxmpttowhMUm1jc8tBniLKBjwhTyiSHwhdeaVZf6x2zCJ0EIOWMNJHp3iFEqpaFvkRZbC1KWK4XPNNKo72HCvXuG7xmGrdHByz91AP7UgIYCy4hT10fnM43gbOE4wW8fqpgnvwCId%2F2u8k4rQoCLBqLYZzqshCRm1DBbsXCQhTwDXiMC2Ek3f63yKgw7rRCAxvs0vqirG%2B4mJ6LADaztAFMtKDPfnd4e%2B7%2FvnKU2NeotrqrkRgOkIAoFumbQXf20ky6mKWyHBk%2FxirVp60vUcLQpUm2Pcp6ythYxUi9IJxRGX8EF6aV4UHuCpUDUE7o8N84KUXIedUpytUZx7Xoxfk9w%2BR3%2FgX4LEHfkrWgiFAS3bVxNGOeV7GTwcXdcAggbdCaiAe46dfv7DDedx0KhVKOPH7obfvShqd6TYc0BjrV4sx61594ZJ3%2FO0ws7Lj8AU67AF17%2B1NZ3Ugu%2BwG9Ys9s7OxG8E4kBJ58vEY1yuBOQK9y2we4%2FTGPuqSxCuezqA%2BseslXYP%2FRc%2FZL9xx%2FUYaSjZhk1p1mhojxgBrckJYU7d8c4ELMPmtVy6R1yd2VDUoawEU8SB7nbNnMKzqQ3RgGgqGJiELys6dt%2FIr%2BVhpqM%2FZT4zadvzs8P%2FLoGzUHJKNZt0f99wLvZilphV92E%2BOUnwC4wbg3i3af3zozULwgEr7T%2FX2VsyREgexlzk76qMALPn0lgnciUyyQXxyUWAilXYQ0mQdXefh9lFfycczvt0UEuarX9p1sMwl8Ve5aw%3D%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20200110T210936Z&X-Amz-SignedHeaders=host&X-Amz-Expires=300&X-Amz-Credential=ASIAQ3PHCVTY23CMDBNC%2F20200110%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=b43525576e1a0fdbab581481a3fe6db2862cbb2c69f2860b70cc8d444ccd73d5&hash=ccd128dfe597e704224bdfb4b3358de29b2be5d95887c71076bdab1236ba9e42&host=68042c943591013ac2b2430a89b270f6af2c76d8dfd086a07176afe7c76c2c61&pii=S2590109519300424&tid=spdf-74468ebd-6be6-43ac-b294-ced86e8eea58&sid=f9676d658285a749c46b6d081d965bb12aa8gxrqa&type=client"
+    m = SCIENCEDIRECT_BOUNCE_URL_REGEX.search(lines)
+    assert m.group(1) == url
+
+
 def extract_fulltext_url(html_url, html_body):
     """
     Takes an HTML document (and URL), assumed to be a landing page, and tries
@@ -90,7 +102,7 @@ def extract_fulltext_url(html_url, html_body):
         m = RESEARCHSQUARE_REGEX.search(html_body.decode('utf-8'))
         if m:
             url = m.group(1)
-            assert len(url) < 1024
+            assert len(url) < 4096
             return dict(release_stage="manuscript", pdf_url=url, technique='publisher')
 
     # elseiver linking hub
@@ -106,6 +118,16 @@ def extract_fulltext_url(html_url, html_body):
                 url = url.split('?via')[0]
                 return dict(next_url=url, technique="elsevier-linkinghub")
 
+    # sciencedirect PDF bounce page
+    # https://www.sciencedirect.com/science/article/pii/S2590109519300424/pdfft?md5=854f43a44de186eb58674b8e20631691&pid=1-s2.0-S2590109519300424-main.pdf
+    if '://www.sciencedirect.com/' in html_url and html_url.endswith(".pdf"):
+        # window.location = 'https://pdf.sciencedirectassets.com/320270/AIP/1-s2.0-S2590109519300424/main.pdf?X-Amz-Security-Token=[...]&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20200110T210936Z&X-Amz-SignedHeaders=host&X-Amz-Expires=300&X-Amz-Credential=ASIAQ3PHCVTY23CMDBNC%2F20200110%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=[...]&hash=[...]&host=[...]&pii=S2590109519300424&tid=spdf-74468ebd-6be6-43ac-b294-ced86e8eea58&sid=[...]&type=client';
+        m = SCIENCEDIRECT_BOUNCE_URL_REGEX.search(html_body.decode('utf-8'))
+        if m:
+            url = m.group(1)
+            assert len(url) < 4000
+            return dict(pdf_url=url, technique="sciencedirect-bounce")
+
     # ieeexplore.ieee.org
     # https://ieeexplore.ieee.org/document/8730316
     if '://ieeexplore.ieee.org/document/' in html_url:
@@ -114,8 +136,8 @@ def extract_fulltext_url(html_url, html_body):
         m = IEEEXPLORE_REGEX.search(html_body.decode('utf-8'))
         if m:
             url = m.group(1)
-            assert len(url) < 1024
-            return dict(release_stage="published", pdf_url=host_prefix+url, technique="publisher")
+            assert len(url) < 4096
+            return dict(release_stage="published", pdf_url=host_prefix+url, technique="ieeexplore")
     # https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=8730313
     if '://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber' in html_url:
         # HTML iframe like:
@@ -152,7 +174,7 @@ def extract_fulltext_url(html_url, html_body):
         m = OVID_JOURNAL_URL_REGEX.search(html_body.decode('utf-8'))
         if m:
             url = m.group(1)
-            assert len(url) < 1024
+            assert len(url) < 4096
             return dict(next_url=url, technique='ovid')
 
     # osf.io
@@ -170,7 +192,7 @@ def extract_fulltext_url(html_url, html_body):
     # wiley
     # https://onlinelibrary.wiley.com/doi/pdf/10.1111/1467-923X.12787
     if "://onlinelibrary.wiley.com/doi/pdf/" in html_url:
-        if "/doi/pdfdirect/" in html_body:
+        if b"/doi/pdfdirect/" in html_body:
             next_url = html_url.replace('/doi/pdf/', '/doi/pdfdirect/')
             return dict(next_url=next_url, technique='wiley-pdfdirect')
 
@@ -182,6 +204,24 @@ def extract_fulltext_url(html_url, html_body):
         if href:
             url = href['href'].strip()
             if "/pdf/" in url:
-                return dict(pdf_url=host_prefix+url, technique='publisher')
+                return dict(pdf_url=host_prefix+url, technique='publisher-href')
+
+    # arxiv abstract pages
+    if "://arxiv.org/abs/" in html_url:
+        url = html_url.replace("/abs/", "/pdf/")
+        return dict(pdf_url=url, technique='arxiv-url')
+
+    # american archivist (OA)
+    # https://americanarchivist.org/doi/abs/10.17723/aarc.62.2.j475270470145630
+    if "://americanarchivist.org/doi/abs/" in html_url:
+        # <a href="/doi/pdf/10.17723/aarc.62.2.j475270470145630" target="_blank">
+        hrefs = soup.find_all('a', attrs={"_target":"blank"})
+        for href in hrefs:
+            url = href['href'].strip()
+            if "/doi/pdf/" in url:
+                if url.startswith('http'):
+                    return dict(pdf_url=url, technique='publisher-href')
+                elif url.startswith('/'):
+                    return dict(pdf_url=host_prefix+url, technique='publisher-href')
 
     return dict()