more progress on file ingest

4 files changed, 75 insertions, 17 deletions

diff --git a/python/ingest_file.py b/python/ingest_file.py
index 40eee4d..4fd44ca 100755
--- a/python/ingest_file.py
+++ b/python/ingest_file.py
@@ -20,9 +20,10 @@ def run_single_ingest(args):
     return result
 
 def run_requests(args):
+    ingester = FileIngester()
     for l in args.json_file:
         request = json.loads(l.strip())
-        result = ingest_file(request)
+        result = ingester.ingest_file(request)
         print(json.dumps(result))
 
 def run_api(args):
diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py
index 3191b66..858e02a 100644
--- a/python/sandcrawler/html.py
+++ b/python/sandcrawler/html.py
@@ -6,6 +6,7 @@ import urllib.parse
 from bs4 import BeautifulSoup
 
 RESEARCHSQUARE_REGEX = re.compile(r'"url":"(https://assets.researchsquare.com/files/.{1,50}/v\d+/Manuscript.pdf)"')
+IEEEXPLORE_REGEX = re.compile(r'"pdfPath":"(/.*?\.pdf)"')
 
 def extract_fulltext_url(html_url, html_body):
     """
@@ -70,4 +71,22 @@ def extract_fulltext_url(html_url, html_body):
                 url = urllib.parse.unquote(url)
                 return dict(next_url=url)
 
+    # ieeexplore.ieee.org
+    # https://ieeexplore.ieee.org/document/8730316
+    if '://ieeexplore.ieee.org/document/' in html_url:
+        # JSON in body with a field like:
+        # "pdfPath":"/iel7/6287639/8600701/08730316.pdf",
+        m = IEEEXPLORE_REGEX.search(html_body.decode('utf-8'))
+        if m:
+            url = m.group(1)
+            assert len(url) < 1024
+            return dict(release_stage="published", pdf_url=host_prefix+url)
+    # https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=8730313
+    if '://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber' in html_url:
+        # HTML iframe like:
+        # <iframe src="http://web.archive.org/web/20191026011528if_/https://ieeexplore.ieee.org/ielx7/6287639/8600701/08730313.pdf?tp=&amp;arnumber=8730313&amp;isnumber=8600701&amp;ref=" frameborder="0"></iframe>
+        iframe = soup.find("iframe")
+        if iframe and '.pdf' in iframe['src']:
+            return dict(pdf_url=iframe['src'])
+
     return dict()
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index a772bd4..455c9f6 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -5,6 +5,7 @@
 
 import os, sys, time
 import requests
+import datetime
 
 import wayback.exception
 from http.client import IncompleteRead
@@ -21,7 +22,7 @@ class CdxApiClient:
     def __init__(self, host_url="https://web.archive.org/cdx/search/cdx"):
         self.host_url = host_url
 
-    def lookup_latest(self, url, follow_redirects=False):
+    def lookup_latest(self, url, recent_only=True, follow_redirects=False):
         """
         Looks up most recent HTTP 200 record for the given URL.
 
@@ -29,21 +30,22 @@ class CdxApiClient:
 
         XXX: should do authorized lookup using cookie to get all fields
         """
+        WAYBACK_ENDPOINT = "https://web.archive.org/web/"
 
+        since = datetime.date.today() - datetime.timedelta(weeks=4)
         params = {
             'url': url,
             'matchType': 'exact',
             'limit': -1,
             'output': 'json',
         }
+        if recent_only:
+            params['from'] = '%04d%02d%02d' % (since.year, since.month, since.day),
         if not follow_redirects:
             params['filter'] = 'statuscode:200'
         resp = requests.get(self.host_url, params=params)
-        if follow_redirects:
-            raise NotImplementedError
-        else:
-            if resp.status_code != 200:
-                raise CdxApiError(resp.text)
+        if resp.status_code != 200:
+            raise CdxApiError(resp.text)
         rj = resp.json()
         if len(rj) <= 1:
             return None
@@ -58,6 +60,12 @@ class CdxApiClient:
             sha1b32=cdx[5],
             sha1hex=b32_hex(cdx[5]),
         )
+        if follow_redirects and cdx['http_status'] in (301, 302):
+            resp = requests.get(WAYBACK_ENDPOINT + cdx['datetime'] + "id_/" + cdx['url'])
+            assert resp.status_code == 200
+            next_url = '/'.join(resp.url.split('/')[5:])
+            assert next_url != url
+            return self.lookup_latest(next_url)
         return cdx
 
 
@@ -183,11 +191,14 @@ class SavePageNowClient:
             status = resp.json()['status']
             if status == 'success':
                 resp = resp.json()
+                if resp.get('message', '').startswith('The same snapshot had been made'):
+                    raise SavePageNowError("SPN2 re-snapshot withing short time window")
                 break
             elif status == 'pending':
                 time.sleep(1.0)
             else:
                 raise SavePageNowError("SPN2 status:{} url:{}".format(status, url))
 
+        #print(resp)
         return resp['resources']
 
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 2469df6..e2dd44c 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -5,7 +5,7 @@ import base64
 import requests
 from http.server import BaseHTTPRequestHandler, HTTPServer
 
-from sandcrawler.ia import SavePageNowClient, CdxApiClient, WaybackClient
+from sandcrawler.ia import SavePageNowClient, CdxApiClient, WaybackClient, WaybackError
 from sandcrawler.grobid import GrobidClient
 from sandcrawler.misc import gen_file_metadata
 from sandcrawler.html import extract_fulltext_url
@@ -39,10 +39,12 @@ class FileIngester:
     
         WAYBACK_ENDPOINT = "https://web.archive.org/web/"
     
-        cdx = self.cdx_client.lookup_latest(url)
+        cdx = self.cdx_client.lookup_latest(url, follow_redirects=True)
         if not cdx:
+            # TODO: refactor this to make adding new domains/patterns easier
             # sciencedirect.com (Elsevier) requires browser crawling (SPNv2)
-            if ('sciencedirect.com' in url and '.pdf' in url):
+            if ('sciencedirect.com' in url and '.pdf' in url) or ('osapublishing.org' in url) or ('pubs.acs.org/doi/' in url) or ('ieeexplore.ieee.org' in url and ('.pdf' in url or '/stamp/stamp.jsp' in url)):
+                #print(url)
                 cdx_list = self.spn_client.save_url_now_v2(url)
                 for cdx_url in cdx_list:
                     if 'pdf.sciencedirectassets.com' in cdx_url and '.pdf' in cdx_url:
@@ -51,8 +53,19 @@ class FileIngester:
                     if 'osapublishing.org' in cdx_url and 'abstract.cfm' in cdx_url:
                         cdx = self.cdx_client.lookup_latest(cdx_url)
                         break
+                    if 'pubs.acs.org' in cdx_url and '/doi/pdf/' in cdx_url:
+                        cdx = self.cdx_client.lookup_latest(cdx_url)
+                        break
+                    if 'ieeexplore.ieee.org' in cdx_url and '.pdf' in cdx_url and 'arnumber=' in cdx_url:
+                        cdx = self.cdx_client.lookup_latest(cdx_url)
+                        break
                 if not cdx:
-                    raise Exception("Failed to crawl sciencedirect.com PDF URL")
+                    # extraction didn't work as expected; fetch whatever SPN2 got
+                    cdx = self.cdx_client.lookup_latest(url, follow_redirects=True)
+                if not cdx:
+                    raise SavePageNowError("")
+                    sys.stderr.write("{}\n".format(cdx_list))
+                    raise Exception("Failed to crawl PDF URL")
             else:
                 return self.spn_client.save_url_now_v1(url)
     
@@ -87,18 +100,31 @@ class FileIngester:
             sys.stderr.write("CDX hit: {}\n".format(cdx_dict))
 
             response['cdx'] = cdx_dict
-            response['terminal'] = dict()
+            # TODO: populate terminal
+            response['terminal'] = dict(url=cdx_dict['url'], http_status=cdx_dict['http_status'])
+            if not body:
+                response['status'] = 'null-body'
+                return response
             file_meta = gen_file_metadata(body)
             mimetype = cdx_dict['mimetype']
             if mimetype in ('warc/revisit', 'binary/octet-stream', 'application/octet-stream'):
                 mimetype = file_meta['mimetype']
+                response['file_meta'] = file_meta
             if 'html' in mimetype:
                 page_metadata = extract_fulltext_url(response['cdx']['url'], body)
                 if page_metadata and page_metadata.get('pdf_url'):
-                    url = page_metadata.get('pdf_url')
+                    next_url = page_metadata.get('pdf_url')
+                    if next_url == url:
+                        response['status'] = 'link-loop'
+                        return response
+                    url = next_url
                     continue
                 elif page_metadata and page_metadata.get('next_url'):
-                    url = page_metadata.get('next_url')
+                    next_url = page_metadata.get('next_url')
+                    if next_url == url:
+                        response['status'] = 'link-loop'
+                        return response
+                    url = next_url
                     continue
                 else:
                     response['terminal']['html'] = page_metadata
@@ -116,7 +142,7 @@ class FileIngester:
 
         # do GROBID
         response['grobid'] = self.grobid_client.process_fulltext(body)
-        sys.stderr.write("GROBID status: {}\n".format(response['grobid']['status']))
+        #sys.stderr.write("GROBID status: {}\n".format(response['grobid']['status']))
 
         # TODO: optionally publish to Kafka here, but continue on failure (but
         # send a sentry exception?)
@@ -129,7 +155,7 @@ class FileIngester:
             response['grobid'].pop('tei_xml')
 
         # Ok, now what?
-        sys.stderr.write("GOT TO END\n")
+        #sys.stderr.write("GOT TO END\n")
         response['status'] = "success"
         response['hit'] = True
         return response
@@ -144,7 +170,8 @@ class IngestFileRequestHandler(BaseHTTPRequestHandler):
         length = int(self.headers.get('content-length'))
         request = json.loads(self.rfile.read(length).decode('utf-8'))
         print("Got request: {}".format(request))
-        result = ingest_file(request)
+        ingester = FileIngester()
+        result = ingester.ingest_file(request)
         self.send_response(200)
         self.end_headers()
         self.wfile.write(json.dumps(result))