From a1b44161e206873be30c0640f5fab7a284023ba1 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Tue, 21 Jan 2020 10:59:27 -0800
Subject: persist: work around GROBID timeouts with no status_code

---
 python/sandcrawler/db.py      | 2 +-
 python/sandcrawler/persist.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'python')

diff --git a/python/sandcrawler/db.py b/python/sandcrawler/db.py
index 3ec325e..e1414ba 100644
--- a/python/sandcrawler/db.py
+++ b/python/sandcrawler/db.py
@@ -161,7 +161,7 @@ class SandcrawlerPostgresClient:
                 r['metadata'] = json.dumps(r['metadata'], sort_keys=True)
         batch = [(d['key'],
                   d.get('grobid_version') or None,
-                  d['status_code'],
+                  d.get('status_code') or None,
                   d['status'],
                   d.get('fatcat_release') or None,
                   d.get('updated') or datetime.datetime.now(),
diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py
index 2463afa..801f76d 100644
--- a/python/sandcrawler/persist.py
+++ b/python/sandcrawler/persist.py
@@ -223,7 +223,7 @@ class PersistGrobidWorker(SandcrawlerWorker):
         self.counts['total'] += len(batch)
 
         for r in batch:
-            if r['status_code'] != 200 or not r.get('tei_xml'):
+            if r.get('status_code') != 200 or not r.get('tei_xml'):
                 self.counts['s3-skip-status'] += 1
                 if r.get('error_msg'):
                     r['metadata'] = {'error_msg': r['error_msg'][:500]}
@@ -291,7 +291,7 @@ class PersistGrobidDiskWorker(SandcrawlerWorker):
 
     def process(self, record):
 
-        if record['status_code'] != 200 or not record.get('tei_xml'):
+        if record.get('status_code') != 200 or not record.get('tei_xml'):
             return False
         assert(len(record['key'])) == 40
         p = "{}/{}".format(self.output_dir, self._blob_path(record['key']))
-- 
cgit v1.2.3