handle petabox read timeouts a bit

2 files changed, 12 insertions, 2 deletions

diff --git a/fatcat_scholar/sim_pipeline.py b/fatcat_scholar/sim_pipeline.py
index c96681a..5657cb7 100644
--- a/fatcat_scholar/sim_pipeline.py
+++ b/fatcat_scholar/sim_pipeline.py
@@ -4,6 +4,7 @@ import io
 import sys
 import sqlite3
 import argparse
+import requests
 from pydantic import BaseModel, validator
 from typing import List, Dict, Tuple, Optional, Any, Sequence
 from fatcat_openapi_client import ReleaseEntity, FileEntity
@@ -113,7 +114,11 @@ class SimPipeline():
             # TODO: more filters; also redundant with IssueDB code?
             if row['issue_item'].endswith('_contents') or row['issue_item'].endswith('_index'):
                 continue
-            full_issue = self.fetch_sim_issue(row)
+            try:
+                full_issue = self.fetch_sim_issue(row)
+            except requests.exceptions.ReadTimeout as e:
+                print(str(e), file=sys.stderr)
+                continue
             if not full_issue:
                 continue
             for leaf in full_issue['page_texts']:
diff --git a/fatcat_scholar/work_pipeline.py b/fatcat_scholar/work_pipeline.py
index 363535c..f0c0e6f 100644
--- a/fatcat_scholar/work_pipeline.py
+++ b/fatcat_scholar/work_pipeline.py
@@ -3,6 +3,7 @@ import os
 import io
 import sys
 import minio
+import requests
 import argparse
 from pydantic import BaseModel, validator
 from typing import List, Dict, Tuple, Optional, Any, Sequence
@@ -255,7 +256,11 @@ class WorkPipeline():
             if not sim_pub:
                 continue
             # XXX: control flow tweak?
-            sim_fulltext = self.fetch_sim(sim_issue, sim_pub, release.pages, release.ident)
+            try:
+                sim_fulltext = self.fetch_sim(sim_issue, sim_pub, release.pages, release.ident)
+            except requests.exceptions.ReadTimeout as e:
+                print(str(e), file=sys.stderr)
+                continue
             if sim_fulltext:
                 break