diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-05-21 20:24:12 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-05-21 20:24:12 -0700 |
| commit | 383349341a09f9d05f58b6104064af7b0c89dcc1 (patch) | |
| tree | 344797bb1fb1c43fb116013ba23a96de7c090389 /fatcat_scholar/sim_pipeline.py | |
| parent | fee17cf6518e13b6f1c3945dd769aba56d7606d5 (diff) | |
| download | fatcat-scholar-383349341a09f9d05f58b6104064af7b0c89dcc1.tar.gz fatcat-scholar-383349341a09f9d05f58b6104064af7b0c89dcc1.zip | |
handle petabox read timeouts a bit
Diffstat (limited to 'fatcat_scholar/sim_pipeline.py')
| -rw-r--r-- | fatcat_scholar/sim_pipeline.py | 7 |
1 files changed, 6 insertions, 1 deletions
diff --git a/fatcat_scholar/sim_pipeline.py b/fatcat_scholar/sim_pipeline.py index c96681a..5657cb7 100644 --- a/fatcat_scholar/sim_pipeline.py +++ b/fatcat_scholar/sim_pipeline.py @@ -4,6 +4,7 @@ import io import sys import sqlite3 import argparse +import requests from pydantic import BaseModel, validator from typing import List, Dict, Tuple, Optional, Any, Sequence from fatcat_openapi_client import ReleaseEntity, FileEntity @@ -113,7 +114,11 @@ class SimPipeline(): # TODO: more filters; also redundant with IssueDB code? if row['issue_item'].endswith('_contents') or row['issue_item'].endswith('_index'): continue - full_issue = self.fetch_sim_issue(row) + try: + full_issue = self.fetch_sim_issue(row) + except requests.exceptions.ReadTimeout as e: + print(str(e), file=sys.stderr) + continue if not full_issue: continue for leaf in full_issue['page_texts']: |