diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2021-12-06 15:51:29 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2021-12-06 16:16:37 -0800 |
| commit | 5b8a58ab37d5187dcf750b498d5e91b91f0c758a (patch) | |
| tree | c3218fd21fedcd75329f5f00a30f2caa750a659c /fatcat_scholar | |
| parent | 25222cd3dcbf396c7d2a7372e32dd6e9ccae9ba4 (diff) | |
| download | fatcat-scholar-5b8a58ab37d5187dcf750b498d5e91b91f0c758a.tar.gz fatcat-scholar-5b8a58ab37d5187dcf750b498d5e91b91f0c758a.zip | |
SIM pipeline: improve exception handling
Diffstat (limited to 'fatcat_scholar')
| -rw-r--r-- | fatcat_scholar/sim_pipeline.py | 11 | ||||
| -rw-r--r-- | fatcat_scholar/work_pipeline.py | 12 |
2 files changed, 12 insertions, 11 deletions
diff --git a/fatcat_scholar/sim_pipeline.py b/fatcat_scholar/sim_pipeline.py index e140816..34aaae3 100644 --- a/fatcat_scholar/sim_pipeline.py +++ b/fatcat_scholar/sim_pipeline.py @@ -213,10 +213,13 @@ class SimPipeline: """ try: full_issue = self.fetch_sim_issue(issue_item, pub_collection) - except requests.exceptions.ConnectionError as e: - print(str(e), file=sys.stderr) - return - except requests.exceptions.ReadTimeout as e: + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.RetryError, + requests.exceptions.ChunkedEncodingError, + urllib3.exceptions.MaxRetryError, + ) as e: print(str(e), file=sys.stderr) return except requests.exceptions.ChunkedEncodingError as e: diff --git a/fatcat_scholar/work_pipeline.py b/fatcat_scholar/work_pipeline.py index 5c32767..f17d69d 100644 --- a/fatcat_scholar/work_pipeline.py +++ b/fatcat_scholar/work_pipeline.py @@ -412,13 +412,11 @@ class WorkPipeline: sim_fulltext = self.fetch_sim( sim_issue, sim_pub, release.pages, release.ident ) - except requests.exceptions.ConnectionError as e: - print(str(e), file=sys.stderr) - continue - except requests.exceptions.ReadTimeout as e: - print(str(e), file=sys.stderr) - continue - except requests.exceptions.ChunkedEncodingError as e: + except ( + requests.exceptions.ConnectionError, + requests.exceptions.ReadTimeout, + requests.exceptions.ChunkedEncodingError, + ) as e: print(str(e), file=sys.stderr) continue if sim_fulltext: |