diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-01-15 23:04:07 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-01-15 23:04:07 -0800 |
| commit | 34ba59a446a43ecf32dbfded5e330cf4178ba355 (patch) | |
| tree | 1b590c4675d59ce3d0d6461a4b11eeccbbd35a69 /python | |
| parent | a8ada6d50e6522d1d8213b5aeb5662db9c96ed79 (diff) | |
| download | sandcrawler-34ba59a446a43ecf32dbfded5e330cf4178ba355.tar.gz sandcrawler-34ba59a446a43ecf32dbfded5e330cf4178ba355.zip | |
increase SPNv2 polling timeout to 4 minutes
Diffstat (limited to 'python')
| -rw-r--r-- | python/sandcrawler/ia.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index 8895500..9fdb52e 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -635,7 +635,9 @@ class SavePageNowClient: 'Accept': 'application/json', 'Authorization': 'LOW {}:{}'.format(self.ia_access_key, self.ia_secret_key), }) - self.poll_count = 30 + + # 4 minutes total; very long, kafka likely to time out + self.poll_count = 80 self.poll_seconds = 3.0 def save_url_now_v2(self, request_url): |