From 36577de5bd84fbc9311d8938b8d5642cf856b1f8 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 19 Oct 2020 17:23:21 -0700 Subject: ingest: add a cdx-error slowdown delay --- python/sandcrawler/ingest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 6f9da86..322859a 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -2,6 +2,7 @@ import sys import json import gzip +import time import base64 import requests from http.server import BaseHTTPRequestHandler, HTTPServer @@ -380,6 +381,8 @@ class IngestFileWorker(SandcrawlerWorker): except CdxApiError as e: result['status'] = 'cdx-error' result['error_message'] = str(e)[:1600] + # add a sleep in cdx-error path as a slow-down + time.sleep(2.0) return result except WaybackError as e: result['status'] = 'wayback-error' -- cgit v1.2.3