From 36577de5bd84fbc9311d8938b8d5642cf856b1f8 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Mon, 19 Oct 2020 17:23:21 -0700
Subject: ingest: add a cdx-error slowdown delay

---
 python/sandcrawler/ingest.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'python/sandcrawler')

diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 6f9da86..322859a 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -2,6 +2,7 @@
 import sys
 import json
 import gzip
+import time
 import base64
 import requests
 from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -380,6 +381,8 @@ class IngestFileWorker(SandcrawlerWorker):
             except CdxApiError as e:
                 result['status'] = 'cdx-error'
                 result['error_message'] = str(e)[:1600]
+                # add a sleep in cdx-error path as a slow-down
+                time.sleep(2.0)
                 return result
             except WaybackError as e:
                 result['status'] = 'wayback-error'
-- 
cgit v1.2.3