From 12a51fd28ca64338fca040ab7c470a70bf7a2a1b Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Fri, 23 Oct 2020 13:17:59 -0700
Subject: ingest: skip JSTOR DOI prefixes

---
 python/sandcrawler/ingest.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index d2a9980..6d8b162 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -101,6 +101,9 @@ class IngestFileWorker(SandcrawlerWorker):
             # Historical non-paper content:
             "dhz.uni-passau.de/",   # newspapers
             "digital.ucd.ie/",      # ireland national historical
+
+            # DOI prefixes
+            "://doi.org/10.2307/",  # JSTOR; slow and many redirects
         ]
 
         self.wall_blocklist = [
-- 
cgit v1.2.3