From 047cfcebd8ff792d6da9c10b0bee27ec689a4e9f Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 8 Sep 2021 14:10:55 -0700 Subject: ingest: more block patterns, for huge databases --- python/sandcrawler/ingest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 630c477..1d33b94 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -120,13 +120,16 @@ class IngestFileWorker(SandcrawlerWorker): "www.plate-archive.org/", "://doi.org/10.25642/ipk/gbis/", "://apex.ipk-gatersleben.de/", + "fao.org/glis/", # Historical non-paper content: "dhz.uni-passau.de/", # newspapers "digital.ucd.ie/", # ireland national historical # DOI prefixes - "://doi.org/10.2307/", # JSTOR; slow and many redirects + "doi.org/10.2307/", # JSTOR; slow and many redirects + "doi.org/10.18730/", # fao.org: database entry + "doi.org/10.15468/", # gbif.org: database entry # deprecated domain (doesn't redirect correctly) "://edoc.mpg.de/", -- cgit v1.2.3