aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-02-24 13:21:57 -0800
committerBryan Newbold <bnewbold@archive.org>2020-02-24 13:21:57 -0800
commit266b0a2d5928921d3b3f992fa249b22f7a5edb16 (patch)
treedc59f501b8e4eb26ac00c05635046ec86ab83fdd
parent1a42a73e8c494d13e671d0fc6729665d85010669 (diff)
downloadsandcrawler-266b0a2d5928921d3b3f992fa249b22f7a5edb16.tar.gz
sandcrawler-266b0a2d5928921d3b3f992fa249b22f7a5edb16.zip
ingest: more direct americanarchivist PDF url guess
-rw-r--r--python/sandcrawler/html.py4
1 files changed, 4 insertions, 0 deletions
diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py
index 7189055..e993e74 100644
--- a/python/sandcrawler/html.py
+++ b/python/sandcrawler/html.py
@@ -224,6 +224,10 @@ def extract_fulltext_url(html_url, html_body):
# american archivist (OA)
# https://americanarchivist.org/doi/abs/10.17723/aarc.62.2.j475270470145630
if "://americanarchivist.org/doi/" in html_url and not "/doi/pdf" in html_url:
+ # use a more aggressive direct guess to avoid rate-limiting...
+ if "/doi/10." in html_url:
+ url = html_url.replace("/doi/10.", "/doi/pdf/10.")
+ return dict(pdf_url=url, technique='archivist-url')
# <a href="/doi/pdf/10.17723/aarc.62.2.j475270470145630" target="_blank">
hrefs = soup.find_all('a', attrs={"target":"_blank"})
for href in hrefs: