summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2022-06-06 12:01:35 +0200
committerBryan Newbold <bnewbold@archive.org>2022-06-06 12:01:35 +0200
commit3b5456b615567fc2340c83a3533890055dd0814c (patch)
tree0a00f05dac8bfe810f16acbff73224add09c9154
parent396a2be0cc7aa912b5b9c13d0139f852f3e940bc (diff)
downloadfatcat-scholar-3b5456b615567fc2340c83a3533890055dd0814c.tar.gz
fatcat-scholar-3b5456b615567fc2340c83a3533890055dd0814c.zip
robots: deny SemanticScholarBot search result pages
-rw-r--r--fatcat_scholar/static/robots.allow.txt1
1 files changed, 1 insertions, 0 deletions
diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt
index ccdfda1..d63fcb9 100644
--- a/fatcat_scholar/static/robots.allow.txt
+++ b/fatcat_scholar/static/robots.allow.txt
@@ -7,6 +7,7 @@ User-agent: SemrushBot
User-agent: YandexBot
User-agent: bingbot
User-agent: Googlebot
+User-agent: SemanticScholarBot
Disallow: /search
# crawling search result pages is expensive, so we do specify a long crawl