From 3b5456b615567fc2340c83a3533890055dd0814c Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 6 Jun 2022 12:01:35 +0200 Subject: robots: deny SemanticScholarBot search result pages --- fatcat_scholar/static/robots.allow.txt | 1 + 1 file changed, 1 insertion(+) (limited to 'fatcat_scholar') diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt index ccdfda1..d63fcb9 100644 --- a/fatcat_scholar/static/robots.allow.txt +++ b/fatcat_scholar/static/robots.allow.txt @@ -7,6 +7,7 @@ User-agent: SemrushBot User-agent: YandexBot User-agent: bingbot User-agent: Googlebot +User-agent: SemanticScholarBot Disallow: /search # crawling search result pages is expensive, so we do specify a long crawl -- cgit v1.2.3