diff options
Diffstat (limited to 'fatcat_scholar/static')
| -rw-r--r-- | fatcat_scholar/static/robots.allow.txt | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt index ccdfda1..d63fcb9 100644 --- a/fatcat_scholar/static/robots.allow.txt +++ b/fatcat_scholar/static/robots.allow.txt @@ -7,6 +7,7 @@ User-agent: SemrushBot User-agent: YandexBot User-agent: bingbot User-agent: Googlebot +User-agent: SemanticScholarBot Disallow: /search # crawling search result pages is expensive, so we do specify a long crawl |