diff options
Diffstat (limited to 'fatcat_scholar/static')
| -rw-r--r-- | fatcat_scholar/static/robots.allow.txt | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt index d63fcb9..6076e75 100644 --- a/fatcat_scholar/static/robots.allow.txt +++ b/fatcat_scholar/static/robots.allow.txt @@ -8,6 +8,7 @@ User-agent: YandexBot User-agent: bingbot User-agent: Googlebot User-agent: SemanticScholarBot +User-agent: yacybot Disallow: /search # crawling search result pages is expensive, so we do specify a long crawl |