From 9da8c9bdb9149f3701f8774c1a220de889e7cc59 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Sun, 30 Oct 2022 13:50:21 -0700
Subject: robots: block yacybot from search page

A couple crawlers causing load.
---
 fatcat_scholar/static/robots.allow.txt | 1 +
 1 file changed, 1 insertion(+)

(limited to 'fatcat_scholar')

diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt
index d63fcb9..6076e75 100644
--- a/fatcat_scholar/static/robots.allow.txt
+++ b/fatcat_scholar/static/robots.allow.txt
@@ -8,6 +8,7 @@ User-agent: YandexBot
 User-agent: bingbot
 User-agent: Googlebot
 User-agent: SemanticScholarBot
+User-agent: yacybot
 Disallow: /search
 
 # crawling search result pages is expensive, so we do specify a long crawl
-- 
cgit v1.2.3