path: root/fatcat_scholar/static/robots.allow.txt

# Allow most queries, but please don't crawl search results (or at least do so very politely, eg no parallel requests)

User-Agent: *
Disallow: /search
Allow: /