aboutsummaryrefslogtreecommitdiffstats
path: root/fatcat_scholar
diff options
context:
space:
mode:
Diffstat (limited to 'fatcat_scholar')
-rw-r--r--fatcat_scholar/static/robots.allow.txt3
1 files changed, 3 insertions, 0 deletions
diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt
index fb0c251..21a343e 100644
--- a/fatcat_scholar/static/robots.allow.txt
+++ b/fatcat_scholar/static/robots.allow.txt
@@ -15,3 +15,6 @@ Crawl-delay: 5
Sitemap: https://scholar.archive.org/sitemap.xml
Sitemap: https://scholar.archive.org/sitemap-index-works.xml
+
+# same info as sitemap-index-works.xml plus following citation_pdf_url
+#Sitemap: https://scholar.archive.org/sitemap-index-access.xml