From 4e2bfc1fc7720849125af221e9afee7d1e5197af Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 30 Jun 2021 18:30:24 -0700 Subject: robots: add commented out reference to sitemap-index-access.xml --- fatcat_scholar/static/robots.allow.txt | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fatcat_scholar/static') diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt index fb0c251..21a343e 100644 --- a/fatcat_scholar/static/robots.allow.txt +++ b/fatcat_scholar/static/robots.allow.txt @@ -15,3 +15,6 @@ Crawl-delay: 5 Sitemap: https://scholar.archive.org/sitemap.xml Sitemap: https://scholar.archive.org/sitemap-index-works.xml + +# same info as sitemap-index-works.xml plus following citation_pdf_url +#Sitemap: https://scholar.archive.org/sitemap-index-access.xml -- cgit v1.2.3