diff options
| -rw-r--r-- | fatcat_scholar/static/robots.allow.txt | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/fatcat_scholar/static/robots.allow.txt b/fatcat_scholar/static/robots.allow.txt index fb0c251..21a343e 100644 --- a/fatcat_scholar/static/robots.allow.txt +++ b/fatcat_scholar/static/robots.allow.txt @@ -15,3 +15,6 @@ Crawl-delay: 5 Sitemap: https://scholar.archive.org/sitemap.xml Sitemap: https://scholar.archive.org/sitemap-index-works.xml + +# same info as sitemap-index-works.xml plus following citation_pdf_url +#Sitemap: https://scholar.archive.org/sitemap-index-access.xml |