summaryrefslogtreecommitdiffstats
path: root/python/fatcat_web/static
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2020-08-19 23:00:34 -0700
committerBryan Newbold <bnewbold@robocracy.org>2020-08-19 23:00:36 -0700
commitc6b33542398c933a6272586e6280f7026b63a124 (patch)
treef95b5eb848ea3eceff910cd44b172e86f0332c1b /python/fatcat_web/static
parent5f282a6267182214080ca36bcec4da1755589b46 (diff)
downloadfatcat-c6b33542398c933a6272586e6280f7026b63a124.tar.gz
fatcat-c6b33542398c933a6272586e6280f7026b63a124.zip
update robots.txt and sitemap.xml
- show minimal robots/sitemap if not in prod environment - default to allow all in robots.txt; link to sitemap index files - basic sitemap.xml without entity-level links
Diffstat (limited to 'python/fatcat_web/static')
-rw-r--r--python/fatcat_web/static/robots.deny_all.txt7
-rw-r--r--python/fatcat_web/static/robots.txt19
-rw-r--r--python/fatcat_web/static/sitemap.xml13
3 files changed, 39 insertions, 0 deletions
diff --git a/python/fatcat_web/static/robots.deny_all.txt b/python/fatcat_web/static/robots.deny_all.txt
new file mode 100644
index 00000000..b88274b1
--- /dev/null
+++ b/python/fatcat_web/static/robots.deny_all.txt
@@ -0,0 +1,7 @@
+# Hello friends!
+
+# You have found a QA/development instance of the Fatcat catalog. The canonical
+# location is https://fatcat.wiki, please crawl and index that location instead.
+
+User-agent: *
+Disallow: /
diff --git a/python/fatcat_web/static/robots.txt b/python/fatcat_web/static/robots.txt
index a168f11b..e89af36e 100644
--- a/python/fatcat_web/static/robots.txt
+++ b/python/fatcat_web/static/robots.txt
@@ -1 +1,20 @@
# Hello friends!
+# If you are considering large or automated crawling, you may want to look at
+# our API (https://api.fatcat.wiki) or bulk database snapshots instead.
+
+# by default, can crawl anything on this domain. HTTP 429 ("backoff") status
+# codes are used for rate-limiting instead of any crawl delay specified here.
+# Up to a handful concurrent requests should be fine.
+User-agent: *
+Allow: /
+
+# crawling search result pages is expensive, so we do specify a long crawl delay for those
+User-agent: *
+Allow: /release/search
+Allow: /container/search
+Allow: /coverage/search
+Crawl-delay: 5
+
+Sitemap: https://fatcat.wiki/sitemap.xml
+Sitemap: https://fatcat.wiki/sitemap-index-releases.xml
+Sitemap: https://fatcat.wiki/sitemap-index-containers.xml
diff --git a/python/fatcat_web/static/sitemap.xml b/python/fatcat_web/static/sitemap.xml
new file mode 100644
index 00000000..e6189aa4
--- /dev/null
+++ b/python/fatcat_web/static/sitemap.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <!-- basic site pages -->
+ <url><loc>https://fatcat.wiki/</loc></url>
+ <url><loc>https://fatcat.wiki/about</loc></url>
+ <url><loc>https://fatcat.wiki/rfc</loc></url>
+ <url><loc>https://fatcat.wiki/stats</loc></url>
+ <url><loc>https://fatcat.wiki/changelog</loc></url>
+ <url><loc>https://fatcat.wiki/release/lookup</loc></url>
+ <url><loc>https://fatcat.wiki/container/lookup</loc></url>
+ <url><loc>https://fatcat.wiki/file/lookup</loc></url>
+ <!-- additional entity-level URL lists are linked from robots.txt -->
+</urlset>