update robots.txt and sitemap.xml

- show minimal robots/sitemap if not in prod environment
- default to allow all in robots.txt; link to sitemap index files
- basic sitemap.xml without entity-level links

1 files changed, 13 insertions, 2 deletions

diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py
index da2bb6cf..c66c7f0c 100644
--- a/python/fatcat_web/routes.py
+++ b/python/fatcat_web/routes.py
@@ -1157,7 +1157,18 @@ def page_rfc():
     return render_template('rfc.html')
 
 @app.route('/robots.txt', methods=['GET'])
-def robots():
+def page_robots_txt():
+    if conf.FATCAT_DOMAIN == "fatcat.wiki":
+        robots_path = "robots.txt"
+    else:
+        robots_path = "robots.deny_all.txt"
     return send_from_directory(os.path.join(app.root_path, 'static'),
-                               'robots.txt',
+                               robots_path,
                                mimetype='text/plain')
+
+@app.route('/sitemap.xml', methods=['GET'])
+def page_sitemap_xml():
+    if conf.FATCAT_DOMAIN == "fatcat.wiki":
+        return redirect('/sitemaps/sitemap.xml')
+    else:
+        abort(404)