From c6b33542398c933a6272586e6280f7026b63a124 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 19 Aug 2020 23:00:34 -0700 Subject: update robots.txt and sitemap.xml - show minimal robots/sitemap if not in prod environment - default to allow all in robots.txt; link to sitemap index files - basic sitemap.xml without entity-level links --- python/fatcat_web/routes.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'python/fatcat_web/routes.py') diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py index da2bb6cf..c66c7f0c 100644 --- a/python/fatcat_web/routes.py +++ b/python/fatcat_web/routes.py @@ -1157,7 +1157,18 @@ def page_rfc(): return render_template('rfc.html') @app.route('/robots.txt', methods=['GET']) -def robots(): +def page_robots_txt(): + if conf.FATCAT_DOMAIN == "fatcat.wiki": + robots_path = "robots.txt" + else: + robots_path = "robots.deny_all.txt" return send_from_directory(os.path.join(app.root_path, 'static'), - 'robots.txt', + robots_path, mimetype='text/plain') + +@app.route('/sitemap.xml', methods=['GET']) +def page_sitemap_xml(): + if conf.FATCAT_DOMAIN == "fatcat.wiki": + return redirect('/sitemaps/sitemap.xml') + else: + abort(404) -- cgit v1.2.3