[enh] robots.txt added

1 files changed, 8 insertions, 0 deletions

diff --git a/searx/webapp.py b/searx/webapp.py
index 403affc37..deb99995e 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -148,6 +148,14 @@ def stats():
     stats = get_engines_stats()
     return render('stats.html', stats=stats)
 
+@app.route('/robots.txt', methods=['GET'])
+def robots():
+    return Response("""User-agent: *
+Allow: /
+Allow: /about
+Disallow: /stats
+""", mimetype='text/plain')
+
 @app.route('/opensearch.xml', methods=['GET'])
 def opensearch():
     global opensearch_xml