From dba569462d0e9c4dbd77a54bb42ef5c3b1916142 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 19 Apr 2023 17:20:03 +0200 Subject: [mod] limiter: reduce request rates for requests without a ping Signed-off-by: Markus Heiser --- searx/webapp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'searx/webapp.py') diff --git a/searx/webapp.py b/searx/webapp.py index 79255652f..67265e542 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -93,7 +93,7 @@ from searx.utils import ( ) from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH from searx.query import RawTextQuery -from searx.plugins import Plugin, plugins, initialize as plugin_initialize +from searx.plugins import limiter, Plugin, plugins, initialize as plugin_initialize from searx.plugins.oa_doi_rewrite import get_doi_resolver from searx.preferences import ( Preferences, @@ -642,6 +642,12 @@ def health(): return Response('OK', mimetype='text/plain') +@app.route('/limiter.css', methods=['GET', 'POST']) +def limiter_css(): + limiter.ping() + return Response('', mimetype='text/css') + + @app.route('/search', methods=['GET', 'POST']) def search(): """Search query in q and return results. -- cgit v1.2.3 From 5226044c13817688a5ca3461743844dca4ed3d2b Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 19 Apr 2023 18:59:23 +0200 Subject: [mod] limiter: add random token to the limiter URL By adding a random component in the limiter URL a bot can no longer send a ping by request a static URL. Related: https://github.com/searxng/searxng/pull/2357#issuecomment-1518525094 Signed-off-by: Markus Heiser --- searx/webapp.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'searx/webapp.py') diff --git a/searx/webapp.py b/searx/webapp.py index 67265e542..815bfcabd 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -416,6 +416,7 @@ def render(template_name: str, **kwargs): kwargs['endpoint'] = 'results' if 'q' in kwargs else request.endpoint kwargs['cookies'] = request.cookies kwargs['errors'] = request.errors + kwargs['limiter_token'] = limiter.get_token() # values from the preferences kwargs['preferences'] = request.preferences @@ -642,9 +643,10 @@ def health(): return Response('OK', mimetype='text/plain') -@app.route('/limiter.css', methods=['GET', 'POST']) -def limiter_css(): - limiter.ping() +@app.route('/limiter.css', methods=['GET', 'POST']) +def limiter_css(token=None): + if limiter.token_is_valid(token): + limiter.ping() return Response('', mimetype='text/css') -- cgit v1.2.3 From 1ec325adccc427fe05cf08da9a2d9d63da7365f4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 23 May 2023 18:16:37 +0200 Subject: [mod] limiter -> botdetection: modularization and documentation In order to be able to meet the outstanding requirements, the implementation is modularized and supplemented with documentation. This patch does not contain functional change, except it fixes issue #2455 ---- Aktivate limiter in the settings.yml and simulate a bot request by:: curl -H 'Accept-Language: de-DE,en-US;q=0.7,en;q=0.3' \ -H 'Accept: text/html' -H 'User-Agent: xyz' \ -H 'Accept-Encoding: gzip' \ 'http://127.0.0.1:8888/search?q=foo' In the LOG: DEBUG searx.botdetection.link_token : missing ping for this request: ..... Since ``BURST_MAX_SUSPICIOUS = 2`` you can repeat the query above two time before you get a "Too Many Requests" response. Closes: https://github.com/searxng/searxng/issues/2455 Signed-off-by: Markus Heiser --- searx/webapp.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'searx/webapp.py') diff --git a/searx/webapp.py b/searx/webapp.py index 815bfcabd..d6322447a 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -93,7 +93,8 @@ from searx.utils import ( ) from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH from searx.query import RawTextQuery -from searx.plugins import limiter, Plugin, plugins, initialize as plugin_initialize +from searx.plugins import Plugin, plugins, initialize as plugin_initialize +from searx.botdetection import link_token from searx.plugins.oa_doi_rewrite import get_doi_resolver from searx.preferences import ( Preferences, @@ -416,7 +417,7 @@ def render(template_name: str, **kwargs): kwargs['endpoint'] = 'results' if 'q' in kwargs else request.endpoint kwargs['cookies'] = request.cookies kwargs['errors'] = request.errors - kwargs['limiter_token'] = limiter.get_token() + kwargs['link_token'] = link_token.get_token() # values from the preferences kwargs['preferences'] = request.preferences @@ -643,10 +644,9 @@ def health(): return Response('OK', mimetype='text/plain') -@app.route('/limiter.css', methods=['GET', 'POST']) -def limiter_css(token=None): - if limiter.token_is_valid(token): - limiter.ping() +@app.route('/client.css', methods=['GET', 'POST']) +def client_token(token=None): + link_token.ping(request, token) return Response('', mimetype='text/css') -- cgit v1.2.3