From 1ec325adccc427fe05cf08da9a2d9d63da7365f4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 23 May 2023 18:16:37 +0200 Subject: [mod] limiter -> botdetection: modularization and documentation In order to be able to meet the outstanding requirements, the implementation is modularized and supplemented with documentation. This patch does not contain functional change, except it fixes issue #2455 ---- Aktivate limiter in the settings.yml and simulate a bot request by:: curl -H 'Accept-Language: de-DE,en-US;q=0.7,en;q=0.3' \ -H 'Accept: text/html' -H 'User-Agent: xyz' \ -H 'Accept-Encoding: gzip' \ 'http://127.0.0.1:8888/search?q=foo' In the LOG: DEBUG searx.botdetection.link_token : missing ping for this request: ..... Since ``BURST_MAX_SUSPICIOUS = 2`` you can repeat the query above two time before you get a "Too Many Requests" response. Closes: https://github.com/searxng/searxng/issues/2455 Signed-off-by: Markus Heiser --- searx/botdetection/http_accept_encoding.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 searx/botdetection/http_accept_encoding.py (limited to 'searx/botdetection/http_accept_encoding.py') diff --git a/searx/botdetection/http_accept_encoding.py b/searx/botdetection/http_accept_encoding.py new file mode 100644 index 000000000..ae630fd68 --- /dev/null +++ b/searx/botdetection/http_accept_encoding.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +""" +Method ``http_accept_encoding`` +------------------------------- + +The ``http_accept_encoding`` method evaluates a request as the request of a +bot if the Accept-Encoding_ header .. + +- did not contain ``gzip`` AND ``deflate`` (if both values are missed) +- did not contain ``text/html`` + +.. _Accept-Encoding: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding + +""" + +from typing import Optional, Tuple +import flask + + +def filter_request(request: flask.Request) -> Optional[Tuple[int, str]]: + accept_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] + if not ('gzip' in accept_list or 'deflate' in accept_list): + return 429, "bot detected, HTTP header Accept-Encoding did not contain gzip nor deflate" + return None -- cgit v1.2.3 From 66fdec0eb92bf11c0bc477d6fb1df3dc783e4dcb Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 26 May 2023 17:24:43 +0200 Subject: [mod] limiter: add config file /etc/searxng/limiter.toml Signed-off-by: Markus Heiser --- searx/botdetection/http_accept_encoding.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'searx/botdetection/http_accept_encoding.py') diff --git a/searx/botdetection/http_accept_encoding.py b/searx/botdetection/http_accept_encoding.py index ae630fd68..191249711 100644 --- a/searx/botdetection/http_accept_encoding.py +++ b/searx/botdetection/http_accept_encoding.py @@ -14,12 +14,15 @@ bot if the Accept-Encoding_ header .. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding """ +# pylint: disable=unused-argument from typing import Optional, Tuple import flask +from searx.tools import config -def filter_request(request: flask.Request) -> Optional[Tuple[int, str]]: + +def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple[int, str]]: accept_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] if not ('gzip' in accept_list or 'deflate' in accept_list): return 429, "bot detected, HTTP header Accept-Encoding did not contain gzip nor deflate" -- cgit v1.2.3 From b8c7c2c9aa604fd1fb7be5559c9ad025ceb17aa4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 28 May 2023 18:58:31 +0200 Subject: [mod] botdetection - improve ip_limit and link_token methods - counting requests in LONG_WINDOW and BURST_WINDOW is not needed when the request is validated by the link_token method [1] - renew a ping-key on validation [2], this is needed for infinite scrolling, where no new token (CSS) is loaded. / this does not fix the BURST_MAX issue in the vanilla limiter - normalize the counter names of the ip_limit method to 'ip_limit.*' - just integrate the ip_limit method straight forward in the limiter plugin / non intermediate code --> ip_limit now returns None or a werkzeug.Response object that can be passed by the plugin to the flask application / non intermediate code that returns a tuple [1] https://github.com/searxng/searxng/pull/2357#issuecomment-1566113277 [2] https://github.com/searxng/searxng/pull/2357#discussion_r1208542206 [3] https://github.com/searxng/searxng/pull/2357#issuecomment-1566125979 Signed-off-by: Markus Heiser --- searx/botdetection/http_accept_encoding.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'searx/botdetection/http_accept_encoding.py') diff --git a/searx/botdetection/http_accept_encoding.py b/searx/botdetection/http_accept_encoding.py index 191249711..5301c5d9d 100644 --- a/searx/botdetection/http_accept_encoding.py +++ b/searx/botdetection/http_accept_encoding.py @@ -16,14 +16,16 @@ bot if the Accept-Encoding_ header .. """ # pylint: disable=unused-argument -from typing import Optional, Tuple +from typing import Optional import flask +import werkzeug from searx.tools import config +from ._helpers import too_many_requests -def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple[int, str]]: +def filter_request(request: flask.Request, cfg: config.Config) -> Optional[werkzeug.Response]: accept_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] if not ('gzip' in accept_list or 'deflate' in accept_list): - return 429, "bot detected, HTTP header Accept-Encoding did not contain gzip nor deflate" + return too_many_requests(request, "HTTP header Accept-Encoding did not contain gzip nor deflate") return None -- cgit v1.2.3 From 281e36f4b7848374535d5e953050ae73423191ca Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 1 Jun 2023 15:41:48 +0200 Subject: [fix] limiter: replace real_ip by IPv4/v6 network Closes: https://github.com/searxng/searxng/issues/2477 Signed-off-by: Markus Heiser --- searx/botdetection/http_accept_encoding.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'searx/botdetection/http_accept_encoding.py') diff --git a/searx/botdetection/http_accept_encoding.py b/searx/botdetection/http_accept_encoding.py index 5301c5d9d..60718a4ca 100644 --- a/searx/botdetection/http_accept_encoding.py +++ b/searx/botdetection/http_accept_encoding.py @@ -16,7 +16,12 @@ bot if the Accept-Encoding_ header .. """ # pylint: disable=unused-argument -from typing import Optional +from __future__ import annotations +from ipaddress import ( + IPv4Network, + IPv6Network, +) + import flask import werkzeug @@ -24,8 +29,13 @@ from searx.tools import config from ._helpers import too_many_requests -def filter_request(request: flask.Request, cfg: config.Config) -> Optional[werkzeug.Response]: +def filter_request( + network: IPv4Network | IPv6Network, + request: flask.Request, + cfg: config.Config, +) -> werkzeug.Response | None: + accept_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')] if not ('gzip' in accept_list or 'deflate' in accept_list): - return too_many_requests(request, "HTTP header Accept-Encoding did not contain gzip nor deflate") + return too_many_requests(network, "HTTP header Accept-Encoding did not contain gzip nor deflate") return None -- cgit v1.2.3