From 597c68b4aad0338b87e5d3443764d7866c86bd6d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 22 Sep 2023 10:57:11 +0200 Subject: [doc] move dosc of botdetection from developer to admin section Signed-off-by: Markus Heiser --- docs/admin/index.rst | 1 + docs/admin/searx.botdetection.rst | 52 +++++++++++++++++++++++++++++++++++++++ docs/src/searx.botdetection.rst | 48 ------------------------------------ searx/botdetection/__init__.py | 17 +++++++++++++ 4 files changed, 70 insertions(+), 48 deletions(-) create mode 100644 docs/admin/searx.botdetection.rst delete mode 100644 docs/src/searx.botdetection.rst diff --git a/docs/admin/index.rst b/docs/admin/index.rst index 583b6b428..f335f1631 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -15,6 +15,7 @@ Administrator documentation installation-apache update-searxng answer-captcha + searx.botdetection api architecture plugins diff --git a/docs/admin/searx.botdetection.rst b/docs/admin/searx.botdetection.rst new file mode 100644 index 000000000..c89f1cd93 --- /dev/null +++ b/docs/admin/searx.botdetection.rst @@ -0,0 +1,52 @@ +.. _botdetection: + +============= +Bot Detection +============= + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.botdetection + :members: + +.. automodule:: searx.botdetection.limiter + :members: + +.. automodule:: searx.botdetection.ip_lists + :members: + + +.. _botdetection rate limit: + +Rate limit +========== + +.. automodule:: searx.botdetection.ip_limit + :members: + +.. automodule:: searx.botdetection.link_token + :members: + + +.. _botdetection probe headers: + +Probe HTTP headers +================== + +.. automodule:: searx.botdetection.http_accept + :members: + +.. automodule:: searx.botdetection.http_accept_encoding + :members: + +.. automodule:: searx.botdetection.http_accept_language + :members: + +.. automodule:: searx.botdetection.http_connection + :members: + +.. automodule:: searx.botdetection.http_user_agent + :members: diff --git a/docs/src/searx.botdetection.rst b/docs/src/searx.botdetection.rst deleted file mode 100644 index 48261ceac..000000000 --- a/docs/src/searx.botdetection.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. _botdetection: - -============= -Bot Detection -============= - -.. contents:: - :depth: 2 - :local: - :backlinks: entry - -.. automodule:: searx.botdetection - :members: - -.. automodule:: searx.botdetection.limiter - :members: - -.. automodule:: searx.botdetection.ip_lists - :members: - - -Rate limit -========== - -.. automodule:: searx.botdetection.ip_limit - :members: - -.. automodule:: searx.botdetection.link_token - :members: - - -Probe HTTP headers -================== - -.. automodule:: searx.botdetection.http_accept - :members: - -.. automodule:: searx.botdetection.http_accept_encoding - :members: - -.. automodule:: searx.botdetection.http_accept_language - :members: - -.. automodule:: searx.botdetection.http_connection - :members: - -.. automodule:: searx.botdetection.http_user_agent - :members: diff --git a/searx/botdetection/__init__.py b/searx/botdetection/__init__.py index fcd8e5630..74f6c4263 100644 --- a/searx/botdetection/__init__.py +++ b/searx/botdetection/__init__.py @@ -2,6 +2,23 @@ # lint: pylint """.. _botdetection src: +The :ref:`limiter ` implements several methods to block bots: + +a. Analysis of the HTTP header in the request / can be easily bypassed. + +b. Block and pass lists in which IPs are listed / difficult to maintain, since + the IPs of bots are not all known and change over the time. + +c. Detection of bots based on the behavior of the requests and blocking and, if + necessary, unblocking of the IPs via a dynamically changeable IP block list. + +For dynamically changeable IP lists a Redis database is needed and for any kind +of IP list the determination of the IP of the client is essential. The IP of +the client is determined via the X-Forwarded-For_ HTTP header + +.. _X-Forwarded-For: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For + X-Forwarded-For =============== -- cgit v1.2.3