[mod] botdetection: HTTP Fetch Metadata Request Headers

HTTP Fetch Metadata Request Headers [1][2] are used to detect bot requests. Bots
with invalid *Fetch Metadata* will be redirected to the intro (`index`)  page.

[1] https://www.w3.org/TR/fetch-metadata/
[2] https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

1 files changed, 7 insertions, 2 deletions

diff --git a/searx/limiter.py b/searx/limiter.py
index 293416366..92b38c68f 100644
--- a/searx/limiter.py
+++ b/searx/limiter.py
@@ -112,6 +112,7 @@ from searx.botdetection import (
     http_accept_encoding,
     http_accept_language,
     http_user_agent,
+    http_sec_fetch,
     ip_limit,
     ip_lists,
     get_network,
@@ -179,16 +180,17 @@ def filter_request(request: SXNG_Request) -> werkzeug.Response | None:
         logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
         return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
 
-    # methods applied on /
+    # methods applied on all requests
 
     for func in [
         http_user_agent,
     ]:
         val = func.filter_request(network, request, cfg)
         if val is not None:
+            logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(sxng_request))
             return val
 
-    # methods applied on /search
+    # methods applied on /search requests
 
     if request.path == '/search':
 
@@ -197,11 +199,14 @@ def filter_request(request: SXNG_Request) -> werkzeug.Response | None:
             http_accept_encoding,
             http_accept_language,
             http_user_agent,
+            http_sec_fetch,
             ip_limit,
         ]:
             val = func.filter_request(network, request, cfg)
             if val is not None:
+                logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(sxng_request))
                 return val
+
     logger.debug(f"OK {network}: %s", dump_request(sxng_request))
     return None