From dca4d276a45cc85f97db326636b4b671a3263225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Sun, 30 Oct 2016 01:01:22 +0200 Subject: add sample config of filtron --- docs/admin/filtron.rst | 114 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 docs/admin/filtron.rst (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst new file mode 100644 index 000000000..c422cb0a8 --- /dev/null +++ b/docs/admin/filtron.rst @@ -0,0 +1,114 @@ +How to protect an instance +========================== + +Searx depens on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. + +An application firewall, ``filtron`` solves exactly this problem. Information on how to install it can be found at the `project page of filtron `__. + +Sample configuration of filtron +------------------------------- + +An example configuration can be find below. This configuration limits the access of + + * scripts or applications (roboagent limit) + + * webcrawlers (botlimit) + + * IPs which send too many requests (IP limit) + + * too many json, csv, etc. requests (rss/json limit) + + * the same UserAgent of if too many requests (useragent limit) + + +.. code:: json + + [ + { + "name": "search request", + "filters": ["Param:q", "Path=^(/|/search)$"], + "interval": , + "limit": , + "subrules": [ + { + "name": "roboagent limit", + "interval": , + "limit": , + "filters": ["Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": ["Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "IP limit", + "interval": , + "limit": , + "stop": true, + "aggregations": ["Header:X-Forwarded-For"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "rss/json limit", + "interval": , + "limit": , + "stop": true, + "filters": ["Param:format=(csv|json|rss)"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "useragent limit", + "interval": , + "limit": , + "aggregations": ["Header:User-Agent"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + } + ] + } + ] + + + +Route request through filtron +----------------------------- + +Filtron can be started using the following command: + +.. code:: bash + + $ filtron -rules rules.json + +It listens on 127.0.0.1:4004 and forwards filtered requests to 127.0.0.1:8888 by default. + +Use it along with ``nginx`` with the following example configuration. + +.. code:: bash + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_pass http://127.0.0.1:4004/; + } + +Requests are coming from port 4004 going through filtron and then forwarded to port 8888 where a searx is being run. -- cgit v1.2.3 From 0a7479f1942a9e5fa63f55eda0977ce777088c71 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 4 Dec 2019 17:30:34 +0100 Subject: doc: [fix] WARNING: Could not lex literal_block as "json" docs/admin/filtron.rst:24: \ WARNING: Could not lex literal_block as "json". Highlighting skipped. Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 155 +++++++++++++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 62 deletions(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index c422cb0a8..9e55d5968 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -23,68 +23,99 @@ An example configuration can be find below. This configuration limits the access .. code:: json - [ - { - "name": "search request", - "filters": ["Param:q", "Path=^(/|/search)$"], - "interval": , - "limit": , - "subrules": [ - { - "name": "roboagent limit", - "interval": , - "limit": , - "filters": ["Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "botlimit", - "limit": 0, - "stop": true, - "filters": ["Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "IP limit", - "interval": , - "limit": , - "stop": true, - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "rss/json limit", - "interval": , - "limit": , - "stop": true, - "filters": ["Param:format=(csv|json|rss)"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "useragent limit", - "interval": , - "limit": , - "aggregations": ["Header:User-Agent"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - } + [{ + "name":"search request", + "filters":[ + "Param:q", + "Path=^(/|/search)$" + ], + "interval":"", + "limit":"", + "subrules":[ + { + "name":"roboagent limit", + "interval":"", + "limit":"", + "filters":[ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions":[ + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } ] - } - ] + }, + { + "name":"botlimit", + "limit":0, + "stop":true, + "filters":[ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions":[ + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + }, + { + "name":"IP limit", + "interval":"", + "limit":"", + "stop":true, + "aggregations":[ + "Header:X-Forwarded-For" + ], + "actions":[ + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + }, + { + "name":"rss/json limit", + "interval":"", + "limit":"", + "stop":true, + "filters":[ + "Param:format=(csv|json|rss)" + ], + "actions":[ + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + }, + { + "name":"useragent limit", + "interval":"", + "limit":"", + "aggregations":[ + "Header:User-Agent" + ], + "actions":[ + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + } + ] + }] @@ -101,7 +132,7 @@ It listens on 127.0.0.1:4004 and forwards filtered requests to 127.0.0.1:8888 by Use it along with ``nginx`` with the following example configuration. -.. code:: bash +.. code:: nginx location / { proxy_set_header Host $http_host; -- cgit v1.2.3 From e9fff4fde6d7a8bec3fae087d2afe1fce2145f22 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 12 Dec 2019 19:20:56 +0100 Subject: doc: proofread of the all reST sources (no content change) Normalize reST sources with best practice and KISS in mind. to name a few points: - simplify reST tables - make use of ``literal`` markup for monospace rendering - fix code-blocks for better rendering in HTML - normalize section header markup - limit all lines to a maximum of 79 characters - add option -H to the sudo command used in code blocks - drop useless indentation of lists - ... [1] https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 55 ++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 26 deletions(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 9e55d5968..07dcb9bc5 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -1,25 +1,26 @@ +========================== How to protect an instance ========================== -Searx depens on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. - -An application firewall, ``filtron`` solves exactly this problem. Information on how to install it can be found at the `project page of filtron `__. - -Sample configuration of filtron -------------------------------- +Searx depens on external search services. To avoid the abuse of these services +it is advised to limit the number of requests processed by searx. -An example configuration can be find below. This configuration limits the access of +An application firewall, ``filtron`` solves exactly this problem. Information +on how to install it can be found at the `project page of filtron +`__. - * scripts or applications (roboagent limit) - * webcrawlers (botlimit) - - * IPs which send too many requests (IP limit) - - * too many json, csv, etc. requests (rss/json limit) +Sample configuration of filtron +=============================== - * the same UserAgent of if too many requests (useragent limit) +An example configuration can be find below. This configuration limits the access +of: +- scripts or applications (roboagent limit) +- webcrawlers (botlimit) +- IPs which send too many requests (IP limit) +- too many json, csv, etc. requests (rss/json limit) +- the same UserAgent of if too many requests (useragent limit) .. code:: json @@ -120,26 +121,28 @@ An example configuration can be find below. This configuration limits the access Route request through filtron ------------------------------ +============================= Filtron can be started using the following command: -.. code:: bash +.. code:: sh - $ filtron -rules rules.json + $ filtron -rules rules.json -It listens on 127.0.0.1:4004 and forwards filtered requests to 127.0.0.1:8888 by default. +It listens on ``127.0.0.1:4004`` and forwards filtered requests to +``127.0.0.1:8888`` by default. Use it along with ``nginx`` with the following example configuration. .. code:: nginx - location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_pass http://127.0.0.1:4004/; - } + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_pass http://127.0.0.1:4004/; + } -Requests are coming from port 4004 going through filtron and then forwarded to port 8888 where a searx is being run. +Requests are coming from port 4004 going through filtron and then forwarded to +port 8888 where a searx is being run. -- cgit v1.2.3