From 39feb141bc8361915b3d80a976852b38851e0419 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 11 Jan 2020 12:50:40 +0100 Subject: docs(admin): add description of the utils/filtron.sh script Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 07dcb9bc5..009ca1c29 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -2,12 +2,40 @@ How to protect an instance ========================== +.. _filtron: https://github.com/asciimoo/filtron + Searx depens on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. -An application firewall, ``filtron`` solves exactly this problem. Information -on how to install it can be found at the `project page of filtron -`__. +An application firewall, filtron_ solves exactly this problem. Filtron is just +a middleware between your web server (nginx, apache, ...) and searx. + + +filtron & go +============ + +.. _Go: https://golang.org/ +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply +installed by ``go get`` package management (see `filtron README`_). If you use +filtron as middleware, a more isolated setup is recommended. + +#. Create a separated user account (``filtron``). +#. Download and install Go_ binary in users $HOME (``~filtron``). +#. Install filtron with the package management of Go_ (``go get -v -u + github.com/asciimoo/filtron``) +#. Setup a proper rule configuration :origin:`[ref] + ` (``/etc/filtron/rules.json``). +#. Setup a systemd service unit :origin:`[ref] + ` + (``/lib/systemd/system/filtron.service``). + +To simplify such a installation and the maintenance of; use our script +``utils/filtron.sh``: + +.. program-output:: ../utils/filtron.sh --help + :ellipsis: 0,5 Sample configuration of filtron -- cgit v1.2.3 From fe64274fb0405c47b3ba9039ab4e5b1440370a4d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 12 Jan 2020 15:53:35 +0100 Subject: docs: filtron.sh add remark about init systems Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 009ca1c29..e8a2bfb15 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -17,6 +17,13 @@ filtron & go .. _Go: https://golang.org/ .. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +.. sidebar:: init system + + ATM the ``filtron.sh`` supports only systemd init process used by debian, + ubuntu and many other dists. If you have a working init.d file to start/stop + filtron service, please contribute. + Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply installed by ``go get`` package management (see `filtron README`_). If you use filtron as middleware, a more isolated setup is recommended. -- cgit v1.2.3 From 94ac560dcc85e2e75d3b14706a65eb9488643eb4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 3 Feb 2020 13:25:51 +0100 Subject: doc: add descriptions of our tooling box in ./utils Initial started with the scripts for searx, filtron and morty installations. Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index e8a2bfb15..332c27203 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -1,14 +1,22 @@ + +.. _searx_filtron: + ========================== How to protect an instance ========================== +.. sidebar:: further reading + + - :ref:`filtron.sh` + .. _filtron: https://github.com/asciimoo/filtron Searx depens on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. An application firewall, filtron_ solves exactly this problem. Filtron is just -a middleware between your web server (nginx, apache, ...) and searx. +a middleware between your web server (nginx, apache, ...) and searx, we describe +such infratructures in chapter: :ref:`architecture`. filtron & go @@ -17,32 +25,10 @@ filtron & go .. _Go: https://golang.org/ .. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md - -.. sidebar:: init system - - ATM the ``filtron.sh`` supports only systemd init process used by debian, - ubuntu and many other dists. If you have a working init.d file to start/stop - filtron service, please contribute. - Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply installed by ``go get`` package management (see `filtron README`_). If you use -filtron as middleware, a more isolated setup is recommended. - -#. Create a separated user account (``filtron``). -#. Download and install Go_ binary in users $HOME (``~filtron``). -#. Install filtron with the package management of Go_ (``go get -v -u - github.com/asciimoo/filtron``) -#. Setup a proper rule configuration :origin:`[ref] - ` (``/etc/filtron/rules.json``). -#. Setup a systemd service unit :origin:`[ref] - ` - (``/lib/systemd/system/filtron.service``). - -To simplify such a installation and the maintenance of; use our script -``utils/filtron.sh``: - -.. program-output:: ../utils/filtron.sh --help - :ellipsis: 0,5 +filtron as middleware, a more isolated setup is recommended. To simplify such +an installation and the maintenance of, use our script :ref:`filtron.sh`. Sample configuration of filtron -- cgit v1.2.3 From eedd63ccd504099836d9f528d02daffa6bc826ad Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 4 Feb 2020 16:42:13 +0100 Subject: docs: revision of the installation instructions Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 332c27203..205269973 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -9,6 +9,11 @@ How to protect an instance - :ref:`filtron.sh` +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + .. _filtron: https://github.com/asciimoo/filtron Searx depens on external search services. To avoid the abuse of these services -- cgit v1.2.3 From 2f40f61f83afb34f0c4b95019a53050a504ce43a Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 4 Feb 2020 17:59:58 +0100 Subject: /etc/filtron/rules.json: normalize rules from docs & tooling box Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 192 +++++++++++++++++++++++++------------------------ 1 file changed, 98 insertions(+), 94 deletions(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 205269973..2bc663411 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -39,6 +39,10 @@ an installation and the maintenance of, use our script :ref:`filtron.sh`. Sample configuration of filtron =============================== +.. sidebar:: Tooling box + + - :origin:`/etc/filtron/rules.json ` + An example configuration can be find below. This configuration limits the access of: @@ -50,100 +54,100 @@ of: .. code:: json - [{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":"", - "limit":"", - "subrules":[ - { - "name":"roboagent limit", - "interval":"", - "limit":"", - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"botlimit", - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"IP limit", - "interval":"", - "limit":"", - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"rss/json limit", - "interval":"", - "limit":"", - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"useragent limit", - "interval":"", - "limit":"", - "aggregations":[ - "Header:User-Agent" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - } - ] - }] - + [ + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": "", + "limit": "", + "subrules": [ + { + "name": "roboagent limit", + "interval": "", + "limit": "", + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "IP limit", + "interval": "", + "limit": "", + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "interval": "", + "limit": "", + "stop": true, + "filters": [ + "Param:format=(csv|json|rss)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": "", + "limit": "", + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } + ] Route request through filtron -- cgit v1.2.3 From 387c6a77691fec514704bdf178b9ab94ad4abb40 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 6 Mar 2020 14:47:00 +0100 Subject: docs: improve description of uwsgi & ngingx setup Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 2bc663411..785b02261 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -1,5 +1,5 @@ -.. _searx_filtron: +.. _searx filtron: ========================== How to protect an instance @@ -8,6 +8,8 @@ How to protect an instance .. sidebar:: further reading - :ref:`filtron.sh` + - :ref:`nginx searx site` + .. contents:: Contents :depth: 2 @@ -150,6 +152,8 @@ of: ] +.. _filtron route request: + Route request through filtron ============================= @@ -167,12 +171,14 @@ Use it along with ``nginx`` with the following example configuration. .. code:: nginx location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_pass http://127.0.0.1:4004/; + proxy_pass http://127.0.0.1:4004/; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; } Requests are coming from port 4004 going through filtron and then forwarded to -port 8888 where a searx is being run. +port 8888 where a searx is being run. For a complete setup see: :ref:`nginx +searx site`. -- cgit v1.2.3 From c15337850e64562c376e5de57d8809a3a05b8a5d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 6 Mar 2020 22:06:19 +0100 Subject: fix: minor typos Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 785b02261..970ce25d2 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -18,7 +18,7 @@ How to protect an instance .. _filtron: https://github.com/asciimoo/filtron -Searx depens on external search services. To avoid the abuse of these services +Searx depends on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. An application firewall, filtron_ solves exactly this problem. Filtron is just -- cgit v1.2.3 From 853be4e11cd8a82b6b08f04db1717329907a5aaf Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 3 Apr 2020 20:24:40 +0200 Subject: utils/filtron.sh: add generic documentation Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 1 + 1 file changed, 1 insertion(+) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 970ce25d2..9615cd935 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -37,6 +37,7 @@ installed by ``go get`` package management (see `filtron README`_). If you use filtron as middleware, a more isolated setup is recommended. To simplify such an installation and the maintenance of, use our script :ref:`filtron.sh`. +.. _Sample configuration of filtron: Sample configuration of filtron =============================== -- cgit v1.2.3 From 58d5da8b57c5aeab92f551e8d175be67537c351c Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 11 Apr 2020 13:19:11 +0200 Subject: nginx: normalize installation (docs and script)s over all distros This is the revision of the documentation about the varous nginx installation variants. It also implements the nginx installation scripts for morty and filtron. Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 9615cd935..8bf641649 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -158,6 +158,12 @@ of: Route request through filtron ============================= +.. sidebar:: further reading + + - :ref:`filtron.sh overview` + - :ref:`installation nginx` + - :ref:`installation apache` + Filtron can be started using the following command: .. code:: sh @@ -171,15 +177,24 @@ Use it along with ``nginx`` with the following example configuration. .. code:: nginx - location / { + # https://example.org/searx + + location /searx { proxy_pass http://127.0.0.1:4004/; proxy_set_header Host $http_host; + proxy_set_header Connection $http_connection; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; } + location /searx/static { + /usr/local/searx/searx-src/searx/static; + } + + Requests are coming from port 4004 going through filtron and then forwarded to port 8888 where a searx is being run. For a complete setup see: :ref:`nginx searx site`. -- cgit v1.2.3 From 6ff20cef7385d6827042f00d295f51bbc213afab Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 18 Jun 2020 18:31:46 +0200 Subject: [fix] indentation of filtron's rules (json) Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 179 +++++++++++++++++++++++-------------------------- 1 file changed, 85 insertions(+), 94 deletions(-) (limited to 'docs/admin/filtron.rst') diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 8bf641649..93e430b1f 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -57,100 +57,91 @@ of: .. code:: json - [ - { "name": "search request", - "filters": [ - "Param:q", - "Path=^(/|/search)$" - ], - "interval": "", - "limit": "", - "subrules": [ - { - "name": "roboagent limit", - "interval": "", - "limit": "", - "filters": [ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "botlimit", - "limit": 0, - "stop": true, - "filters": [ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "IP limit", - "interval": "", - "limit": "", - "stop": true, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "rss/json limit", - "interval": "", - "limit": "", - "stop": true, - "filters": [ - "Param:format=(csv|json|rss)" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "useragent limit", - "interval": "", - "limit": "", - "aggregations": [ - "Header:User-Agent" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - } - ] - } - ] + [ + { + "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": "" + "limit": "", + "subrules": [ + { + "name": "missing Accept-Language", + "filters": ["!Header:Accept-Language"], + "limit": "", + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "suspiciously Connection=close header", + "filters": ["Header:Connection=close"], + "limit": "", + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "IP limit", + "interval": "" + "limit": "", + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "filters": [ + "Param:format=(csv|json|rss)" + ], + "interval": "" + "limit": "", + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": "" + "limit": "", + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } + ] .. _filtron route request: -- cgit v1.2.3