diff options
62 files changed, 2210 insertions, 4190 deletions
diff --git a/.config.sh b/.config.sh deleted file mode 100644 index cc7663648..000000000 --- a/.config.sh +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8; mode: sh -*- -# SPDX-License-Identifier: AGPL-3.0-or-later -# shellcheck shell=bash disable=SC2034 -# -# This file should be edited only ones just before the installation of any -# service is done. After the installation of the searx service a copy of this -# file is placed into the $SEARX_SRC of the instance, e.g.:: -# -# /usr/local/searx/searx-src/.config.sh -# -# .. hint:: -# -# Before you change a value here, You have to fully uninstall any previous -# installation of searx, morty and filtron services! - -# utils/searx.sh -# -------------- - -# The setup of the SearXNG instance is done in the settings.yml -# (SEARXNG_SETTINGS_PATH). Read the remarks in [1] carefully and don't forget to -# rebuild instance's environment (make buildenv) if needed. The settings.yml -# file of an already installed instance is shown by:: -# -# $ ./utils/searx.sh --help -# ---- SearXNG instance setup (already installed) -# SEARXNG_SETTINGS_PATH : /etc/searxng/settings.yml -# SEARX_SRC : /usr/local/searx/searx-src -# -# [1] https://docs.searxng.org/admin/engines/settings.html - -# utils/filtron.sh -# ---------------- - -# FILTRON_API="127.0.0.1:4005" -# FILTRON_LISTEN="127.0.0.1:4004" - -# utils/morty.sh -# -------------- - -# morty listen address -# MORTY_LISTEN="127.0.0.1:3000" -# PUBLIC_URL_PATH_MORTY="/morty/" - -# system services -# --------------- - -# Common $HOME folder of the service accounts -# SERVICE_HOME_BASE="/usr/local" - -# **experimental**: Set SERVICE_USER to run all services by one account, but be -# aware that removing discrete components might conflict! -# SERVICE_USER=searx diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index fba56f120..d20cd6c63 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -26,7 +26,7 @@ jobs: - name: Install Ubuntu packages run: | - sudo ./utils/searx.sh install packages + sudo ./utils/searxng.sh install packages - name: Set up Python uses: actions/setup-python@v2 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ea9dc0972..f43f467e5 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v2 - name: Install Ubuntu packages run: | - sudo ./utils/searx.sh install packages + sudo ./utils/searxng.sh install packages sudo apt install firefox - name: Set up Python uses: actions/setup-python@v2 @@ -55,7 +55,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Install Ubuntu packages - run: sudo ./utils/searx.sh install buildhost + run: sudo ./utils/searxng.sh install buildhost - name: Set up Python uses: actions/setup-python@v2 with: @@ -82,7 +82,7 @@ jobs: fetch-depth: '0' persist-credentials: false - name: Install Ubuntu packages - run: sudo ./utils/searx.sh install buildhost + run: sudo ./utils/searxng.sh install buildhost - name: Set up Python uses: actions/setup-python@v2 with: @@ -59,17 +59,16 @@ test.shell: utils/brand.env \ $(MTOOLS) \ utils/lib.sh \ - utils/lib_install.sh \ utils/lib_nvm.sh \ utils/lib_static.sh \ utils/lib_go.sh \ utils/lib_redis.sh \ utils/filtron.sh \ utils/searx.sh \ + utils/searxng.sh \ utils/morty.sh \ utils/lxc.sh \ - utils/lxc-searx.env \ - .config.sh + utils/lxc-searxng.env $(Q)$(MTOOLS) build_msg TEST "$@ OK" diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot index c4ee5f3c1..526fb53da 100644 --- a/docs/admin/arch_public.dot +++ b/docs/admin/arch_public.dot @@ -1,33 +1,30 @@ digraph G { - node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"]; + node [style=filled, shape=box, fillcolor="#ffffcc", fontname=Sans]; edge [fontname="Sans"]; - browser [label="Browser", shape=Mdiamond]; - rp [label="Reverse Proxy", href="https://docs.searxng.org/utils/filtron.sh.html#public-reverse-proxy"]; - filtron [label="Filtron", href="https://docs.searxng.org/utils/filtron.sh.html"]; - morty [label="Morty", href="https://docs.searxng.org/utils/morty.sh.html"]; - static [label="Static files", href="url to configure static files"]; - uwsgi [label="uwsgi", href="https://docs.searxng.org/utils/searx.sh.html"] - searx1 [label="Searx #1"]; - searx2 [label="Searx #2"]; - searx3 [label="Searx #3"]; - searx4 [label="Searx #4"]; + browser [label="browser", shape=tab, fillcolor=aliceblue]; + rp [label="reverse proxy"]; + static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray]; + uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searx.sh.html"] + redis [label="redis DB", shape=cylinder]; + searxng1 [label="SearXNG #1", fontcolor=blue3]; + searxng2 [label="SearXNG #2", fontcolor=blue3]; + searxng3 [label="SearXNG #3", fontcolor=blue3]; + searxng4 [label="SearXNG #4", fontcolor=blue3]; browser -> rp [label="HTTPS"] - subgraph cluster_searx { - label = "Searx instance" fontname="Sans"; + subgraph cluster_searxng { + label = "SearXNG instance" fontname=Sans; bgcolor="#fafafa"; { rank=same; static rp }; - rp -> morty [label="optional: images and HTML pages proxy"]; - rp -> static [label="optional: reverse proxy serves directly static files"]; - rp -> filtron [label="HTTP"]; - filtron -> uwsgi [label="HTTP"]; - uwsgi -> searx1; - uwsgi -> searx2; - uwsgi -> searx3; - uwsgi -> searx4; + rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray]; + rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"]; + uwsgi -> searxng1 -> redis; + uwsgi -> searxng2 -> redis; + uwsgi -> searxng3 -> redis; + uwsgi -> searxng4 -> redis; } } diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst index db99c9f55..d0d40715d 100644 --- a/docs/admin/architecture.rst +++ b/docs/admin/architecture.rst @@ -8,17 +8,19 @@ Architecture - Reverse Proxy: :ref:`Apache <apache searxng site>` & :ref:`nginx <nginx searxng site>` - - Filtron: :ref:`searxng filtron` - - Morty: :ref:`searxng morty` - uWSGI: :ref:`searxng uwsgi` - SearXNG: :ref:`installation basic` Herein you will find some hints and suggestions about typical architectures of SearXNG infrastructures. -We start with a contribution from :pull-searx:`@dalf <1776#issuecomment-567917320>`. -It shows a *reference* setup for public SearXNG instances which can build up and -maintained by the scripts from our :ref:`toolboxing`. +.. _architecture uWSGI: + +uWSGI Setup +=========== + +We start with a *reference* setup for public SearXNG instances which can be build +up and maintained by the scripts from our :ref:`toolboxing`. .. _arch public: @@ -26,3 +28,11 @@ maintained by the scripts from our :ref:`toolboxing`. :alt: arch_public.dot Reference architecture of a public SearXNG setup. + +The reference installation activates ``server.limiter``, ``server.image_proxy`` +and ``ui.static_use_hash`` (:origin:`/etc/searxng/settings.yml +<utils/templates/etc/searxng/settings.yml>`) + +.. literalinclude:: ../../utils/templates/etc/searxng/settings.yml + :language: yaml + :end-before: # preferences: diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index e23327ba8..6926469ef 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -15,19 +15,19 @@ Buildhosts :backlinks: entry To get best results from build, its recommend to install additional packages -on build hosts (see :ref:`searx.sh`).:: +on build hosts (see :ref:`searxng.sh`).:: - sudo -H ./utils/searx.sh install buildhost + sudo -H ./utils/searxng.sh install buildhost This will install packages needed by searx: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START distro-packages :end-before: END distro-packages and packages needed to build docuemtation and run tests: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START build-packages :end-before: END build-packages diff --git a/docs/admin/engines/nosql-engines.rst b/docs/admin/engines/nosql-engines.rst index a50b9c367..68fd0e8bc 100644 --- a/docs/admin/engines/nosql-engines.rst +++ b/docs/admin/engines/nosql-engines.rst @@ -42,11 +42,11 @@ Extra Dependencies For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searx-src`) you can use -:ref:`searx.sh`:: +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... .. _engine redis_server: diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 71ad29b43..0b4b984d7 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -207,10 +207,14 @@ Global Settings ``secret_key`` : ``$SEARXNG_SECRET`` Used for cryptography purpose. +.. _limiter: + ``limiter`` : Rate limit the number of request on the instance, block some bots. The :ref:`limiter plugin` requires a :ref:`settings redis` database. +.. _image_proxy: + ``image_proxy`` : Allow your instance of SearXNG of being able to proxy images. Uses memory space. @@ -225,9 +229,13 @@ Global Settings ``ui:`` ------- +.. _cache busting: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#caching_static_assets_with_cache_busting + .. code:: yaml ui: + static_use_hash: false default_locale: "" query_in_title: false infinite_scroll: false @@ -236,6 +244,11 @@ Global Settings theme_args: simple_style: auto +.. _static_use_hash: + +``static_use_hash`` : + Enables `cache busting`_ of static files. + ``default_locale`` : SearXNG interface language. If blank, the locale is detected by using the browser language. If it doesn't work, or you are deploying a language diff --git a/docs/admin/engines/sql-engines.rst b/docs/admin/engines/sql-engines.rst index d91383214..6b6a4cb68 100644 --- a/docs/admin/engines/sql-engines.rst +++ b/docs/admin/engines/sql-engines.rst @@ -98,11 +98,11 @@ Extra Dependencies For using :ref:`engine postgresql` or :ref:`engine mysql_server` you need to install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searx-src`) you can use -:ref:`searx.sh`:: +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... .. _engine postgresql: diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst deleted file mode 100644 index 60d7cbeba..000000000 --- a/docs/admin/filtron.rst +++ /dev/null @@ -1,193 +0,0 @@ - -.. _searxng filtron: - -========================== -How to protect an instance -========================== - -.. tip:: - - To protect your instance a installation of filtron (as described here) is no - longer needed, alternatively activate the :ref:`limiter plugin` in your - ``settings.yml``. Note that the :ref:`limiter plugin` requires a :ref:`Redis - <settings redis>` database. - - -.. sidebar:: further reading - - - :ref:`filtron.sh` - - :ref:`nginx searxng site` - -.. _filtron: https://github.com/searxng/filtron - -SearXNG depends on external search services. To avoid the abuse of these services -it is advised to limit the number of requests processed by SearXNG. - -An application firewall, filtron_ solves exactly this problem. Filtron is just -a middleware between your web server (nginx, apache, ...) and searx, we describe -such infrastructures in chapter: :ref:`architecture`. - - -filtron & go -============ - -.. _Go: https://golang.org/ -.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md - -Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply -installed by ``go get`` package management (see `filtron README`_). If you use -filtron as middleware, a more isolated setup is recommended. To simplify such -an installation and the maintenance of, use our script :ref:`filtron.sh`. - -.. _Sample configuration of filtron: - -Sample configuration of filtron -=============================== - -.. sidebar:: Tooling box - - - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>` - -An example configuration can be find below. This configuration limits the access -of: - -- scripts or applications (roboagent limit) -- webcrawlers (botlimit) -- IPs which send too many requests (IP limit) -- too many json, csv, etc. requests (rss/json limit) -- the same UserAgent of if too many requests (useragent limit) - -.. code:: json - - [ - { - "name": "search request", - "filters": [ - "Param:q", - "Path=^(/|/search)$" - ], - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "subrules": [ - { - "name": "missing Accept-Language", - "filters": ["!Header:Accept-Language"], - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "suspiciously Connection=close header", - "filters": ["Header:Connection=close"], - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "IP limit", - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "rss/json limit", - "filters": [ - "Param:format=(csv|json|rss)" - ], - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "useragent limit", - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "aggregations": [ - "Header:User-Agent" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - } - ] - } - ] - - -.. _filtron route request: - -Route request through filtron -============================= - -.. sidebar:: further reading - - - :ref:`filtron.sh overview` - - :ref:`installation nginx` - - :ref:`installation apache` - -Filtron can be started using the following command: - -.. code:: sh - - $ filtron -rules rules.json - -It listens on ``127.0.0.1:4004`` and forwards filtered requests to -``127.0.0.1:8888`` by default. - -Use it along with ``nginx`` with the following example configuration. - -.. code:: nginx - - # https://example.org/searx - - location /searx { - proxy_pass http://127.0.0.1:4004/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - } - - location /searx/static { - /usr/local/searx/searx-src/searx/static; - } - - -Requests are coming from port 4004 going through filtron and then forwarded to -port 8888 where a SearXNG is being run. For a complete setup see: :ref:`nginx -searxng site`. diff --git a/docs/admin/index.rst b/docs/admin/index.rst index e6c0636b2..705167306 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -7,17 +7,15 @@ Administrator documentation :caption: Contents installation + installation-docker + installation-scripts installation-searxng installation-uwsgi installation-nginx installation-apache - installation-docker - installation-switch2ng update-searxng engines/index api architecture - filtron - morty plugins buildhosts diff --git a/docs/admin/installation-apache.rst b/docs/admin/installation-apache.rst index b60e20ad1..673a37ee4 100644 --- a/docs/admin/installation-apache.rst +++ b/docs/admin/installation-apache.rst @@ -1,13 +1,13 @@ .. _installation apache: -=================== -Install with apache -=================== +====== +Apache +====== .. _Apache: https://httpd.apache.org/ .. _Apache Debian: https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x): -.. _README.Debian: +.. _apache2.README.Debian: https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian .. _Apache Arch Linux: https://wiki.archlinux.org/index.php/Apache_HTTP_Server @@ -23,7 +23,9 @@ Install with apache https://httpd.apache.org/docs/current/en/configuring.html .. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost .. _LoadModule: - https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule + https://httpd.apache.org/docs/mod/mod_so.html#loadmodule +.. _IncludeOptional: + https://httpd.apache.org/docs/mod/core.html#includeoptional .. _DocumentRoot: https://httpd.apache.org/docs/trunk/mod/core.html#documentroot .. _Location: @@ -32,11 +34,30 @@ Install with apache https://uwsgi-docs.readthedocs.io/en/latest/Apache.html .. _mod_proxy_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi +.. _mod_proxy_http: + https://httpd.apache.org/docs/current/mod/mod_proxy_http.html +.. _mod_proxy: + https://httpd.apache.org/docs/current/mod/mod_proxy.html + + +This section explains how to set up a SearXNG instance using the HTTP server Apache_. +If you did use the :ref:`installation scripts` and do not have any special preferences +you can install the :ref:`SearXNG site <apache searxng site>` using +:ref:`searxng.sh <searxng.sh overview>`: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install apache + +If you have special interests or problems with setting up Apache, the following +section might give you some guidance. + .. sidebar:: further read - `Apache Arch Linux`_ - - `Apache Debian`_ and `README.Debian`_ + - `Apache Debian`_ + - `apache2.README.Debian`_ - `Apache Fedora`_ - `Apache directives`_ @@ -45,23 +66,8 @@ Install with apache :local: :backlinks: entry ----- - -**Install** :ref:`apache searxng site` using :ref:`filtron.sh <filtron.sh overview>` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh apache install - -**Install** :ref:`apache searxng site` using :ref:`morty.sh <morty.sh overview>` - -.. code:: bash - $ sudo -H ./utils/morty.sh apache install - ----- - -The apache HTTP server +The Apache HTTP server ====================== If Apache_ is not installed, install it now. If apache_ is new to you, the @@ -73,13 +79,13 @@ Directives`_ documentation gives first orientation. There is also a list of .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H apt-get install apache2 .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H pacman -S apache sudo -H systemctl enable httpd @@ -87,21 +93,21 @@ Directives`_ documentation gives first orientation. There is also a list of .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H dnf install httpd sudo -H systemctl enable httpd sudo -H systemctl start httpd -Now at http://localhost you should see any kind of *Welcome* or *Test* page. -How this default intro site is configured, depends on the linux distribution +Now at http://localhost you should see some kind of *Welcome* or *Test* page. +How this default site is configured, depends on the linux distribution (compare `Apache directives`_). .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash less /etc/apache2/sites-enabled/000-default.conf @@ -115,7 +121,7 @@ How this default intro site is configured, depends on the linux distribution .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash less /etc/httpd/conf/httpd.conf @@ -130,8 +136,8 @@ How this default intro site is configured, depends on the linux distribution Require all granted </Directory> - The *welcome* page of Arch Linux is a page showing directory located at - ``DocumentRoot``. This is *directory* page is generated by the Module + The *welcome* page of Arch Linux is a page showing the directory located + at ``DocumentRoot``. This *directory* page is generated by the Module `mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_: .. code:: apache @@ -142,7 +148,7 @@ How this default intro site is configured, depends on the linux distribution .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash less /etc/httpd/conf/httpd.conf @@ -163,323 +169,204 @@ How this default intro site is configured, depends on the linux distribution less /etc/httpd/conf.d/welcome.conf -.. _apache searxng site: -Apache Reverse Proxy -==================== +.. _Debian's Apache layout: -.. sidebar:: public to the internet? +Debian's Apache layout +---------------------- - If your SearXNG instance is public, stop here and first install :ref:`filtron - reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see - :ref:`installation scripts`. If already done, follow setup: *SearXNG via - filtron plus morty*. +Be aware, Debian's Apache layout is quite different from the standard Apache +configuration. For details look at the apache2.README.Debian_ +(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on +Debian: -To setup a Apache revers proxy you have to enable the *headers* and *proxy* -modules and create a `Location`_ configuration for the SearXNG site. In most -distributions you have to un-comment the lines in the main configuration file, -except in :ref:`The Debian Layout`. +* :man:`apache2ctl`: Apache HTTP server control interface +* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules +* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations +* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites + +.. _apache modules: + +Apache modules +-------------- + +To load additional modules, in most distributions you have to un-comment the +lines with the corresponding LoadModule_ directive, except in :ref:`Debian's +Apache layout`. .. tabs:: .. group-tab:: Ubuntu / debian - In the Apache setup, enable headers and proxy modules: + :ref:`Debian's Apache layout` uses :man:`a2enmod` and :man:`a2dismod` to + activate or disable modules: - .. code:: sh + .. code:: bash + sudo -H a2enmod ssl sudo -H a2enmod headers sudo -H a2enmod proxy sudo -H a2enmod proxy_http - - In :ref:`The Debian Layout` you create a ``searxng.conf`` with the - ``<Location /searx >`` directive and save this file in the *sites - available* folder at ``/etc/apache2/sites-available``. To enable the - ``searxng.conf`` use :man:`a2ensite`: - - .. code:: sh - - sudo -H a2ensite searxng.conf + sudo -H a2enmod proxy_uwsgi .. group-tab:: Arch Linux - In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy - modules (LoadModule_): + In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_ + directives: .. code:: apache - FIXME needs test - - LoadModule headers_module modules/mod_headers.so - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule ssl_module modules/mod_ssl.so + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so .. group-tab:: Fedora / RHEL - In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy - modules (LoadModule_): + In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_ + directives: .. code:: apache - FIXME needs test + LoadModule ssl_module modules/mod_ssl.so + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - LoadModule headers_module modules/mod_headers.so - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_http_module modules/mod_proxy_http.so -With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the -proxied host. +.. _apache sites: -.. _apache searxng via filtron plus morty: - -.. tabs:: - - .. group-tab:: SearXNG via filtron plus morty - - Use this setup, if your instance is public to the internet, compare - figure: :ref:`architecture <arch public>` and :ref:`installation scripts`. - - 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on - *localhost 4004* (:ref:`filtron route request`): - - .. code:: apache - - <Location /searx > - - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://127.0.0.1:4004 - RequestHeader set X-Script-Name /searx - - </Location> - - 2. Configure reverse proxy for :ref:`morty <searxng morty>`, listening on - *localhost 3000* - - .. code:: apache - - ProxyPreserveHost On - - <Location /morty > - - # SetEnvIf Request_URI "/morty" dontlog - # CustomLog /dev/null combined env=dontlog - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPass http://127.0.0.1:3000 - RequestHeader set X-Script-Name /morty - - </Location> - - For a fully result proxification add :ref:`morty's <searxng morty>` **public - URL** to your :origin:`searx/settings.yml`: - - .. code:: yaml - - result_proxy: - # replace example.org with your server's public name - url : https://example.org/morty - key : !!binary "insert_your_morty_proxy_key_here" - - server: - image_proxy : True - -uWSGI support -============= - -Be warned, with this setup, your instance isn't :ref:`protected <searxng -filtron>`, nevertheless it is good enough for intranet usage. In modern Linux -distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache -package and you need to install only the :ref:`uWSGI <searxng uwsgi>` package: +Apache sites +------------ .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + In :ref:`Debian's Apache layout` you create a ``searxng.conf`` with the + ``<Location /searxng >`` directive and save this file in the *sites + available* folder at ``/etc/apache2/sites-available``. To enable the + ``searxng.conf`` use :man:`a2ensite`: - sudo -H apt-get install uwsgi + .. code:: bash - # Ubuntu =< 18.04 - sudo -H apt-get install libapache2-mod-proxy-uwsgi + sudo -H a2ensite searxng.conf .. group-tab:: Arch Linux - .. code:: sh - - sudo -H pacman -S uwsgi - - .. group-tab:: Fedora / RHEL - - .. code:: sh - - sudo -H dnf install uwsgi - -The next example shows a configuration using the `uWSGI Apache support`_ via -unix sockets and `mod_proxy_uwsgi`_. - -For socket communication, you have to activate ``socket = -/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888`` -configuration in your :ref:`uwsgi ini file <uwsgi configuration>`. If not -already exists, create a folder for the unix sockets, which can be used by the -SearXNG account (see :ref:`create searxng user`): - -.. code:: bash - - sudo -H mkdir -p /run/uwsgi/app/searx/ - sudo -H chown -R searx:searx /run/uwsgi/app/searx/ - -If the server is public; to limit access to your intranet replace ``Allow from -all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class. - -.. tabs:: - - .. group-tab:: Ubuntu / debian + In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_ + directive: .. code:: apache - LoadModule headers_module /usr/lib/apache2/mod_headers.so - LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so - LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so - - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + IncludeOptional sites-enabled/*.conf - <Location /searx> + Create two folders, one for the *available sites* and one for the *enabled sites*: - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. code:: bash - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + mkdir -p /etc/httpd/sites-available + mkdir -p /etc/httpd/sites-enabled - </Location> + Create configuration at ``/etc/httpd/sites-available`` and place a + symlink to ``sites-enabled``: - .. group-tab:: Arch Linux - - .. code:: apache + .. code:: bash - FIXME needs test + sudo -H ln -s /etc/httpd/sites-available/searxng.conf \ + /etc/httpd/sites-enabled/searxng.conf - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + .. group-tab:: Fedora / RHEL - <Location /searx> + In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_ + directive: - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. code:: apache - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + IncludeOptional sites-enabled/*.conf - </Location> + Create two folders, one for the *available sites* and one for the *enabled sites*: - .. group-tab:: Fedora / RHEL + .. code:: bash - .. code:: apache + mkdir -p /etc/httpd/sites-available + mkdir -p /etc/httpd/sites-enabled - FIXME needs test + Create configuration at ``/etc/httpd/sites-available`` and place a + symlink to ``sites-enabled``: - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - <IfModule proxy_uwsgi_module> + .. code:: bash - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + sudo -H ln -s /etc/httpd/sites-available/searxng.conf \ + /etc/httpd/sites-enabled/searxng.conf - <Location /searx> - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all +.. _apache searxng site: - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ +Apache's SearXNG site +===================== - </Location> +.. _mod_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi - </IfModule> +.. sidebar:: uWSGI - .. group-tab:: old mod_wsgi + Use mod_proxy_uwsgi_ / don't use the old mod_uwsgi_ anymore. - We show this only for historical reasons, DON'T USE `mod_uwsgi - <https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi>`_. - ANYMORE! +To proxy the incoming requests to the SearXNG instance Apache needs the +mod_proxy_ module (:ref:`apache modules`). - .. code:: apache +.. sidebar:: HTTP headers - <IfModule mod_uwsgi.c> + With ProxyPreserveHost_ the incoming ``Host`` header is passed to the proxied + host. - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog +Depending on what your SearXNG installation is listening on, you need a http +mod_proxy_http_) or socket (mod_proxy_uwsgi_) communication to upstream. - <Location /searx > +The :ref:`installation scripts` installs the :ref:`reference setup +<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default. +You can install and activate your own ``searxng.conf`` like shown in +:ref:`apache sites`. - Require all granted +.. tabs:: - Options FollowSymLinks Indexes - SetHandler uwsgi-handler - uWSGISocket /run/uwsgi/app/searx/socket + .. group-tab:: socket - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START apache socket + :end-before: END apache socket - </Location> + .. group-tab:: http - </IfModule> + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START apache http + :end-before: END apache http .. _restart apache: -Restart service -=============== +Restart service: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H systemctl restart apache2 - sudo -H service uwsgi restart searx + sudo -H service uwsgi restart searxng .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H systemctl restart httpd - sudo -H systemctl restart uwsgi@searx + sudo -H systemctl restart uwsgi@searxng .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H systemctl restart httpd sudo -H touch /etc/uwsgi.d/searxng.ini @@ -489,27 +376,13 @@ disable logs ============ For better privacy you can disable Apache logs. In the examples above activate -one of the lines and `restart apache`_:: - +one of the lines and `restart apache`_: - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog +.. code:: apache -The ``CustomLog`` directive disable logs for the whole (virtual) server, use it -when the URL of the service does not have a path component (``/searx``) / is -located at root (``/``). + SetEnvIf Request_URI "/searxng" dontlog + # CustomLog /dev/null combined env=dontlog -.. _The Debian Layout: - -The Debian Layout -================= - -Be aware that the Debian layout is quite different from the standard Apache -configuration. For details look at the README.Debian_ -(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on -Debian: - -* :man:`apache2ctl`: Apache HTTP server control interface -* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules -* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations -* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites +The ``CustomLog`` directive disables logs for the entire (virtual) server, use it +when the URL of the service does not have a path component (``/searxng``), so when +SearXNG is located at root (``/``). diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst index cd91af382..1457d6a6b 100644 --- a/docs/admin/installation-docker.rst +++ b/docs/admin/installation-docker.rst @@ -1,37 +1,60 @@ - .. _installation docker: -=================== -Docker installation -=================== +================ +Docker Container +================ .. _ENTRYPOINT: https://docs.docker.com/engine/reference/builder/#entrypoint +.. _searxng/searxng @dockerhub: https://hub.docker.com/r/searxng/searxng .. _searxng-docker: https://github.com/searxng/searxng-docker -.. _[filtron]: https://hub.docker.com/r/dalf/filtron -.. _[morty]: https://hub.docker.com/r/dalf/morty .. _[caddy]: https://hub.docker.com/_/caddy +.. _Redis: https://redis.io/ + +---- .. sidebar:: info + - `searxng/searxng @dockerhub`_ - :origin:`Dockerfile` - - `searxng/searxng @dockerhub <https://hub.docker.com/r/searxng/searxng>`_ - `Docker overview <https://docs.docker.com/get-started/overview>`_ - - `Docker Cheat Sheet <https://www.docker.com/sites/default/files/d8/2019-09/docker-cheat-sheet.pdf>`_ - - `Alpine Linux <https://alpinelinux.org>`_ `(wiki) <https://en.wikipedia.org/wiki/Alpine_Linux>`__ `apt packages <https://pkgs.alpinelinux.org/packages>`_ + - `Docker Cheat Sheet <https://docs.docker.com/get-started/docker_cheatsheet.pdf>`_ + - `Alpine Linux <https://alpinelinux.org>`_ + `(wiki) <https://en.wikipedia.org/wiki/Alpine_Linux>`__ + `apt packages <https://pkgs.alpinelinux.org/packages>`_ - Alpine's ``/bin/sh`` is :man:`dash` -.. tip:: +**If you intend to create a public instance using Docker, use our well maintained +docker container** + +- `searxng/searxng @dockerhub`_. + +.. sidebar:: hint + + The rest of this article is of interest only to those who want to create and + maintain their own Docker images. + +The sources are hosted at searxng-docker_ and the container includes: + +- a HTTPS reverse proxy `[caddy]`_ and +- a Redis_ DB + +The `default SearXNG setup <https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml>`_ +of this container: + +- enables :ref:`limiter <limiter>` to protect against bots +- enables :ref:`image proxy <image_proxy>` for better privacy +- enables :ref:`cache busting <static_use_hash>` to save bandwith + +---- - If you intend to create a public instance using Docker, use our well - maintained searxng-docker_ image which includes - - :ref:`protection <searxng filtron>` `[filtron]`_, - - a :ref:`result proxy <searxng morty>` `[morty]`_ and - - a HTTPS reverse proxy `[caddy]`_. +Get Docker +========== -Make sure you have `installed Docker <https://docs.docker.com/get-docker/>`_ and -on Linux, don't forget to add your user to the docker group (log out and log -back in so that your group membership is re-evaluated): +If you plan to build and maintain a docker image by yourself, make sure you have +`Docker installed <https://docs.docker.com/get-docker/>`_. On Linux don't +forget to add your user to the docker group (log out and log back in so that +your group membership is re-evaluated): .. code:: sh diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst index fdbcdf032..8e5299588 100644 --- a/docs/admin/installation-nginx.rst +++ b/docs/admin/installation-nginx.rst @@ -1,8 +1,8 @@ .. _installation nginx: -================== -Install with nginx -================== +===== +NGINX +===== .. _nginx: https://docs.nginx.com/nginx/admin-guide/ @@ -19,6 +19,19 @@ Install with nginx .. _SCRIPT_NAME: https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name +This section explains how to set up a SearXNG instance using the HTTP server nginx_. +If you have used the :ref:`installation scripts` and do not have any special preferences +you can install the :ref:`SearXNG site <nginx searxng site>` using +:ref:`searxng.sh <searxng.sh overview>`: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install nginx + +If you have special interests or problems with setting up nginx, the following +section might give you some guidance. + + .. sidebar:: further reading - nginx_ @@ -27,39 +40,23 @@ Install with nginx - `Getting Started wiki`_ - `uWSGI support from nginx`_ + .. contents:: Contents :depth: 2 :local: :backlinks: entry ----- - -**Install** :ref:`nginx searxng site` using :ref:`filtron.sh <filtron.sh overview>` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh nginx install - -**Install** :ref:`nginx searxng site` using :ref:`morty.sh <morty.sh overview>` - -.. code:: bash - - $ sudo -H ./utils/morty.sh nginx install - ----- - The nginx HTTP server ===================== -If nginx_ is not installed (uwsgi will not work with the package nginx-light), -install it now. +If nginx_ is not installed, install it now. .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H apt-get install nginx @@ -81,18 +78,18 @@ install it now. Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you see a *Fedora Webserver - Test Page*. The test page comes from the default -`nginx server configuration`_. How this default intro site is configured, +`nginx server configuration`_. How this default site is configured, depends on the linux distribution: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash less /etc/nginx/nginx.conf - there is a line including site configurations from: + There is one line that includes site configurations from: .. code:: nginx @@ -104,7 +101,7 @@ depends on the linux distribution: less /etc/nginx/nginx.conf - in there is a configuration section named ``server``: + There is a configuration section named ``server``: .. code-block:: nginx @@ -120,249 +117,121 @@ depends on the linux distribution: less /etc/nginx/nginx.conf - there is a line including site configurations from: + There is one line that includes site configurations from: .. code:: nginx include /etc/nginx/conf.d/*.conf; + .. _nginx searxng site: -A nginx SearXNG site +NGINX's SearXNG site ==================== -.. sidebar:: public to the internet? - - If your SearXNG instance is public, stop here and first install :ref:`filtron - reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see - :ref:`installation scripts`. If already done, follow setup: *SearXNG via - filtron plus morty*. +Now you have to create a configuration file (``searxng.conf``) for the SearXNG +site. If nginx_ is new to you, the `nginx beginners guide`_ is a good starting +point and the `Getting Started wiki`_ is always a good resource *to keep in the +pocket*. -Now you have to create a configuration for the SearXNG site. If nginx_ is new to -you, the `nginx beginners guide`_ is a good starting point and the `Getting -Started wiki`_ is always a good resource *to keep in the pocket*. +Depending on what your SearXNG installation is listening on, you need a http or socket +communication to upstream. .. tabs:: - .. group-tab:: Ubuntu / debian - - Create configuration at ``/etc/nginx/sites-available/searxng`` and place a - symlink to sites-enabled: - - .. code:: sh - - sudo -H ln -s /etc/nginx/sites-available/searxng /etc/nginx/sites-enabled/searxng - - .. group-tab:: Arch Linux + .. group-tab:: socket - In the ``/etc/nginx/nginx.conf`` file, replace the configuration section - named ``server``. + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START nginx socket + :end-before: END nginx socket - .. group-tab:: Fedora / RHEL + .. group-tab:: http - Create configuration at ``/etc/nginx/conf.d/searxng`` and place a - symlink to sites-enabled: + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START nginx http + :end-before: END nginx http -.. _nginx searxng via filtron plus morty: +The :ref:`installation scripts` installs the :ref:`reference setup +<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default. .. tabs:: - .. group-tab:: SearXNG via filtron plus morty - - Use this setup, if your instance is public to the internet, compare - figure: :ref:`architecture <arch public>` and :ref:`installation scripts`. - - 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on - *localhost 4004* (:ref:`filtron route request`): - - .. code:: nginx - - # https://example.org/searx - - location /searx { - proxy_pass http://127.0.0.1:4004/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - } - - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/static/; - } - - - 2. Configure reverse proxy for :ref:`morty <searxng morty>`, listening on - *localhost 3000*: - - .. code:: nginx - - # https://example.org/morty - - location /morty { - proxy_pass http://127.0.0.1:3000/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - } - - For a fully result proxification add :ref:`morty's <searxng morty>` **public - URL** to your :origin:`searx/settings.yml`: - - .. code:: yaml - - result_proxy: - # replace example.org with your server's public name - url : https://example.org/morty - key : !!binary "insert_your_morty_proxy_key_here" - - server: - image_proxy : True - - - .. group-tab:: proxy or uWSGI - - Be warned, with this setup, your instance isn't :ref:`protected <searxng - filtron>`. Nevertheless it is good enough for intranet usage and it is a - excellent example of; *how different services can be set up*. The next - example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI - application <uwsgi configuration>`, listening on ``http = - 127.0.0.1:8888``. - - .. code:: nginx + .. group-tab:: Ubuntu / debian - # https://hostname.local/ + Create configuration at ``/etc/nginx/sites-available/`` and place a + symlink to ``sites-enabled``: - location / { - proxy_pass http://127.0.0.1:8888; + .. code:: bash - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_buffering off; - } + sudo -H ln -s /etc/nginx/sites-available/searxng.conf \ + /etc/nginx/sites-enabled/searxng.conf - Alternatively you can use the `uWSGI support from nginx`_ via unix - sockets. For socket communication, you have to activate ``socket = - /run/uwsgi/app/searx/socket`` and comment out the ``http = - 127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file <uwsgi - configuration>`. + .. group-tab:: Arch Linux - The example shows a nginx virtual ``server`` configuration, listening on - port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at - location ``/`` by importing the `uwsgi_params`_ and passing requests to - the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the - :ref:`searx-src clone <searx-src>` and wraps directly the - :origin:`searx/static/` content at ``location /static``. + In the ``/etc/nginx/nginx.conf`` file, in the ``server`` section add a + `include <https://nginx.org/en/docs/ngx_core_module.html#include>`_ + directive: .. code:: nginx server { - # replace hostname.local with your server's name - server_name hostname.local; - - listen 80; - listen [::]:80; - - location / { - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - - root /usr/local/searx/searx-src/searx; - location /static { } + # ... + include /etc/nginx/default.d/*.conf; + # ... } - If not already exists, create a folder for the unix sockets, which can be - used by the SearXNG account: + Create two folders, one for the *available sites* and one for the *enabled sites*: .. code:: bash - mkdir -p /run/uwsgi/app/searx/ - sudo -H chown -R searx:searx /run/uwsgi/app/searx/ - - .. group-tab:: \.\. at subdir URL - - Be warned, with these setups, your instance isn't :ref:`protected <searxng - filtron>`. The examples are just here to demonstrate how to export the - SearXNG application from a subdirectory URL ``https://example.org/searx/``. - - .. code:: nginx - - # https://hostname.local/searx - - location /searx { - proxy_pass http://127.0.0.1:8888; + mkdir -p /etc/nginx/default.d + mkdir -p /etc/nginx/default.apps-available - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - proxy_buffering off; - } - - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/static/; - } + Create configuration at ``/etc/nginx/default.apps-available`` and place a + symlink to ``default.d``: - The ``X-Script-Name /searx`` is needed by the SearXNG implementation to - calculate relative URLs correct. The next example shows a uWSGI - configuration. Since there are no HTTP headers in a (u)WSGI protocol, the - value is shipped via the SCRIPT_NAME_ in the WSGI environment. + .. code:: bash - .. code:: nginx + sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \ + /etc/nginx/default.d/searxng.conf - # https://hostname.local/searx + .. group-tab:: Fedora / RHEL - location /searx { - uwsgi_param SCRIPT_NAME /searx; - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } + Create a folder for the *available sites*: - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/; - } + .. code:: bash - For SearXNG to work correctly the ``base_url`` must be set in the - :origin:`searx/settings.yml`. + mkdir -p /etc/nginx/default.apps-available - .. code:: yaml + Create configuration at ``/etc/nginx/default.apps-available`` and place a + symlink to ``conf.d``: - server: - # replace example.org with your server's public name - base_url : https://example.org/searx/ + .. code:: bash + sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \ + /etc/nginx/conf.d/searxng.conf -Restart service: +Restart services: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx - sudo -H service uwsgi restart searx + sudo -H service uwsgi restart searxng .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx - sudo -H systemctl restart uwsgi@searx + sudo -H systemctl restart uwsgi@searxng - .. group-tab:: Fedora + .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx sudo -H touch /etc/uwsgi.d/searxng.ini diff --git a/docs/admin/installation-scripts.rst b/docs/admin/installation-scripts.rst new file mode 100644 index 000000000..e256a2461 --- /dev/null +++ b/docs/admin/installation-scripts.rst @@ -0,0 +1,62 @@ +.. _installation scripts: + +=================== +Installation Script +=================== + +.. sidebar:: Update the OS first! + + To avoid unwanted side effects, update your OS before installing SearXNG. + +The following will install a setup as shown in :ref:`the reference architecture +<arch public>`. First you need to get a clone of the repository. The clone is only needed for +the installation procedure and some maintenance tasks. + +.. sidebar:: further read + + - :ref:`toolboxing` + +Jump to a folder that is readable by *others* and start to clone SearXNG, +alternatively you can create your own fork and clone from there. + +.. code:: bash + + $ cd ~/Downloads + $ git clone https://github.com/searxng/searxng.git searxng + $ cd searxng + +.. sidebar:: further read + + - :ref:`inspect searxng` + +To install a SearXNG :ref:`reference setup <use_default_settings.yml>` +including a :ref:`uWSGI setup <architecture uWSGI>` as described in the +:ref:`installation basic` and in the :ref:`searxng uwsgi` section type: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install all + +.. attention:: + + For the installation procedure, use a *sudoer* login to run the scripts. If + you install from ``root``, take into account that the scripts are creating a + ``searxng`` user. In the installation procedure this new created user does + need read access to the cloned SearXNG repository, which is not the case if you clone + it into a folder below ``/root``! + +.. sidebar:: further read + + - :ref:`update searxng` + +.. _caddy: https://hub.docker.com/_/caddy + +When all services are installed and running fine, you can add SearXNG to your +HTTP server. We do not have any preferences for the HTTP server, you can use +whatever you prefer. + +We use caddy in our :ref:`docker image <installation docker>` and we have +implemented installation procedures for: + +- :ref:`installation nginx` +- :ref:`installation apache` diff --git a/docs/admin/installation-searxng.rst b/docs/admin/installation-searxng.rst index b14139310..9152784fc 100644 --- a/docs/admin/installation-searxng.rst +++ b/docs/admin/installation-searxng.rst @@ -9,15 +9,16 @@ Step by step installation :local: :backlinks: entry -Step by step installation with virtualenv. For Ubuntu, be sure to have enable -universe repository. + +In this section we show the setup of a SearXNG instance that will be installed +by the :ref:`installation scripts`. .. _install packages: Install packages ================ -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START distro-packages :end-before: END distro-packages @@ -30,32 +31,32 @@ Install packages Create user =========== -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START create user :end-before: END create user -.. _searx-src: +.. _searxng-src: Install SearXNG & dependencies ============================== -Start a interactive shell from new created user and clone searx: +Start a interactive shell from new created user and clone SearXNG: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START clone searxng :end-before: END clone searxng In the same shell create *virtualenv*: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START create virtualenv :end-before: END create virtualenv -To install searx's dependencies, exit the SearXNG *bash* session you opened above -and restart a new. Before install, first check if your *virtualenv* was sourced +To install SearXNG's dependencies, exit the SearXNG *bash* session you opened above +and start a new one. Before installing, check if your *virtualenv* was sourced from the login (*~/.profile*): -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START manage.sh update_packages :end-before: END manage.sh update_packages @@ -77,30 +78,41 @@ Configuration - :ref:`settings use_default_settings` - :origin:`/etc/searxng/settings.yml <utils/templates/etc/searxng/settings.yml>` -To create a initial ``/etc/searxng/settings.yml`` you can start with a copy of -the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup +To create a initial ``/etc/searxng/settings.yml`` we recommend to start with a +copy of the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup :ref:`use default settings <settings use_default_settings>` from -:origin:`searx/settings.yml`. +:origin:`searx/settings.yml` and is shown in the tab *"Use default settings"* +below. This setup: -For a *minimal setup*, configure like shown below – replace ``searx@$(uname --n)`` with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit -``/etc/searxng/settings.yml`` to your needs. +- enables :ref:`limiter <limiter>` to protect against bots +- enables :ref:`image proxy <image_proxy>` for better privacy +- enables :ref:`cache busting <static_use_hash>` to save bandwith -.. kernel-include:: $DOCS_BUILD/includes/searx.rst - :start-after: START searxng config - :end-before: END searxng config +Modify the ``/etc/searxng/settings.yml`` to your needs: .. tabs:: .. group-tab:: Use default settings - .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml - :language: yaml + .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml + :language: yaml + :end-before: # hostname_replace: + + To see the entire file jump to :origin:`utils/templates/etc/searxng/settings.yml` .. group-tab:: searx/settings.yml - .. literalinclude:: ../../searx/settings.yml - :language: yaml + .. literalinclude:: ../../searx/settings.yml + :language: yaml + :end-before: # hostname_replace: + + To see the entire file jump to :origin:`searx/settings.yml` + +For a *minimal setup* you need to set ``server:secret_key``. + +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START searxng config + :end-before: END searxng config Check @@ -110,11 +122,11 @@ To check your SearXNG setup, optional enable debugging and start the *webapp*. SearXNG looks at the exported environment ``$SEARXNG_SETTINGS_PATH`` for a configuration file. -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START check searxng installation :end-before: END check searxng installation If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the -debug option in ``settings.yml``. You can now exit SearXNG user bash (enter exit +debug option in ``settings.yml``. You can now exit SearXNG user bash session (enter exit command twice). At this point SearXNG is not demonized; uwsgi allows this. diff --git a/docs/admin/installation-switch2ng.rst b/docs/admin/installation-switch2ng.rst deleted file mode 100644 index 8863cd71e..000000000 --- a/docs/admin/installation-switch2ng.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. _installation switch2ng: - -============================ -Switch from searx to SearXNG -============================ - -.. sidebar:: info - - - :pull:`456` - - :pull:`A comment about rolling release <446#issuecomment-954730358>` - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - -If you have a searx installation on your sever and want to switch to SearXNG, -you need to uninstall searx first. If you have an old searx docker installation -replace your docker image / see :ref:`installation docker`. - -If your searx instance was installed *"Step by step"* or by the *"Installation -scripts"*, you need to undo the installation procedure completely. If you have -morty & filtron installed, it is recommended to uninstall these services also. -In case of scripts, to uninstall use the scripts from the origin you installed -searx from. - -If you have removed the old searx installation, clone from SearXNG and and start -with your installation procedure (e.g. :ref:`installation scripts`): - -.. code:: bash - - $ cd ~/Downloads - $ git clone https://github.com/searxng/searxng.git searxng - $ cd searxng - $ ... - -``.config.sh`` -============== - -Please take into account; SearXNG has normalized ``.config.sh`` with -``settings.yml`` and some of the environment settings has been removed from or -renamed in the ``.config.sh``: - -- :patch:`[mod] normalize .config.sh with settings.yml <f61c918d>` -- :patch:`[fix] ./utils/filtron.sh - FILTRON_TARGET from YAML settings <7196a9b5>` -- :patch:`SearXNG: SEARXNG_SETTINGS_PATH <253b8503>` - - -Check after Installation -======================== - -Once you have done your installation, you can run a SearXNG *check* procedure, -to see if there are some left overs. In this example there exists a *old* -``/etc/searx/settings.yml``:: - - $ sudo -H ./utils/searx.sh install check - - ============================ - SearXNG (check installation) - ============================ - ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/ - INFO: SearXNG instance already installed at: /usr/local/searx/searx-src - ... - INFO: Service account searx exists. - INFO: ~searx: python environment is available. - INFO: ~searx: SearXNG software is installed. - INFO: uWSGI app searxng.ini is enabled. - INFO searx : merge the default settings ( /usr/local/searx/searx-src/searx/settings.yml ) and the user setttings ( /etc/searxng/settings.yml ) - INFO searx : max_request_timeout=None - - -To *check* the filtron & morty installations, use similar commands:: - - $ sudo -H /utils/filtron.sh install check - $ sudo -H /utils/morty.sh install check diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst index 1021283f9..a6ea85608 100644 --- a/docs/admin/installation-uwsgi.rst +++ b/docs/admin/installation-uwsgi.rst @@ -1,7 +1,7 @@ .. _searxng uwsgi: ===== -uwsgi +uWSGI ===== .. sidebar:: further reading @@ -29,51 +29,77 @@ uwsgi Origin uWSGI ============ -How uWSGI is implemented by distributors is different. uWSGI itself -recommend two methods +.. _Tyrant mode: + https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting -`systemd.unit`_ template files as described here `One service per app in systemd`_. +How uWSGI is implemented by distributors varies. The uWSGI project itself +recommends two methods: - There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app - placed at dedicated locations. Take archlinux and a searxng.ini as example:: +1. `systemd.unit`_ template file as described here `One service per app in systemd`_: - unit template --> /usr/lib/systemd/system/uwsgi@.service - uwsgi ini files --> /etc/uwsgi/searxng.ini + There is one `systemd unit template`_ on the system installed and one `uwsgi + ini file`_ per uWSGI-app placed at dedicated locations. Take archlinux and a + ``searxng.ini`` as example:: - The SearXNG app can be maintained as know from common systemd units:: + systemd template unit: /usr/lib/systemd/system/uwsgi@.service + contains: [Service] + ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/%I.ini - systemctl enable uwsgi@searx - systemctl start uwsgi@searx - systemctl restart uwsgi@searx - systemctl stop uwsgi@searx + SearXNG application: /etc/uwsgi/searxng.ini + links to: /etc/uwsgi/apps-available/searxng.ini -The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps. + The SearXNG app (template ``/etc/uwsgi/%I.ini``) can be maintained as known + from common systemd units: + + .. code:: sh + + $ systemctl enable uwsgi@searxng + $ systemctl start uwsgi@searxng + $ systemctl restart uwsgi@searxng + $ systemctl stop uwsgi@searxng + +2. The `uWSGI Emperor`_ which fits for maintaining a large range of uwsgi + apps and there is a `Tyrant mode`_ to secure multi-user hosting. The Emperor mode is a special uWSGI instance that will monitor specific - events. The Emperor mode (service) is started by a (common, not template) - systemd unit. The Emperor service will scan specific directories for `uwsgi - ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the - timestamp is modified, a corresponding action takes place: a new uWSGI - instance is started, reload or stopped. Take Fedora and a searxng.ini as - example:: + events. The Emperor mode (the service) is started by a (common, not template) + systemd unit. + + The Emperor service will scan specific directories for `uwsgi ini file`_\s + (also know as *vassals*). If a *vassal* is added, removed or the timestamp is + modified, a corresponding action takes place: a new uWSGI instance is started, + reload or stopped. Take Fedora and a ``searxng.ini`` as example:: + + to install & start SearXNG instance create --> /etc/uwsgi.d/searxng.ini + to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini + to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini - to start a new SearXNG instance create --> /etc/uwsgi.d/searxng.ini - to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini - to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini Distributors ============ The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors mostly offer their users, even if they differ in the way they implement both -modes and their defaults. Another point they might differ is the packaging of +modes and their defaults. Another point they might differ in is the packaging of plugins (if so, compare :ref:`install packages`) and what the default python interpreter is (python2 vs. python3). -Fedora starts a Emperor by default, while archlinux does not start any uwsgi -service by default. Worth to know; debian (ubuntu) follow a complete different -approach. *debian*: your are familiar with the apache infrastructure? .. they -do similar for the uWSGI infrastructure (with less comfort), the folders are:: +While archlinux does not start a uWSGI service by default, Fedora (RHEL) starts +a Emperor in `Tyrant mode`_ by default (you should have read :ref:`uWSGI Tyrant +mode pitfalls`). Worth to know; debian (ubuntu) follow a complete different +approach, read see :ref:`Debian's uWSGI layout`. + +.. _Debian's uWSGI layout: + +Debian's uWSGI layout +--------------------- + +.. _uwsgi.README.Debian: + https://salsa.debian.org/uwsgi-team/uwsgi/-/raw/debian/latest/debian/uwsgi.README.Debian + +Be aware, Debian's uWSGI layout is quite different from the standard uWSGI +configuration. Your are familiar with :ref:`Debian's Apache layout`? .. they do a +similar thing for the uWSGI infrastructure. The folders are:: /etc/uwsgi/apps-available/ /etc/uwsgi/apps-enabled/ @@ -82,29 +108,52 @@ The `uwsgi ini file`_ is enabled by a symbolic link:: ln -s /etc/uwsgi/apps-available/searxng.ini /etc/uwsgi/apps-enabled/ -From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You -could control specific instance(s) by issuing:: +More details can be found in the uwsgi.README.Debian_ +(``/usr/share/doc/uwsgi/README.Debian.gz``). Some commands you should know on +Debian: - service uwsgi <command> <confname> <confname> ... +.. code:: none - sudo -H service uwsgi start searx - sudo -H service uwsgi stop searx + Commands recognized by init.d script + ==================================== -My experience is, that this command is a bit buggy. + You can issue to init.d script following commands: + * start | starts daemon + * stop | stops daemon + * reload | sends to daemon SIGHUP signal + * force-reload | sends to daemon SIGTERM signal + * restart | issues 'stop', then 'start' commands + * status | shows status of daemon instance (running/not running) -.. _uwsgi configuration: + 'status' command must be issued with exactly one argument: '<confname>'. -Alltogether -=========== + Controlling specific instances of uWSGI + ======================================= -Create the configuration ini-file according to your distribution (see below) and -restart the uwsgi application. + You could control specific instance(s) by issuing: + + SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi <command> <confname> <confname>... + + where: + * <command> is one of 'start', 'stop' etc. + * <confname> is the name of configuration file (without extension) + + For example, this is how instance for /etc/uwsgi/apps-enabled/hello.xml is + started: + + SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi start hello + + +.. _uWSGI maintenance: + +uWSGI maintenance +================= .. tabs:: .. group-tab:: Ubuntu / debian - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description ubuntu-20.04 :end-before: END searxng uwsgi-description ubuntu-20.04 @@ -112,7 +161,7 @@ restart the uwsgi application. .. group-tab:: Arch Linux - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description arch :end-before: END searxng uwsgi-description arch @@ -120,16 +169,28 @@ restart the uwsgi application. .. group-tab:: Fedora / RHEL - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description fedora :end-before: END searxng uwsgi-description fedora +.. _uwsgi setup: + +uWSGI setup +=========== + +Create the configuration ini-file according to your distribution and restart the +uwsgi application. As shown below, the :ref:`installation scripts` installs by +default: + +- a uWSGI setup that listens on a socket and +- enables :ref:`cache busting <static_use_hash>`. + .. tabs:: .. group-tab:: Ubuntu / debian - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini ubuntu-20.04 :end-before: END searxng uwsgi-appini ubuntu-20.04 @@ -137,7 +198,7 @@ restart the uwsgi application. .. group-tab:: Arch Linux - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini arch :end-before: END searxng uwsgi-appini arch @@ -145,6 +206,63 @@ restart the uwsgi application. .. group-tab:: Fedora / RHEL - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini fedora :end-before: END searxng uwsgi-appini fedora + + +.. _uWSGI Tyrant mode pitfalls: + +Pitfalls of the Tyrant mode +=========================== + +The implementation of the process owners and groups in the `Tyrant mode`_ is +somewhat unusual and requires special consideration. In `Tyrant mode`_ mode the +Emperor will run the vassal using the UID/GID of the vassal configuration file +(user and group of the app ``.ini`` file). + +.. _#2099@uWSGI: https://github.com/unbit/uwsgi/issues/2099 +.. _#752@uWSGI: https://github.com/unbit/uwsgi/pull/752 +.. _#2425uWSGI: https://github.com/unbit/uwsgi/issues/2425 + +Without option ``emperor-tyrant-initgroups=true`` in ``/etc/uwsgi.ini`` the +process won't get the additional groups, but this option is not available in +2.0.x branch (see `#2099@uWSGI`_) the feature `#752@uWSGI`_ has been merged (on +Oct. 2014) to the master branch of uWSGI but had never been released; the last +major release is from Dec. 2013, since the there had been only bugfix releases +(see `#2425uWSGI`_). To shorten up: + + **In Tyrant mode, there is no way to get additional groups, and the uWSGI + process misses additional permissions that may be needed.** + +For example on Fedora (RHEL): If you try to install a redis DB with socket +communication and you want to connect to it from the SearXNG uWSGI, you will see a +*Permission denied* in the log of your instance:: + + ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... + ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + ERROR:searx.plugins.limiter: init limiter DB failed!!! + +Even if your *searxng* user of the uWSGI process is added to additional groups +to give access to the socket from the redis DB:: + + $ groups searxng + searxng : searxng searxng-redis + +To see the effective groups of the uwsgi process, you have to look at the status +of the process, by example:: + + $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' + searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini + searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini + +Here you can see that the additional "Groups" of PID 186 are unset (missing gid +of ``searxng-redis``):: + + $ cat /proc/186/task/186/status + ... + Uid: 993 993 993 993 + Gid: 993 993 993 993 + FDSize: 128 + Groups: + ... diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 91f82e504..cae51be63 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -4,109 +4,19 @@ Installation ============ -.. sidebar:: info - - :ref:`installation switch2ng` - *You're spoilt for choice*, choose your preferred method of installation. - :ref:`installation docker` - :ref:`installation scripts` - :ref:`installation basic` -The :ref:`installation basic` is good enough for intranet usage and it is a -excellent illustration of *how a SearXNG instance is build up*. If you place your -instance public to the internet you should really consider to install a -:ref:`filtron reverse proxy <filtron.sh>` and for privacy a :ref:`result proxy -<morty.sh>` is mandatory. - -Therefore, if you do not have any special preferences, its recommend to use the -:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling -box <toolboxing>` as described below. - -.. _installation scripts: - -Installation scripts -==================== - -.. sidebar:: Update OS first! - - To avoid unwanted side effects, update your OS before installing SearXNG. - -The following will install a setup as shown in :ref:`architecture`. First you -need to get a clone. The clone is only needed for the installation procedure -and some maintenance tasks (alternatively you can create your own fork). - -For the installation procedure, use a *sudoer* login to run the scripts. If you -install from ``root``, take into account that the scripts are creating a -``searx``, a ``filtron`` and a ``morty`` user. In the installation procedure -these new created users do need read access to the clone of searx, which is not -the case if you clone into a folder below ``/root``. - -.. code:: bash - - $ cd ~/Downloads - $ git clone https://github.com/searxng/searxng.git searxng - $ cd searxng - -.. sidebar:: further read - - - :ref:`toolboxing` - - :ref:`update searxng` - - :ref:`inspect searxng` - -**Install** :ref:`SearXNG service <searx.sh>` - -This installs SearXNG as described in :ref:`installation basic`. - -.. code:: bash - - $ sudo -H ./utils/searx.sh install all - -**Install** :ref:`filtron reverse proxy <filtron.sh>` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh install all - -**Install** :ref:`result proxy <morty.sh>` - -.. code:: bash - - $ sudo -H ./utils/morty.sh install all - -If all services are running fine, you can add it to your HTTP server: - -**Install** HTTP - -- :ref:`installation apache` -- :ref:`installation nginx` - -**Install** :ref:`external plugins <dev plugin>` - -Use SearXNG's ``shell`` to install external plugins. In the example below we -install the SearXNG plugins from **The Green Web Foundation** `[ref] -<https://www.thegreenwebfoundation.org/news/searching-the-green-web-with-searx/>`__: - -.. code:: bash - - $ sudo -H ./utils/searx.sh shell - // exit with [CTRL-D] - (searx-pyenv) searx@ryzen:~$ pip install git+https://github.com/return42/tgwf-searx-plugins - -In the :ref:`settings.yml` activate the ``plugins:`` section and add module -``only_show_green_results`` from tgwf-searx-plugins. - -.. code:: yaml - - plugins: - - only_show_green_results - -.. _git stash: https://git-scm.com/docs/git-stash +The :ref:`installation basic` is an excellent illustration of *how a SearXNG +instance is build up* (see :ref:`architecture uWSGI`). If you do not have any +special preferences, its recommend to use the :ref:`installation docker` or the +:ref:`installation scripts`. -.. tip:: +.. attention:: - About script's installation options have a look at chapter :ref:`toolboxing - setup`. How to brand your instance see chapter :ref:`settings global`. To - *stash* your instance's setup, `git stash`_ your clone's :origin:`.config.sh` - file . + SearXNG is growing rapidly, you should regularly read our :ref:`migrate and + stay tuned` section. If you want to upgrade an existing instance or migrate + from searx to SearXNG, you should read this section first! diff --git a/docs/admin/morty.rst b/docs/admin/morty.rst deleted file mode 100644 index b6bd2ea56..000000000 --- a/docs/admin/morty.rst +++ /dev/null @@ -1,40 +0,0 @@ - -.. _searxng morty: - -========================= -How to setup result proxy -========================= - -.. sidebar:: further reading - - - :ref:`morty.sh` - -.. _morty: https://github.com/asciimoo/morty -.. _morty's README: https://github.com/asciimoo/morty - -By default SearXNG can only act as an image proxy for result images, but it is -possible to proxify all the result URLs with an external service, morty_. - -To use this feature, morty has to be installed and activated in SearXNG's -``settings.yml``. Add the following snippet to your ``settings.yml`` and -restart searx: - -.. code:: yaml - - result_proxy: - url : http://127.0.0.1:3000/ - key : !!binary "insert_your_morty_proxy_key_here" - -Note that the example above (``http://127.0.0.1:3000``) is only for single-user -instances without a HTTP proxy. If your morty service is public, the url is the -address of the reverse proxy (e.g ``https://example.org/morty``). - -For more information about *result proxy* have a look at *"SearXNG via filtron -plus morty"* in the :ref:`nginx <nginx searxng via filtron plus morty>` and -:ref:`apache <apache searxng via filtron plus morty>` sections. - -``url`` - Is the address of the running morty service. - -``key`` - Is an optional argument, see `morty's README`_ for more information. diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index 4c541bf57..15735b0ba 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -1,59 +1,115 @@ +=================== +SearXNG maintenance +=================== + +.. sidebar:: further read + + - :ref:`toolboxing` + - :ref:`uWSGI maintenance` + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + .. _update searxng: -============= How to update ============= How to update depends on the :ref:`installation` method. If you have used the -:ref:`installation scripts`, use ``update`` command from the scripts. - -**Update** :ref:`SearXNG service <searx.sh>` +:ref:`installation scripts`, use the ``update`` command from the :ref:`searxng.sh` +script. .. code:: sh - sudo -H ./utils/searx.sh update searx + sudo -H ./utils/searxng.sh instance update + +.. _inspect searxng: -**Update** :ref:`filtron reverse proxy <filtron.sh>` +How to inspect & debug +====================== + +How to debug depends on the :ref:`installation` method. If you have used the +:ref:`installation scripts`, use the ``inspect`` command from the :ref:`searxng.sh` +script. .. code:: sh - sudo -H ./utils/filtron.sh update filtron + sudo -H ./utils/searxng.sh instance inspect -**Update** :ref:`result proxy <morty.sh>` +.. _migrate and stay tuned: -.. code:: bash +Migrate and stay tuned! +======================= - $ sudo -H ./utils/morty.sh update morty +.. sidebar:: info -.. _inspect searxng: + - :pull:`1332` + - :pull:`456` + - :pull:`A comment about rolling release <446#issuecomment-954730358>` -====================== -How to inspect & debug -====================== +SearXNG is a *rolling release*; each commit to the master branch is a release. +SearXNG is growing rapidly, the services and opportunities are change every now +and then, to name just a few: -.. sidebar:: further read +- Bot protection has been switched from filtron to SearXNG's :ref:`limiter + <limiter>`, this requires a :ref:`Redis <settings redis>` database. - - :ref:`toolboxing` - - :ref:`Makefile` +- The image proxy morty is no longer needed, it has been replaced by the + :ref:`image proxy <image_proxy>` from SearXNG. -How to debug depends on the :ref:`installation` method. If you have used the -:ref:`installation scripts`, use ``inspect`` command from the scripts. +- To save bandwith :ref:`cache busting <static_use_hash>` has been implemented. + To get in use, the ``static-expires`` needs to be set in the :ref:`uwsgi + setup`. -**Inspect** :ref:`SearXNG service <searx.sh>` +To stay tuned and get in use of the new features, instance maintainers have to +update the SearXNG code regularly (see :ref:`update searxng`). As the above +examples show, this is not always enough, sometimes services have to be set up +or reconfigured and sometimes services that are no longer needed should be +uninstalled. -.. code:: sh +.. hint:: - sudo -H ./utils/searx.sh inspect service + First of all: SearXNG is installed by the script :ref:`searxng.sh`. If you + have old filtron, morty or searx setup you should consider complete + uninstall/reinstall. -**Inspect** :ref:`filtron reverse proxy <filtron.sh>` -.. code:: sh +remove obsolete services +------------------------ + +If your searx instance was installed *"Step by step"* or by the *"Installation +scripts"*, you need to undo the installation procedure completely. If you have +morty & filtron installed, it is recommended to uninstall these services also. +In case of scripts, to uninstall use the scripts from the origin you installed +searx from or try:: + + $ sudo -H ./utils/filtron.sh remove all + $ sudo -H ./utils/morty.sh remove all + $ sudo -H ./utils/searx.sh remove all + +.. hint:: + + If you are migrate from searx take into account that the ``.config.sh`` is no + longer used. - sudo -H ./utils/filtron.sh inspect service -**Inspect** :ref:`result proxy <morty.sh>` +Check after Installation +------------------------ -.. code:: bash +Once you have done your installation, you can run a SearXNG *check* procedure, +to see if there are some left overs. In this example there exists a *old* +``/etc/searx/settings.yml``:: - $ sudo -H ./utils/morty.sh inspect service + $ sudo -H ./utils/searxng.sh instance check + SearXNG checks + -------------- + ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/ + INFO: [OK] (old) account 'searx' does not exists + INFO: [OK] (old) account 'filtron' does not exists + INFO: [OK] (old) account 'morty' does not exists + ... + INFO searx.shared : Use shared_simple implementation + INFO searx.shared.redis : connected redis DB --> default diff --git a/docs/build-templates/searx.rst b/docs/build-templates/searxng.rst index 626cce6bd..14b385468 100644 --- a/docs/build-templates/searx.rst +++ b/docs/build-templates/searxng.rst @@ -1,4 +1,4 @@ -.. template evaluated by: ./utils/searx.sh docs +.. template evaluated by: ./utils/searxng.sh searxng.doc.rst .. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$ .. START distro-packages @@ -65,7 +65,8 @@ ${fedora_build} $ sudo -H useradd --shell /bin/bash --system \\ --home-dir \"$SERVICE_HOME\" \\ - --comment 'Privacy-respecting metasearch engine' $SERVICE_USER + --comment 'Privacy-respecting metasearch engine' \\ + $SERVICE_USER $ sudo -H mkdir \"$SERVICE_HOME\" $ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\" @@ -81,7 +82,8 @@ ${fedora_build} .. code-block:: sh $ sudo -H -u ${SERVICE_USER} -i - (${SERVICE_USER})$ git clone \"$GIT_URL\" \"$SEARX_SRC\" + (${SERVICE_USER})$ git clone \"$GIT_URL\" \\ + \"$SEARXNG_SRC\" .. END clone searxng @@ -93,8 +95,9 @@ ${fedora_build} .. code-block:: sh - (${SERVICE_USER})$ python3 -m venv \"${SEARX_PYENV}\" - (${SERVICE_USER})$ echo \". ${SEARX_PYENV}/bin/activate\" >> \"$SERVICE_HOME/.profile\" + (${SERVICE_USER})$ python3 -m venv \"${SEARXNG_PYENV}\" + (${SERVICE_USER})$ echo \". ${SEARXNG_PYENV}/bin/activate\" \\ + >> \"$SERVICE_HOME/.profile\" .. END create virtualenv @@ -109,7 +112,7 @@ ${fedora_build} $ sudo -H -u ${SERVICE_USER} -i (${SERVICE_USER})$ command -v python && python --version - $SEARX_PYENV/bin/python + $SEARXNG_PYENV/bin/python Python 3.8.1 # update pip's boilerplate .. @@ -119,7 +122,7 @@ ${fedora_build} pip install -U pyyaml # jump to SearXNG's working tree and install SearXNG into virtualenv - (${SERVICE_USER})$ cd \"$SEARX_SRC\" + (${SERVICE_USER})$ cd \"$SEARXNG_SRC\" (${SERVICE_USER})$ pip install -e . @@ -134,24 +137,15 @@ ${fedora_build} .. code-block:: sh $ sudo -H mkdir -p \"$(dirname ${SEARXNG_SETTINGS_PATH})\" - $ sudo -H cp \"$SEARX_SRC/utils/templates/etc/searxng/settings.yml\" \\ + $ sudo -H cp \"$SEARXNG_SRC/utils/templates/etc/searxng/settings.yml\" \\ \"${SEARXNG_SETTINGS_PATH}\" - .. group-tab:: searx/settings.yml - - .. code-block:: sh - - $ sudo -H mkdir -p \"$(dirname ${SEARXNG_SETTINGS_PATH})\" - $ sudo -H cp \"$SEARX_SRC/searx/settings.yml\" \\ - \"${SEARXNG_SETTINGS_PATH}\" - -.. tabs:: - .. group-tab:: minimal setup .. code-block:: sh - $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \"$SEARXNG_SETTINGS_PATH\" + $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \\ + \"$SEARXNG_SETTINGS_PATH\" .. END searxng config @@ -168,14 +162,14 @@ ${fedora_build} # start webapp $ sudo -H -u ${SERVICE_USER} -i - (${SERVICE_USER})$ cd ${SEARX_SRC} + (${SERVICE_USER})$ cd ${SEARXNG_SRC} (${SERVICE_USER})$ export SEARXNG_SETTINGS_PATH=\"${SEARXNG_SETTINGS_PATH}\" (${SERVICE_USER})$ python searx/webapp.py # disable debug $ sudo -H sed -i -e \"s/debug : True/debug : False/g\" \"$SEARXNG_SETTINGS_PATH\" -Open WEB browser and visit http://$SEARX_INTERNAL_HTTP . If you are inside a +Open WEB browser and visit http://$SEARXNG_INTERNAL_HTTP . If you are inside a container or in a script, test with curl: .. tabs:: @@ -184,13 +178,13 @@ container or in a script, test with curl: .. code-block:: sh - $ xdg-open http://$SEARX_INTERNAL_HTTP + $ xdg-open http://$SEARXNG_INTERNAL_HTTP .. group-tab:: curl .. code-block:: none - $ curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP + $ curl --location --verbose --head --insecure $SEARXNG_INTERNAL_HTTP * Trying 127.0.0.1:8888... * TCP_NODELAY set diff --git a/docs/conf.py b/docs/conf.py index 44d1c2ad9..8e0c3ab1b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,5 +195,5 @@ html_show_sourcelink = True # LaTeX ---------------------------------------------------------------- latex_documents = [ - (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual") + (master_doc, "searxng-{}.tex".format(VERSION_STRING), html_title, author, "manual") ] diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst index 439875491..e950ae667 100644 --- a/docs/dev/engine_overview.rst +++ b/docs/dev/engine_overview.rst @@ -66,11 +66,11 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin .. table:: Common options in the engine setup (``settings.yml``) :width: 100% - ======================= =========== =============================================== + ======================= =========== ================================================== argument type information - ======================= =========== =============================================== + ======================= =========== ================================================== name string name of search-engine - engine string name of searx-engine (filename without ``.py``) + engine string name of searxng-engine (file name without ``.py``) enable_http bool enable HTTP (by default only HTTPS is enabled). shortcut string shortcut of search-engine timeout string specific timeout for search-engine @@ -78,7 +78,7 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin proxies dict set proxies for a specific engine (e.g. ``proxies : {http: socks5://proxy:port, https: socks5://proxy:port}``) - ======================= =========== =============================================== + ======================= =========== ================================================== .. _engine overrides: diff --git a/docs/dev/lxcdev.rst b/docs/dev/lxcdev.rst index 597f99925..6195a575d 100644 --- a/docs/dev/lxcdev.rst +++ b/docs/dev/lxcdev.rst @@ -45,9 +45,7 @@ be set on a *production* system. The scripts from :ref:`searx_utils` can divide in those to install and maintain software: -- :ref:`searx.sh` -- :ref:`filtron.sh` -- :ref:`morty.sh` +- :ref:`searxng.sh` and the script :ref:`lxc.sh`, with we can scale our installation, maintenance or even development tasks over a stack of isolated containers / what we call the: @@ -73,7 +71,7 @@ once: .. group-tab:: desktop - .. code:: sh + .. code:: bash $ snap install lxd $ lxd init --auto @@ -85,28 +83,28 @@ fork: .. group-tab:: desktop - .. code:: sh + .. code:: bash $ cd ~/Downloads $ git clone https://github.com/searxng/searxng.git searxng $ cd searxng -The :ref:`lxc-searx.env` consists of several images, see ``export -LXC_SUITE=(...`` near by :origin:`utils/lxc-searx.env#L19`. For this blog post +The :ref:`lxc-searxng.env` consists of several images, see ``export +LXC_SUITE=(...`` near by :origin:`utils/lxc-searxng.env#L19`. For this blog post we exercise on a archlinux_ image. The container of this image is named -``searx-archlinux``. Lets build the container, but be sure that this container +``searxng-archlinux``. Lets build the container, but be sure that this container does not already exists, so first lets remove possible old one: .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh remove searx-archlinux - $ sudo -H ./utils/lxc.sh build searx-archlinux + $ sudo -H ./utils/lxc.sh remove searxng-archlinux + $ sudo -H ./utils/lxc.sh build searxng-archlinux -.. sidebar:: The ``searx-archlinux`` container +.. sidebar:: The ``searxng-archlinux`` container is the base of all our exercises here. @@ -117,9 +115,9 @@ In this container we install all services :ref:`including searx, morty & filtron .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh install suite searx-archlinux + $ sudo -H ./utils/lxc.sh install suite searxng-archlinux To proxy HTTP from filtron and morty in the container to the outside of the container, install nginx into the container. Once for the bot blocker filtron: @@ -128,9 +126,9 @@ container, install nginx into the container. Once for the bot blocker filtron: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/filtron.sh nginx install ... INFO: got 429 from http://10.174.184.156/searx @@ -141,9 +139,9 @@ and once for the content sanitizer (content proxy morty): .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/morty.sh nginx install ... INFO: got 200 from http://10.174.184.156/morty/ @@ -154,7 +152,7 @@ and once for the content sanitizer (content proxy morty): blocker (filtron) and WEB content sanitizer (content proxy morty), both are needed for a *privacy protecting* search engine. -On your system, the IP of your ``searx-archlinux`` container differs from +On your system, the IP of your ``searxng-archlinux`` container differs from http://10.174.184.156/searx, just open the URL reported in your installation protocol in your WEB browser from the desktop to test the instance from outside of the container. @@ -169,27 +167,27 @@ In containers, work as usual Usually you open a root-bash using ``sudo -H bash``. In case of LXC containers open the root-bash in the container using ``./utils/lxc.sh cmd -searx-archlinux``: +searxng-archlinux``: .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux bash - INFO: [searx-archlinux] bash - [root@searx-archlinux searx]# pwd + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash + INFO: [searxng-archlinux] bash + [root@searxng-archlinux searx]# pwd /share/searxng -The prompt ``[root@searx-archlinux ...]`` signals, that you are the root user in -the searx-container. To debug the running SearXNG instance use: +The prompt ``[root@searxng-archlinux ...]`` signals, that you are the root user in +the searxng-container. To debug the running SearXNG instance use: .. tabs:: - .. group-tab:: root@searx-archlinux + .. group-tab:: root@searxng-archlinux - .. code:: sh + .. code:: bash $ ./utils/searx.sh inspect service ... @@ -202,56 +200,42 @@ above. You can stop monitoring using ``CTRL-C``, this also disables the *"debug option"* in SearXNG's settings file and restarts the SearXNG uwsgi application. To debug services from filtron and morty analogous use: -.. tabs:: - - .. group-tab:: root@searx-archlinux - - .. code:: sh - - $ ./utils/filtron.sh inspect service - $ ./utils/morty.sh inspect service - -Another point we have to notice is that each service (:ref:`SearXNG <searx.sh>`, -:ref:`filtron <filtron.sh>` and :ref:`morty <morty.sh>`) runs under dedicated -system user account with the same name (compare :ref:`create searxng user`). To -get a shell from theses accounts, simply call one of the scripts: +Another point we have to notice is that the service (:ref:`SearXNG <searxng.sh>` +runs under dedicated system user account with the same name (compare +:ref:`create searxng user`). To get a shell from theses accounts, simply call: .. tabs:: - .. group-tab:: root@searx-archlinux + .. group-tab:: root@searxng-archlinux - .. code:: sh + .. code:: bash - $ ./utils/searx.sh shell - $ ./utils/filtron.sh shell - $ ./utils/morty.sh shell + $ ./utils/searxng.sh instance cmd bash -To get in touch, open a shell from the service user (searx@searx-archlinux): +To get in touch, open a shell from the service user (searxng@searxng-archlinux): .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ./utils/searx.sh shell - // exit with [CTRL-D] - (searx-pyenv) [searx@searx-archlinux ~]$ ... + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh instance cmd bash + INFO: [searxng-archlinux] ./utils/searxng.sh instance cmd bash + [searxng@searxng-archlinux ~]$ -The prompt ``[searx@searx-archlinux]`` signals that you are logged in as system -user ``searx`` in the ``searx-archlinux`` container and the python *virtualenv* -``(searx-pyenv)`` environment is activated. +The prompt ``[searxng@searxng-archlinux]`` signals that you are logged in as system +user ``searx`` in the ``searxng-archlinux`` container and the python *virtualenv* +``(searxng-pyenv)`` environment is activated. .. tabs:: - .. group-tab:: searx@searx-archlinux - - .. code:: sh + .. group-tab:: searxng@searxng-archlinux - (searx-pyenv) [searx@searx-archlinux ~]$ pwd - /usr/local/searx + .. code:: bash + (searxng-pyenv) [searxng@searxng-archlinux ~]$ pwd + /usr/local/searxng Wrap production into developer suite @@ -262,23 +246,22 @@ from a LXC container (which is quite ready for production) into a developer suite. For this, we have to keep an eye on the :ref:`installation basic`: - SearXNG setup in: ``/etc/searxng/settings.yml`` -- SearXNG user's home: ``/usr/local/searx`` -- virtualenv in: ``/usr/local/searx/searx-pyenv`` -- SearXNG software in: ``/usr/local/searx/searx-src`` +- SearXNG user's home: ``/usr/local/searxng`` +- virtualenv in: ``/usr/local/searxng/searxng-pyenv`` +- SearXNG software in: ``/usr/local/searxng/searxng-src`` -With the use of the :ref:`searx.sh` the SearXNG service was installed as +With the use of the :ref:`searxng.sh` the SearXNG service was installed as :ref:`uWSGI application <searxng uwsgi>`. To maintain this service, we can use -``systemctl`` (compare :ref:`service architectures on distributions <uwsgi -configuration>`). +``systemctl`` (compare :ref:`uWSGI maintenance`). .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - systemctl stop uwsgi@searx + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + systemctl stop uwsgi@searxng With the command above, we stopped the SearXNG uWSGI-App in the archlinux container. @@ -291,29 +274,29 @@ least you should attend the settings of ``uid``, ``chdir``, ``env`` and env = SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml http = 127.0.0.1:8888 - chdir = /usr/local/searx/searx-src/searx - virtualenv = /usr/local/searx/searx-pyenv - pythonpath = /usr/local/searx/searx-src + chdir = /usr/local/searxng/searxng-src/searx + virtualenv = /usr/local/searxng/searxng-pyenv + pythonpath = /usr/local/searxng/searxng-src If you have read the :ref:`"Good to know section" <lxc.sh>` you remember, that each container shares the root folder of the repository and the command ``utils/lxc.sh cmd`` handles relative path names **transparent**. To wrap the SearXNG installation into a developer one, we simple have to create a smylink to the **transparent** reposetory from the desktop. Now lets replace the -repository at ``searx-src`` in the container with the working tree from outside +repository at ``searxng-src`` in the container with the working tree from outside of the container: .. tabs:: .. group-tab:: container becomes a developer suite - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ln -s /share/searx/ /usr/local/searx/searx-src + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + ln -s /share/searx/ /usr/local/searxng/searxng-src Now we can develop as usual in the working tree of our desktop system. Every time the software was changed, you have to restart the SearXNG service (in the @@ -323,9 +306,9 @@ conatiner): .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ systemctl restart uwsgi@searx @@ -338,30 +321,30 @@ daily usage: To *inspect* the SearXNG instance (already described above): - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/searx.sh inspect service Run :ref:`makefile`, e.g. to test inside the container: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ make test To install all prerequisites needed for a :ref:`buildhosts`: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ./utils/searx.sh install buildhost + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + ./utils/searxng.sh install buildhost To build the docs on a buildhost :ref:`buildhosts`: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ make docs.html .. _lxcdev summary: @@ -371,18 +354,18 @@ Summary We build up a fully functional SearXNG suite in a archlinux container: -.. code:: sh +.. code:: bash - $ sudo -H ./utils/lxc.sh install suite searx-archlinux + $ sudo -H ./utils/lxc.sh install suite searxng-archlinux To access HTTP from the desktop we installed nginx for the services inside the conatiner: .. tabs:: - .. group-tab:: [root@searx-archlinux] + .. group-tab:: [root@searxng-archlinux] - .. code:: sh + .. code:: bash $ ./utils/filtron.sh nginx install $ ./utils/morty.sh nginx install @@ -393,12 +376,12 @@ the container : .. tabs:: - .. group-tab:: [root@searx-archlinux] + .. group-tab:: [root@searxng-archlinux] - .. code:: sh + .. code:: bash - $ mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old - $ ln -s /share/searx/ /usr/local/searx/searx-src + $ mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old + $ ln -s /share/searx/ /usr/local/searxng/searxng-src $ systemctl restart uwsgi@searx To get information about the searxNG suite in the archlinux container we can @@ -408,13 +391,13 @@ use: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh show suite searx-archlinux + $ sudo -H ./utils/lxc.sh show suite searxng-archlinux ... - [searx-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx - [searx-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/ - [searx-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/ - [searx-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65] + [searxng-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx + [searxng-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/ + [searxng-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/ + [searxng-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65] ... diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index ceb76bf10..68c708a85 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -40,7 +40,7 @@ We do no longer need to build up the virtualenv manually. Jump into your git working tree and release a ``make install`` to get a virtualenv with a *developer install* of SearXNG (:origin:`setup.py`). :: - $ cd ~/searx-clone + $ cd ~/searxng-clone $ make install PYENV [virtualenv] installing ./requirements*.txt into local/py3 ... @@ -288,27 +288,3 @@ To filter out HTTP redirects (3xx_):: https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0 https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None -- - - -``make pybuild`` -================ - -.. _PyPi: https://pypi.org/ -.. _twine: https://twine.readthedocs.io/en/latest/ - -Build Python packages in ``./dist/py``:: - - $ make pybuild - ... - BUILD pybuild - running sdist - running egg_info - ... - running bdist_wheel - - $ ls ./dist - searx-0.18.0-py3-none-any.whl searx-0.18.0.tar.gz - -To upload packages to PyPi_, there is also a ``pypi.upload`` target (to test use -``pypi.upload.test``). Since you are not the owner of :pypi:`searx` you will -never need to upload. diff --git a/docs/dev/offline_engines.rst b/docs/dev/offline_engines.rst index 9320c6262..bfb2664f5 100644 --- a/docs/dev/offline_engines.rst +++ b/docs/dev/offline_engines.rst @@ -55,10 +55,10 @@ admins can install packages in advance. If there is a need to install additional packages in *Python's Virtual Environment* of your SearXNG instance you need to switch into the environment -(:ref:`searx-src`) first, for this you can use :ref:`searx.sh`:: +(:ref:`searxng-src`) first, for this you can use :ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... Private engines (Security) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 44401e34f..36a443230 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -33,17 +33,26 @@ Example plugin External plugins ================ -External plugins are standard python modules implementing all the requirements of the standard plugins. -Plugins can be enabled by adding them to :ref:`settings.yml`'s ``plugins`` section. -Example external plugin can be found `here <https://github.com/asciimoo/searx_external_plugin_example>`_. +SearXNG supports *external plugins* / there is no need to install one, SearXNG +runs out of the box. But to demonstrate; in the example below we install the +SearXNG plugins from *The Green Web Foundation* `[ref] +<https://www.thegreenwebfoundation.org/news/searching-the-green-web-with-searx/>`__: -Register your plugin -==================== +.. code:: bash + + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install git+https://github.com/return42/tgwf-searx-plugins + +In the :ref:`settings.yml` activate the ``plugins:`` section and add module +``only_show_green_results`` from ``tgwf-searx-plugins``. + +.. code:: yaml + + plugins: + ... + - only_show_green_results + ... -To enable your plugin register your plugin in -searx > plugin > __init__.py. -And at the bottom of the file add your plugin like. -``plugins.register(name_of_python_file)`` Plugin entry points =================== diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index db52a2d80..921384aab 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -10,7 +10,7 @@ Development Quickstart SearXNG loves developers, just clone and start hacking. All the rest is done for you simply by using :ref:`make <makefile>`. -.. code:: sh +.. code:: bash git clone https://github.com/searxng/searxng.git searxng @@ -27,21 +27,21 @@ to our ":ref:`how to contribute`" guideline. If you implement themes, you will need to setup a :ref:`make node.env` once: -.. code:: sh +.. code:: bash make node.env Before you call *make run* (2.), you need to compile the modified styles and JavaScript: -.. code:: sh +.. code:: bash make themes.all Alternatively you can also compile selective the theme you have modified, e.g. the *simple* theme. -.. code:: sh +.. code:: bash make themes.simple @@ -52,7 +52,7 @@ e.g. the *simple* theme. If you finished your *tests* you can start to commit your changes. To separate the modified source code from the build products first run: -.. code:: sh +.. code:: bash make static.build.restore @@ -60,13 +60,13 @@ This will restore the old build products and only your changes of the code remain in the working tree which can now be added & commited. When all sources are commited, you can commit the build products simply by: -.. code:: sh +.. code:: bash make static.build.commit Commiting the build products should be the last step, just before you send us your PR. There is also a make target to rewind this last build commit: -.. code:: sh +.. code:: bash make static.build.drop diff --git a/docs/utils/filtron.sh.rst b/docs/utils/filtron.sh.rst deleted file mode 100644 index 1f7302392..000000000 --- a/docs/utils/filtron.sh.rst +++ /dev/null @@ -1,80 +0,0 @@ - -.. _filtron.sh: - -==================== -``utils/filtron.sh`` -==================== - -.. sidebar:: further reading - - - :ref:`searxng filtron` - - :ref:`architecture` - - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache - <installation apache>`) - -.. _Go: https://golang.org/ -.. _filtron: https://github.com/searxng/filtron -.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md - -To simplify installation and maintenance of a filtron instance you can use the -script :origin:`utils/filtron.sh`. In most cases you will install filtron_ -simply by running the command: - -.. code:: bash - - sudo -H ./utils/filtron.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_ -into this user account: - -#. Create a separated user account (``filtron``). -#. Download and install Go_ binary in user's $HOME (``~filtron``). -#. Install filtron with the package management from Go_ (``go get -v -u - github.com/searxng/filtron``) -#. Setup a proper rule configuration :origin:`[ref] - <utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``). -#. Setup a systemd service unit :origin:`[ref] - <utils/templates/lib/systemd/system/filtron.service>` - (``/lib/systemd/system/filtron.service``). - - -Create user -=========== - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START create user - :end-before: END create user - - -Install go -========== - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install go - :end-before: END install go - - -Install filtron -=============== - -Install :origin:`rules.json <utils/templates/etc/filtron/rules.json>` at -``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and -install filtron software and systemd unit: - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install filtron - :end-before: END install filtron - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install systemd unit - :end-before: END install systemd unit - -.. _filtron.sh overview: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/filtron.sh --help diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 2077bbb13..2da26ed12 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -1,52 +1,30 @@ .. _searx_utils: .. _toolboxing: -=================== -Admin's tooling box -=================== +================== +DevOps tooling box +================== -In the folder :origin:`utils/` we maintain some tools useful for administrators. +In the folder :origin:`utils/` we maintain some tools useful for administrators +and developers. .. toctree:: :maxdepth: 2 :caption: Contents - searx.sh - filtron.sh - morty.sh + searxng.sh lxc.sh -.. _toolboxing common: +Common command environments +=========================== -Common commands & environment -============================= - -Scripts to maintain services often dispose of common commands and environments. - -``shell`` : command - Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for - troubleshooting. - -``inspect service`` : command - Shows status and log of the service, most often you have a option to enable - more verbose debug logs. Very helpful for debugging, but be careful not to - enable debugging in a production environment! +The scripts in our tooling box often dispose of common environments: ``FORCE_TIMEOUT`` : environment Sets timeout for interactive prompts. If you want to run a script in batch job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a - reverse proxy for filtron on all containers of the :ref:`SearXNG suite - <lxc-searx.env>` use :: - - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install - -.. _toolboxing setup: - -Tooling box setup -================= - -The main setup is done in the :origin:`.config.sh` (read also :ref:`settings -global`). + SearXNG server and nginx proxy on all containers of the :ref:`SearXNG suite + <lxc-searxng.env>` use:: -.. literalinclude:: ../../.config.sh - :language: bash + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install all + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst index f09b033f4..4308a12cc 100644 --- a/docs/utils/lxc.sh.rst +++ b/docs/utils/lxc.sh.rst @@ -23,7 +23,7 @@ With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of containers, what we call the: *lxc suite*. The *SearXNG suite* -(:origin:`lxc-searx.env <utils/lxc-searx.env>`) is loaded by default, every time +(:origin:`lxc-searxng.env <utils/lxc-searxng.env>`) is loaded by default, every time you start the ``lxc.sh`` script (*you do not need to care about*). Before you can start with containers, you need to install and initiate LXD_ @@ -49,7 +49,7 @@ help>`. If you do not want to build all containers, **you can build just one**:: - $ sudo -H ./utils/lxc.sh build searx-ubu1804 + $ sudo -H ./utils/lxc.sh build searxng-archlinux *Good to know ...* @@ -62,9 +62,9 @@ of:: In the containers, you can run what ever you want, e.g. to start a bash use:: - $ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash - INFO: [searx-ubu1804] bash - root@searx-ubu1804:/share/searx# + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash + INFO: [searxng-archlinux] bash + [root@searxng-archlinux SearXNG]# If there comes the time you want to **get rid off all** the containers and **clean up local images** just type:: @@ -121,28 +121,26 @@ Install suite ============= To install the complete :ref:`SearXNG suite (includes searx, morty & filtron) -<lxc-searx.env>` into all LXC_ use:: +<lxc-searxng.env>` into all LXC_ use:: $ sudo -H ./utils/lxc.sh install suite -The command above installs a SearXNG suite (see :ref:`installation scripts`). To -get the IP (URL) of the filtron service in the containers use ``show suite`` +The command above installs a SearXNG suite (see :ref:`installation scripts`). +To :ref:`install a nginx <installation nginx>` reverse proxy (or alternatively +use :ref:`apache <installation apache>`):: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx + +To get the IP (URL) of the SearXNG service in the containers use ``show suite`` command. To test instances from containers just open the URLs in your WEB-Browser:: - $ sudo ./utils/lxc.sh show suite | grep filtron - [searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx - [searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx - [searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx - [searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx - [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx - [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx - -To :ref:`install a nginx <installation nginx>` reverse proxy for filtron and -morty use (or alternatively use :ref:`apache <installation apache>`):: + $ sudo ./utils/lxc.sh show suite | grep SEARXNG_URL - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install + [searxng-ubu2110] SEARXNG_URL : http://n.n.n.147/searxng + [searxng-ubu2004] SEARXNG_URL : http://n.n.n.246/searxng + [searxnggfedora35] SEARXNG_URL : http://n.n.n.140/searxng + [searxng-archlinux] SEARXNG_URL : http://n.n.n.165/searxng Running commands @@ -152,8 +150,8 @@ Running commands :ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the Makefile target ``test`` in the archlinux_ container:: - sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost - sudo -H ./utils/lxc.sh cmd searx-archlinux make test + sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh install buildhost + sudo -H ./utils/lxc.sh cmd searxng-archlinux make test Setup SearXNG buildhost @@ -164,11 +162,11 @@ The installation procedure to set up a :ref:`build host<buildhosts>` takes its time. Installation in all containers will take more time (time for another cup of coffee).:: - sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost + sudo -H ./utils/lxc.sh cmd -- ./utils/searxng.sh install buildhost To build (live) documentation inside a archlinux_ container:: - sudo -H ./utils/lxc.sh cmd searx-archlinux make docs.clean docs.live + sudo -H ./utils/lxc.sh cmd searxng-archlinux make docs.clean docs.live ... [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080 @@ -176,7 +174,7 @@ To get IP of the container and the port number *live docs* is listening:: $ sudo ./utils/lxc.sh show suite | grep docs.live ... - [searx-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/ + [searxng-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/ .. _lxc.sh help: @@ -189,10 +187,10 @@ The ``--help`` output of the script is largely self-explanatory: .. program-output:: ../utils/lxc.sh --help -.. _lxc-searx.env: +.. _lxc-searxng.env: SearXNG suite ============= -.. literalinclude:: ../../utils/lxc-searx.env +.. literalinclude:: ../../utils/lxc-searxng.env :language: bash diff --git a/docs/utils/morty.sh.rst b/docs/utils/morty.sh.rst deleted file mode 100644 index 99103b1cb..000000000 --- a/docs/utils/morty.sh.rst +++ /dev/null @@ -1,80 +0,0 @@ - -.. _morty: https://github.com/asciimoo/morty -.. _morty's README: https://github.com/asciimoo/morty -.. _Go: https://golang.org/ - -.. _morty.sh: - -================== -``utils/morty.sh`` -================== - -.. sidebar:: further reading - - - :ref:`architecture` - - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache - <installation apache>`) - - :ref:`searxng morty` - -To simplify installation and maintenance of a morty_ instance you can use the -script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by -running the command: - -.. code:: bash - - sudo -H ./utils/morty.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_ -into this user account: - -#. Create a separated user account (``morty``). -#. Download and install Go_ binary in user's $HOME (``~morty``). -#. Install morty_ with the package management from Go_ (``go get -v -u - github.com/asciimoo/morty``) -#. Setup a systemd service unit :origin:`[ref] - <utils/templates/lib/systemd/system/morty.service>` - (``/lib/systemd/system/morty.service``). - -.. hint:: - - To add morty to your SearXNG instance read chapter :ref:`searxng morty`. - -Create user -=========== - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START create user - :end-before: END create user - - -Install go -========== - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install go - :end-before: END install go - - -Install morty -============= - -Install morty software and systemd unit: - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install morty - :end-before: END install morty - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install systemd unit - :end-before: END install systemd unit - -.. _morty.sh overview: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/morty.sh --help - diff --git a/docs/utils/searx.sh.rst b/docs/utils/searx.sh.rst deleted file mode 100644 index 6d412d23d..000000000 --- a/docs/utils/searx.sh.rst +++ /dev/null @@ -1,39 +0,0 @@ - -.. _searx.sh: - -================== -``utils/searx.sh`` -================== - -.. sidebar:: further reading - - - :ref:`architecture` - - :ref:`installation` - - :ref:`installation nginx` - - :ref:`installation apache` - -To simplify installation and maintenance of a SearXNG instance you can use the -script :origin:`utils/searx.sh`. - -Install -======= - -In most cases you will install SearXNG simply by running the command: - -.. code:: bash - - sudo -H ./utils/searx.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs SearXNG -into this user account. The installation is described in chapter -:ref:`installation basic`. - -.. _intranet reverse proxy: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/searx.sh --help diff --git a/docs/utils/searxng.sh.rst b/docs/utils/searxng.sh.rst new file mode 100644 index 000000000..f6578f7b1 --- /dev/null +++ b/docs/utils/searxng.sh.rst @@ -0,0 +1,36 @@ + +.. _searxng.sh: + +==================== +``utils/searxng.sh`` +==================== + +.. sidebar:: further reading + + - :ref:`architecture` + - :ref:`installation` + - :ref:`installation nginx` + - :ref:`installation apache` + +To simplify the installation and maintenance of a SearXNG instance you can use the +script :origin:`utils/searxng.sh`. + +Install +======= + +In most cases you will install SearXNG simply by running the command: + +.. code:: bash + + sudo -H ./utils/searx.sh install all + +The installation is described in chapter :ref:`installation basic`. + +.. _searxng.sh overview: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory: + +.. program-output:: ../utils/searxng.sh --help @@ -416,9 +416,7 @@ docs.prebuild() { set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" - ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst" - ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst" - ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst" + ./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" pyenv.cmd searxng_extra/docs_prebuild ) dump_return $? diff --git a/searx/shared/redisdb.py b/searx/shared/redisdb.py index da71d169c..bb7a0eeb4 100644 --- a/searx/shared/redisdb.py +++ b/searx/shared/redisdb.py @@ -19,10 +19,13 @@ A redis DB connect can be tested by:: """ +import os +import pwd import logging import redis from searx import get_setting + logger = logging.getLogger('searx.shared.redis') _client = None @@ -42,6 +45,7 @@ def init(): logger.info("connected redis DB --> %s", c.acl_whoami()) return True except redis.exceptions.ConnectionError as exc: - logger.error("can't connet redis DB ...") + _pw = pwd.getpwuid(os.getuid()) + logger.error("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid) logger.error(" %s", exc) return False diff --git a/utils/filtron.sh b/utils/filtron.sh index 2536214e4..ab207bbd8 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -4,56 +4,19 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/lib_go.sh -source "${REPO_ROOT}/utils/lib_go.sh" -# shellcheck source=utils/lib_install.sh -source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" - -FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ -| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" -[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ - FILTRON_ETC="/etc/filtron" -FILTRON_RULES="$FILTRON_ETC/rules.json" -FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/filtron/rules.json}" - -FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" -FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" - -# The filtron target is the SearXNG installation, listenning on server.port at -# server.bind_address. The default of FILTRON_TARGET is taken from the YAML -# configuration, do not change this value without reinstalling the entire -# SearXNG suite including filtron & morty. -FILTRON_TARGET="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" SERVICE_NAME="filtron" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" -SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" -SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" - -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" -GO_ENV="${SERVICE_HOME}/.go_env" -GO_VERSION="go1.17.2" - -APACHE_FILTRON_SITE="searxng.conf" -NGINX_FILTRON_SITE="searxng.conf" - -# shellcheck disable=SC2034 -CONFIG_FILES=( - "${FILTRON_RULES}" - "${SERVICE_SYSTEMD_UNIT}" -) +APACHE_FILTRON_SITE="searx.conf" +NGINX_FILTRON_SITE="searx.conf" # ---------------------------------------------------------------------------- usage() { @@ -62,248 +25,45 @@ usage() { # shellcheck disable=SC1117 cat <<EOF usage:: - $(basename "$0") shell - $(basename "$0") install [all|user|rules] - $(basename "$0") reinstall all - $(basename "$0") update [filtron] - $(basename "$0") remove [all] - $(basename "$0") activate [service] - $(basename "$0") deactivate [service] - $(basename "$0") inspect [service] - $(basename "$0") option [debug-on|debug-off] - $(basename "$0") apache [install|remove] - $(basename "$0") nginx [install|remove] + $(basename "$0") remove all] + $(basename "$0") apache remove + $(basename "$0") nginx remove -shell - start interactive shell from user ${SERVICE_USER} -install / remove - :all: complete setup of filtron service - :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) - :rules: reinstall filtron rules $FILTRON_RULES -install - :check: check the filtron installation -reinstall: - :all: runs 'install/remove all' -update filtron - Update filtron installation ($SERVICE_HOME) -activate service - activate and start service daemon (systemd unit) -deactivate service - stop and deactivate service daemon (systemd unit) -inspect service - show service status and log -option - set one of the available options -apache (${PUBLIC_URL}) - :install: apache site with a reverse proxy (ProxyPass) - :remove: apache site ${APACHE_FILTRON_SITE} -nginx (${PUBLIC_URL}) - :install: nginx site with a reverse proxy (ProxyPass) - :remove: nginx site ${NGINX_FILTRON_SITE} -filtron rules: ${FILTRON_RULES_TEMPLATE} ----- sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} - FILTRON_URL_PATH : ${FILTRON_URL_PATH} +remove all : drop all components of the filtron service +apache remove : drop apache site ${APACHE_FILTRON_SITE} +nginx remove : drop nginx site ${NGINX_FILTRON_SITE} EOF - install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } main() { - required_commands \ - sudo install git wget curl \ - || exit - local _usage="unknown or missing $1 command $2" case $1 in - --getenv) var="$2"; echo "${!var}"; exit 0;; -h|--help) usage; exit 0;; - - shell) - sudo_or_exit - interactive_shell "${SERVICE_USER}" - ;; - inspect) - case $2 in - service) - sudo_or_exit - inspect_service - ;; - *) usage "$_usage"; exit 42;; - esac ;; - reinstall) - rst_title "re-install $SERVICE_NAME" part - sudo_or_exit - case $2 in - all) - remove_all - install_all - ;; - *) usage "$_usage"; exit 42;; - esac ;; - install) - rst_title "$SERVICE_NAME" part - sudo_or_exit - case $2 in - check) - rst_title "Check filtron installation" part - install_check - ;; - all) install_all ;; - user) assert_user ;; - rules) - install_rules - systemd_restart_service "${SERVICE_NAME}" - ;; - *) usage "$_usage"; exit 42;; - esac ;; - update) - sudo_or_exit - case $2 in - filtron) update_filtron ;; - *) usage "$_usage"; exit 42;; - esac ;; remove) sudo_or_exit case $2 in all) remove_all;; - user) drop_service_account "${SERVICE_USER}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - activate) - sudo_or_exit - case $2 in - service) systemd_activate_service "${SERVICE_NAME}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - deactivate) - sudo_or_exit - case $2 in - service) systemd_deactivate_service "${SERVICE_NAME}" ;; *) usage "$_usage"; exit 42;; esac ;; apache) sudo_or_exit case $2 in - install) install_apache_site ;; remove) remove_apache_site ;; *) usage "$_usage"; exit 42;; esac ;; nginx) sudo_or_exit case $2 in - install) install_nginx_site ;; remove) remove_nginx_site ;; *) usage "$_usage"; exit 42;; esac ;; - option) - sudo_or_exit - case $2 in - debug-on) echo; enable_debug ;; - debug-off) echo; disable_debug ;; - *) usage "$_usage"; exit 42;; - esac ;; - doc) rst-doc ;; *) usage "unknown or missing command $1"; exit 42;; esac } -install_all() { - rst_title "Install $SERVICE_NAME (service)" - assert_user - wait_key - go.golang "${GO_VERSION}" "${SERVICE_USER}" - wait_key - install_filtron - install_rules - wait_key - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - wait_key - echo - if ! service_is_available "http://${FILTRON_LISTEN}" ; then - err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}" - fi - if apache_is_installed; then - info_msg "Apache is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_apache_site - fi - elif nginx_is_installed; then - info_msg "nginx is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_nginx_site - fi - fi - if ask_yn "Do you want to inspect the installation?" Ny; then - inspect_service - fi - -} - -install_check() { - - if service_account_is_available "$SERVICE_USER"; then - info_msg "service account $SERVICE_USER available." - else - err_msg "service account $SERVICE_USER not available!" - fi - if go_is_available "$SERVICE_USER"; then - info_msg "~$SERVICE_USER: go is installed" - else - err_msg "~$SERVICE_USER: go is not installed" - fi - if filtron_is_installed; then - info_msg "~$SERVICE_USER: filtron app is installed" - else - err_msg "~$SERVICE_USER: filtron app is not installed!" - fi - - if ! service_is_available "http://${FILTRON_API}"; then - err_msg "API not available at: http://${FILTRON_API}" - fi - - if ! service_is_available "http://${FILTRON_LISTEN}" ; then - err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}" - fi - - if service_is_available "http://${FILTRON_TARGET}" ; then - info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" - fi - - if ! service_is_available "${PUBLIC_URL}"; then - warn_msg "Public service at ${PUBLIC_URL} is not available!" - if ! in_container; then - warn_msg "Check if public name is correct and routed or use the public IP from above." - fi - fi - - if [[ "${GO_VERSION}" > "$(go_version)" ]]; then - warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - else - info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)" - fi - - if [ -f "${APACHE_SITES_AVAILABLE}/searx.conf" ]; then - warn_msg "old searx.conf apache site exists" - fi - - if [ -f "${NGINX_APPS_AVAILABLE}/searx.conf" ]; then - warn_msg "old searx.conf nginx site exists" - fi - -} - -go_version(){ - go.version "${SERVICE_USER}" -} - remove_all() { rst_title "De-Install $SERVICE_NAME (service)" @@ -321,219 +81,6 @@ installations that were installed with this script." fi } -assert_user() { - rst_title "user $SERVICE_USER" section - echo - tee_stderr 1 <<EOF | bash | prefix_stdout -useradd --shell /bin/bash --system \ - --home-dir "$SERVICE_HOME" \ - --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER -mkdir "$SERVICE_HOME" -chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" -groups $SERVICE_USER -EOF - SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" - export SERVICE_HOME - echo "export SERVICE_HOME=$SERVICE_HOME" - - tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" -touch "$GO_ENV" -grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile -EOF -} - -filtron_is_installed() { - [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] -} - -install_filtron() { - rst_title "Install filtron in user's ~/go-apps" section - echo - go.install github.com/searxng/filtron@latest "${SERVICE_USER}" -} - -update_filtron() { - rst_title "Update filtron" section - echo - go.install github.com/searxng/filtron@latest "${SERVICE_USER}" -} - -install_rules() { - rst_title "Install filtron rules" - echo - if [[ ! -f "${FILTRON_RULES}" ]]; then - info_msg "install rules ${FILTRON_RULES_TEMPLATE}" - info_msg " --> ${FILTRON_RULES}" - mkdir -p "$(dirname "${FILTRON_RULES}")" - cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - return - fi - - if cmp --silent "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"; then - info_msg "${FILTRON_RULES} is up to date with" - info_msg "${FILTRON_RULES_TEMPLATE}" - return - fi - - rst_para "Diff between origin's rules file (+) and current (-):" - echo "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}" - $DIFF_CMD "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}" - - local action - choose_one action "What should happen to the rules file? " \ - "keep configuration unchanged" \ - "use origin rules" \ - "start interactive shell" - case $action in - "keep configuration unchanged") - info_msg "leave rules file unchanged" - ;; - "use origin rules") - backup_file "${FILTRON_RULES}" - info_msg "install origin rules" - cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - ;; - "start interactive shell") - backup_file "${FILTRON_RULES}" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -i - rst_para 'Diff between new rules file (-) and current (+):' - echo - $DIFF_CMD "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}" - wait_key - ;; - esac -} - -inspect_service() { - - rst_title "service status & log" - - cat <<EOF - -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - FILTRON_TARGET : ${FILTRON_TARGET} - FILTRON_API : ${FILTRON_API} - FILTRON_LISTEN : ${FILTRON_LISTEN} - FILTRON_URL_PATH : ${FILTRON_URL_PATH} -EOF - install_log_searx_instance - - install_check - - if in_container; then - lxc_suite_info - else - info_msg "public URL --> ${PUBLIC_URL}" - info_msg "internal URL --> http://${FILTRON_LISTEN}" - fi - - - local _debug_on - if ask_yn "Enable filtron debug mode?"; then - enable_debug - _debug_on=1 - fi - echo - systemctl --no-pager -l status "${SERVICE_NAME}" - echo - - info_msg "public URL --> ${PUBLIC_URL}" - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - while true; do - trap break 2 - journalctl -f -u "${SERVICE_NAME}" - done - - if [[ $_debug_on == 1 ]]; then - disable_debug - fi - return 0 -} - - -enable_debug() { - info_msg "try to enable debug mode ..." - python <<EOF -import sys, json - -debug = { - u'name': u'debug request' - , u'filters': [] - , u'interval': 0 - , u'limit': 0 - , u'actions': [{u'name': u'log'}] -} - -with open('$FILTRON_RULES') as rules: - j = json.load(rules) - -pos = None -for i in range(len(j)): - if j[i].get('name') == 'debug request': - pos = i - break -if pos is not None: - j[pos] = debug -else: - j.append(debug) -with open('$FILTRON_RULES', 'w') as rules: - json.dump(j, rules, indent=2, sort_keys=True) - -EOF - systemctl restart "${SERVICE_NAME}.service" -} - -disable_debug() { - info_msg "try to disable debug mode ..." - python <<EOF -import sys, json -with open('$FILTRON_RULES') as rules: - j = json.load(rules) - -pos = None -for i in range(len(j)): - if j[i].get('name') == 'debug request': - pos = i - break -if pos is not None: - del j[pos] - with open('$FILTRON_RULES', 'w') as rules: - json.dump(j, rules, indent=2, sort_keys=True) -EOF - systemctl restart "${SERVICE_NAME}.service" -} - -install_apache_site() { - - rst_title "Install Apache site $APACHE_FILTRON_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})" - - ! apache_is_installed && info_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_apache - fi - - "${REPO_ROOT}/utils/searx.sh" install uwsgi - - apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL}"; then - err_msg "Public service at ${PUBLIC_URL} is not available!" - fi -} - remove_apache_site() { rst_title "Remove Apache site $APACHE_FILTRON_SITE" @@ -551,35 +98,6 @@ This removes apache site ${APACHE_FILTRON_SITE}." } -install_nginx_site() { - - rst_title "Install nginx site $NGINX_FILTRON_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})" - - ! nginx_is_installed && info_msg "nginx is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_nginx - fi - - "${REPO_ROOT}/utils/searx.sh" install uwsgi - - # shellcheck disable=SC2034 - SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) - # shellcheck disable=SC2034 - SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH) - nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL}"; then - err_msg "Public service at ${PUBLIC_URL} is not available!" - fi -} - remove_nginx_site() { rst_title "Remove nginx site $NGINX_FILTRON_SITE" @@ -593,35 +111,8 @@ This removes nginx site ${NGINX_FILTRON_SITE}." return fi - nginx_remove_site "$FILTRON_FILTRON_SITE" - -} - - -rst-doc() { - - eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\"" - - echo -e "\n.. START install systemd unit" - cat <<EOF -.. tabs:: - - .. group-tab:: systemd - - .. code:: bash - -EOF - eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " - echo -e "\n.. END install systemd unit" + nginx_remove_app "$FILTRON_FILTRON_SITE" - # for DIST_NAME in ubuntu-20.04 arch fedora centos; do - # ( - # DIST_ID=${DIST_NAME%-*} - # DIST_VERS=${DIST_NAME#*-} - # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= - # # ... - # ) - # done } # ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh index 257c15024..32db47524 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -195,7 +195,7 @@ wait_key(){ [[ -n $_t ]] && _t="-t $_t" printf "$msg" # shellcheck disable=SC2086 - read -r -s -n1 $_t + read -r -s -n1 $_t || true echo clean_stdin } @@ -1027,7 +1027,7 @@ nginx_include_apps_enabled() { local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;" local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;" - info_msg "checking existence: '${include_directive}' in file ${server_conf}" + info_msg "checking existence: '${include_directive}' in file ${server_conf}" if grep "${include_directive_re}" "${server_conf}"; then info_msg "OK, already exists." return @@ -1117,7 +1117,7 @@ apache_distro_setup() { APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" APACHE_MODULES="modules" - APACHE_PACKAGES="httpd" + APACHE_PACKAGES="httpd mod_ssl" ;; *) err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented" @@ -1249,8 +1249,6 @@ apache_dissable_site() { # ----- uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" -uWSGI_USER= -uWSGI_GROUP= # How distros manage uWSGI apps is very different. From uWSGI POV read: # - https://uwsgi-docs.readthedocs.io/en/latest/Management.html @@ -1276,13 +1274,14 @@ uWSGI_distro_setup() { ;; fedora-*|centos-7) # systemd --> /usr/lib/systemd/system/uwsgi.service - # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see - # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # There are some quirks abbout additional POSIX groups in uWSGI + # 2.0.x, read at least: https://github.com/unbit/uwsgi/issues/2099 uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d" uWSGI_PACKAGES="uwsgi" - uWSGI_USER="uwsgi" - uWSGI_GROUP="uwsgi" ;; *) err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" @@ -1344,30 +1343,6 @@ uWSGI_restart() { esac } -uWSGI_prepare_app() { - - # usage: uWSGI_prepare_app <myapp.ini> - - [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>" - - local APP="${1%.*}" - - case $DIST_ID-$DIST_VERS in - fedora-*|centos-7) - # in emperor mode, the uwsgi user is the owner of the sockets - info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}" - ;; - *) - info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}" - ;; - esac -} - - uWSGI_app_available() { # usage: uWSGI_app_available <myapp.ini> local CONF="$1" @@ -1378,7 +1353,7 @@ uWSGI_app_available() { uWSGI_install_app() { - # usage: uWSGI_install_app [<template option> ...] <myapp.ini> + # usage: uWSGI_install_app [<template option> ...] <myapp.ini> [{owner} [{group} [{chmod}]]] # # <template option>: see install_template @@ -1390,11 +1365,10 @@ uWSGI_install_app() { *) pos_args+=("$i");; esac done - uWSGI_prepare_app "${pos_args[1]}" mkdir -p "${uWSGI_APPS_AVAILABLE}" install_template "${template_opts[@]}" \ "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \ - root root 644 + "${pos_args[2]:-root}" "${pos_args[3]:-root}" "${pos_args[4]:-644}" uWSGI_enable_app "${pos_args[1]}" uWSGI_restart "${pos_args[1]}" info_msg "uWSGI app: ${pos_args[1]} is installed" @@ -1468,7 +1442,6 @@ uWSGI_enable_app() { mkdir -p "${uWSGI_APPS_ENABLED}" rm -f "${uWSGI_APPS_ENABLED}/${CONF}" ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" - chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}" info_msg "enabled uWSGI app: ${CONF}" ;; *) diff --git a/utils/lib_install.sh b/utils/lib_install.sh deleted file mode 100755 index 5d84c066f..000000000 --- a/utils/lib_install.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-or-later - -# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285 -# shellcheck source=utils/lib.sh -. /dev/null - -# Initialize installation procedures: -# -# - Modified source_dot_config function that -# - loads .config.sh from an existing installation (at SEARX_SRC). -# - initialize **SEARX_SRC_INIT_FILES** -# - functions like: -# - install_log_searx_instance() -# - install_searx_get_state() -# -# usage: -# source lib_install.sh -# -# **Installation scripts** -# -# The utils/lib_install.sh is sourced by the installations scripts: -# -# - utils/searx.sh -# - utils/morty.sh -# - utils/filtron.sh -# -# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function -# loads this configuration (instead of './.config.sh'). - -# **SEARX_SRC_INIT_FILES** -# -# Array of file names to sync into a installation at $SEARX_SRC. The file names -# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES(). -# Most often theses are files like: -# - .config.sh -# - searx/settings.yml -# - utils/brand.env -# - ... - - -SEARX_SRC_INIT_FILES=() - -eval orig_"$(declare -f source_dot_config)" - -source_dot_config() { - - # Modified source_dot_config function that - # - loads .config.sh from an existing installation (at SEARX_SRC). - # - initialize SEARX_SRC_INIT_FILES - - if [ -z "$eval_SEARX_SRC" ]; then - export eval_SEARX_SRC='true' - SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) - SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV) - SEARXNG_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_SETTINGS_PATH) - if [ ! -r "${SEARX_SRC}" ]; then - info_msg "not yet cloned: ${SEARX_SRC}" - orig_source_dot_config - return 0 - fi - info_msg "using instance at: ${SEARX_SRC}" - - # set and log DOT_CONFIG - if [ -r "${SEARX_SRC}/.config.sh" ]; then - info_msg "switching to ${SEARX_SRC}/.config.sh" - DOT_CONFIG="${SEARX_SRC}/.config.sh" - else - info_msg "using local config: ${DOT_CONFIG}" - fi - init_SEARX_SRC_INIT_FILES - fi -} - -init_SEARX_SRC_INIT_FILES(){ - # init environment SEARX_SRC_INIT_FILES - - # Monitor modified files in the working-tree from the local repository, only - # if the local file differs to the corresponding file in the instance. Most - # often theses are files like: - # - # - .config.sh - # - searx/settings.yml - # - utils/brand.env - # - ... - - # keep list empty if there is no installation - SEARX_SRC_INIT_FILES=() - if [ ! -r "$SEARX_SRC" ]; then - return 0 - fi - - local fname - local msg="" - local _prefix="" - if [[ -n ${SUDO_USER} ]]; then - _prefix="sudo -u ${SUDO_USER}" - fi - - # Monitor local modified files from the repository, only if the local file - # differs to the corresponding file in the instance - - while IFS= read -r fname; do - if [ -z "$fname" ]; then - continue - fi - if [ -r "${SEARX_SRC}/${fname}" ]; then - # diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" - if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then - SEARX_SRC_INIT_FILES+=("${fname}") - info_msg "local clone (workingtree), modified file: ./$fname" - msg="to update use: sudo -H ./utils/searx.sh install init-src" - fi - fi - done <<< "$($_prefix git diff --name-only)" - [ -n "$msg" ] && info_msg "$msg" -} - -install_log_searx_instance() { - - echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}" - echo -e " SEARXNG_SETTINGS_PATH : ${_BBlue}${SEARXNG_SETTINGS_PATH}${_creset}" - echo -e " SEARX_PYENV : ${_BBlue}${SEARX_PYENV}${_creset}" - echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}" - echo -e " SEARXNG_URL : ${_BBlue}${SEARXNG_URL:-none}${_creset}" - - if in_container; then - # SearXNG is listening on 127.0.0.1 and not available from outside container - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - echo -e "---- container setup" - echo -e " ${_BBlack}HINT:${_creset} SearXNG only listen on loopback device" \ - "${_BBlack}inside${_creset} the container." - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container (IPv6): [${ip#*|}]" - else - # IPv4: - echo " container (IPv4): ${ip#*|}" - fi - done - fi -} - -install_searx_get_state(){ - - # usage: install_searx_get_state - # - # Prompts a string indicating the status of the installation procedure - # - # missing-searx-clone: - # There is no clone at ${SEARX_SRC} - # missing-searx-pyenv: - # There is no pyenv in ${SEARX_PYENV} - # installer-modified: - # There are files modified locally in the installer (clone), - # see ${SEARX_SRC_INIT_FILES} description. - # python-installed: - # Scripts can be executed in instance's environment - # - user: ${SERVICE_USER} - # - pyenv: ${SEARX_PYENV} - - if [ -f /etc/searx/settings.yml ]; then - err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/" - fi - - if ! [ -r "${SEARX_SRC}" ]; then - echo "missing-searx-clone" - return - fi - if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then - echo "missing-searx-pyenv" - return - fi - if ! [ -r "${SEARXNG_SETTINGS_PATH}" ]; then - echo "missing-settings" - return - fi - if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then - echo "installer-modified" - return - fi - echo "python-installed" -} - -# Initialization of the installation procedure -# -------------------------------------------- - -# shellcheck source=utils/brand.env -source "${REPO_ROOT}/utils/brand.env" - -# SEARXNG_URL aka PUBLIC_URL: the public URL of the instance (e.g. -# "https://example.org/searx"). The value is taken from environment $SEARXNG_URL -# in ./utils/brand.env. This variable is a empty string if server.base_url in -# the settings.yml is set to 'false'. - -SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)}" -if in_container; then - # hint: Linux containers do not have DNS entries, lets use IPs - SEARXNG_URL="http://$(primary_ip)" -fi -PUBLIC_URL="${SEARXNG_URL}" - -source_dot_config - -# shellcheck source=utils/lxc-searx.env -source "${REPO_ROOT}/utils/lxc-searx.env" -in_container && lxc_set_suite_env diff --git a/utils/lib_redis.sh b/utils/lib_redis.sh index 5eaa1770f..ba1435a86 100755 --- a/utils/lib_redis.sh +++ b/utils/lib_redis.sh @@ -42,6 +42,8 @@ REDIS_GIT_URL="https://github.com/redis/redis.git" REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}" REDIS_USER="searxng-redis" +REDIS_GROUP="searxng-redis" + REDIS_HOME="/usr/local/${REDIS_USER}" REDIS_HOME_BIN="${REDIS_HOME}/.local/bin" REDIS_ENV="${REDIS_HOME}/.redis_env" @@ -113,7 +115,7 @@ redis.devpkg() { case ${DIST_ID} in ubuntu|debian) - pkg_install git build-essential + pkg_install git build-essential gawk ;; arch) pkg_install git base-devel @@ -139,15 +141,20 @@ redis.build() { rst_title "get redis sources" section redis.src "${CACHE}/redis" - if ! required_commands gcc nm make gawk; then - sudo -H "$0" redis.devpkg + if ! required_commands gcc nm make gawk ; then + info_msg "install development tools to get missing command(s) .." + if [[ -n ${SUDO_USER} ]]; then + sudo -H "$0" redis.devpkg + else + redis.devpkg + fi fi rst_title "compile redis sources" section pushd "${CACHE}/redis" &>/dev/null - if ask_yn "Do you run 'make distclean' first'?" Ny; then + if ask_yn "Do you run 'make distclean' first'?" Yn; then $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout fi @@ -158,7 +165,7 @@ redis.build() { popd &>/dev/null - tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout + tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout mkdir -p "$(redis._get_dist)" cd "${CACHE}/redis/src" cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)" @@ -233,7 +240,7 @@ useradd --shell /bin/bash --system \ --home-dir "${REDIS_HOME}" \ --comment 'user that runs a redis instance' "${REDIS_USER}" mkdir -p "${REDIS_HOME}" -chown -R "${REDIS_USER}:${REDIS_USER}" "${REDIS_HOME}" +chown -R "${REDIS_USER}:${REDIS_GROUP}" "${REDIS_HOME}" groups "${REDIS_USER}" EOF @@ -248,7 +255,7 @@ EOF redis.userdel() { sudo_or_exit drop_service_account "${REDIS_USER}" - groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true + groupdel "${REDIS_GROUP}" 2>&1 | prefix_stdout || true } redis.addgrp() { @@ -256,7 +263,7 @@ redis.addgrp() { # usage: redis.addgrp <user> [[ -z $1 ]] && die_caller 42 "missing argument <user>" - sudo -H gpasswd -a "$1" "${REDIS_USER}" + sudo -H gpasswd -a "$1" "${REDIS_GROUP}" } redis.rmgrp() { @@ -264,7 +271,7 @@ redis.rmgrp() { # usage: redis.rmgrp <user> [[ -z $1 ]] && die_caller 42 "missing argument <user>" - sudo -H gpasswd -d "$1" "${REDIS_USER}" + sudo -H gpasswd -d "$1" "${REDIS_GROUP}" } @@ -278,7 +285,7 @@ redis._install_bin() { ( set -e for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do - install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \ + install -v -o "${REDIS_USER}" -g "${REDIS_GROUP}" \ "${src}/${redis_exe}" "${REDIS_HOME_BIN}" done diff --git a/utils/lxc-searx.env b/utils/lxc-searxng.env index 13b15522c..86279d4b3 100644 --- a/utils/lxc-searx.env +++ b/utils/lxc-searxng.env @@ -4,24 +4,18 @@ # This file is a setup of a LXC suite. It is sourced from different context, do # not manipulate the environment directly, implement functions and manipulate -# environment only is subshells! +# environment only in subshells. -# ---------------------------------------------------------------------------- -# config -# ---------------------------------------------------------------------------- - -# shellcheck disable=SC2034 -LXC_SUITE_NAME="searx" lxc_set_suite_env() { + + export LXC_SUITE_NAME="searxng" + # name of https://images.linuxcontainers.org export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}" export LXC_SUITE=( - # to disable containers, comment out lines .. - # end of standard support see https://wiki.ubuntu.com/Releases - "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023 "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # April 2025 "$LINUXCONTAINERS_ORG_NAME:ubuntu/21.10" "ubu2110" # July 2027 @@ -30,49 +24,27 @@ lxc_set_suite_env() { # rolling releases see https://www.archlinux.org/releng/releases/ "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" - - # EOL 30 June 2024 - "$LINUXCONTAINERS_ORG_NAME:centos/7" "centos7" ) - - PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" - if in_container; then - # container hostnames do not have a DNS entry: use primary IP! - PUBLIC_URL="http://$(primary_ip)/searx" - - # make GUEST's services public to the HOST - FILTRON_API="0.0.0.0:4005" - FILTRON_LISTEN="0.0.0.0:4004" - MORTY_LISTEN="0.0.0.0:3000" - - # export LXC specific environment - export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN - fi } lxc_suite_install_info() { ( lxc_set_suite_env cat <<EOF -LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL} - suite includes searx, morty & filtron -suite images: -$(echo " ${LOCAL_IMAGES[*]}" | $FMT) -suite containers: -$(echo " ${CONTAINERS[*]}" | $FMT) +LXC suite: ${LXC_SUITE_NAME} + Suite includes installation of SearXNG + images: ${LOCAL_IMAGES[*]} + containers: ${CONTAINERS[*]} EOF ) - } +} lxc_suite_install() { ( lxc_set_suite_env FORCE_TIMEOUT=0 export FORCE_TIMEOUT - "${LXC_REPO_ROOT}/utils/searx.sh" install all - "${LXC_REPO_ROOT}/utils/morty.sh" install all - "${LXC_REPO_ROOT}/utils/filtron.sh" install all - + "${LXC_REPO_ROOT}/utils/searxng.sh" install all rst_title "suite installation finished ($(hostname))" part lxc_suite_info echo @@ -88,10 +60,9 @@ lxc_suite_info() { else # IPv4: # shellcheck disable=SC2034,SC2031 - info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL" - info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY" info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/" fi done + "${LXC_REPO_ROOT}/utils/searxng.sh" searxng.instance.env ) } diff --git a/utils/lxc.sh b/utils/lxc.sh index 9754b5d75..418aea938 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -4,12 +4,11 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -source_dot_config # shellcheck source=utils/brand.env source "${REPO_ROOT}/utils/brand.env" # load environment of the LXC suite -LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}" +LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searxng.env}" source "$LXC_ENV" lxc_set_suite_env diff --git a/utils/morty.sh b/utils/morty.sh index c4e7bdf52..a85b04bb8 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -3,10 +3,6 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/lib_go.sh -source "${REPO_ROOT}/utils/lib_go.sh" -# shellcheck source=utils/lib_install.sh -source "${REPO_ROOT}/utils/lib_install.sh" # ---------------------------------------------------------------------------- # config @@ -16,24 +12,9 @@ MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}" PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}" -# shellcheck disable=SC2034 -MORTY_TIMEOUT=5 - SERVICE_NAME="morty" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" -SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" -SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" -# shellcheck disable=SC2034 -SERVICE_ENV_DEBUG=false - -GO_ENV="${SERVICE_HOME}/.go_env" -GO_VERSION="go1.17.2" - -# shellcheck disable=SC2034 -CONFIG_FILES=() # Apache Settings @@ -47,267 +28,45 @@ usage() { # shellcheck disable=SC1117 cat <<EOF usage:: - $(basename "$0") shell - $(basename "$0") install [all|check|user] - $(basename "$0") reinstall all - $(basename "$0") update [morty] - $(basename "$0") remove [all] - $(basename "$0") activate [service] - $(basename "$0") deactivate [service] - $(basename "$0") inspect [service] - $(basename "$0") option [debug-on|debug-off|new-key] - $(basename "$0") apache [install|remove] - $(basename "$0") nginx [install|remove] - $(basename "$0") info [searx] + $(basename "$0") remove all + $(basename "$0") apache remove + $(basename "$0") nginx remove -shell - start interactive shell from user ${SERVICE_USER} -install / remove - :all: complete setup of morty service - :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) -install - :check: check the morty installation -reinstall: - :all: runs 'install/remove all' -update morty - Update morty installation ($SERVICE_HOME) -activate service - activate and start service daemon (systemd unit) -deactivate service - stop and deactivate service daemon (systemd unit) -inspect service - show service status and log -option - set one of the available options - :new-key: set new morty key -apache : ${PUBLIC_URL_MORTY} - :install: apache site with a reverse proxy (ProxyPass) - :remove: apache site ${APACHE_MORTY_SITE} -nginx (${PUBLIC_URL_MORTY}) - :install: nginx site with a reverse proxy (ProxyPass) - :remove: nginx site ${NGINX_MORTY_SITE} ----- -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} - MORTY_LISTEN: : ${MORTY_LISTEN} +remove all : drop all components of the morty service +apache remove : drop apache site ${APACHE_MORTY_SITE} +nginx remove : drop nginx site ${NGINX_MORTY_SITE} EOF - install_log_searx_instance - if in_container; then - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) - for ip in $(global_IPs) ; do - if [[ $ip =~ .*:.* ]]; then - echo " container URL (IPv6): http://[${ip#*|}]:3000/" - else - # IPv4: - echo " container URL (IPv4): http://${ip#*|}:3000/" - fi - done - fi - echo - info_searx - [[ -n ${1} ]] && err_msg "$1" } -info_searx() { - # shellcheck disable=SC1117 - cat <<EOF -To activate result and image proxy in SearXNG read: - https://docs.searxng.org/admin/morty.html -Check settings in file ${SEARXNG_SETTINGS_PATH} ... - result_proxy: - url : ${PUBLIC_URL_MORTY} - server: - image_proxy : True -EOF -} - main() { - required_commands \ - sudo install git wget curl \ - || exit - local _usage="ERROR: unknown or missing $1 command $2" case $1 in - --getenv) var="$2"; echo "${!var}"; exit 0;; -h|--help) usage; exit 0;; - - shell) - sudo_or_exit - interactive_shell "${SERVICE_USER}" - ;; - inspect) - case $2 in - service) - sudo_or_exit - inspect_service - ;; - *) usage "$_usage"; exit 42;; - esac ;; - reinstall) - rst_title "re-install $SERVICE_NAME" part - sudo_or_exit - case $2 in - all) - remove_all - install_all - ;; - *) usage "$_usage"; exit 42;; - esac ;; - install) - rst_title "$SERVICE_NAME" part - sudo_or_exit - case $2 in - all) install_all ;; - check) - rst_title "Check morty installation" part - install_check - ;; - user) assert_user ;; - *) usage "$_usage"; exit 42;; - esac ;; - update) - sudo_or_exit - case $2 in - morty) update_morty ;; - *) usage "$_usage"; exit 42;; - esac ;; remove) sudo_or_exit case $2 in all) remove_all;; - user) drop_service_account "${SERVICE_USER}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - activate) - sudo_or_exit - case $2 in - service) systemd_activate_service "${SERVICE_NAME}" ;; - *) usage "$_usage"; exit 42;; - esac ;; - deactivate) - sudo_or_exit - case $2 in - service) systemd_deactivate_service "${SERVICE_NAME}" ;; *) usage "$_usage"; exit 42;; esac ;; apache) sudo_or_exit case $2 in - install) install_apache_site ;; remove) remove_apache_site ;; *) usage "$_usage"; exit 42;; esac ;; nginx) sudo_or_exit case $2 in - install) install_nginx_site ;; remove) remove_nginx_site ;; *) usage "$_usage"; exit 42;; esac ;; - info) - case $2 in - searx) info_searx ;; - *) usage "$_usage"; exit 42;; - esac ;; - option) - sudo_or_exit - case $2 in - new-key) set_new_key ;; - debug-on) enable_debug ;; - debug-off) disable_debug ;; - *) usage "$_usage"; exit 42;; - esac ;; - doc) rst-doc ;; *) usage "ERROR: unknown or missing command $1"; exit 42;; esac } -install_all() { - - MORTY_KEY="$(head -c 32 /dev/urandom | base64)" - - rst_title "Install $SERVICE_NAME (service)" - assert_user - wait_key - go.golang "${GO_VERSION}" "${SERVICE_USER}" - wait_key - install_morty - wait_key - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - wait_key - if ! service_is_available "http://${MORTY_LISTEN}" ; then - err_msg "Morty is not listening on: http://${MORTY_LISTEN}" - fi - if apache_is_installed; then - info_msg "Apache is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_apache_site - fi - elif nginx_is_installed; then - info_msg "nginx is installed on this host." - if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then - install_nginx_site - fi - fi - info_searx - if ask_yn "Add image and result proxy to SearXNG settings.yml?" Yn; then - "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" - "${REPO_ROOT}/utils/searx.sh" option image-proxy-on - fi - - if ask_yn "Do you want to inspect the installation?" Ny; then - inspect_service - fi - -} - -install_check() { - - if service_account_is_available "$SERVICE_USER"; then - info_msg "service account $SERVICE_USER available." - else - err_msg "service account $SERVICE_USER not available!" - fi - if go_is_available "$SERVICE_USER"; then - info_msg "~$SERVICE_USER: go is installed" - else - err_msg "~$SERVICE_USER: go is not installed" - fi - if morty_is_installed; then - info_msg "~$SERVICE_USER: morty app is installed" - else - err_msg "~$SERVICE_USER: morty app is not installed!" - fi - - if ! service_is_available "http://${MORTY_LISTEN}" ; then - err_msg "Morty is not listening on: http://${MORTY_LISTEN}" - echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." - wait_key - fi - - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - if ! in_container; then - warn_msg "Check if public name is correct and routed or use the public IP from above." - fi - fi - - if [[ "${GO_VERSION}" > "$(go_version)" ]]; then - warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - else - info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)" - fi -} - -go_version(){ - go.version "${SERVICE_USER}" -} remove_all() { rst_title "De-Install $SERVICE_NAME (service)" @@ -321,152 +80,6 @@ installations that were installed with this script." fi } -assert_user() { - rst_title "user $SERVICE_USER" section - echo - tee_stderr 1 <<EOF | bash | prefix_stdout -useradd --shell /bin/bash --system \ - --home-dir "$SERVICE_HOME" \ - --comment 'Web content sanitizer proxy' $SERVICE_USER -mkdir "$SERVICE_HOME" -chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" -groups $SERVICE_USER -EOF - SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" - export SERVICE_HOME - echo "export SERVICE_HOME=$SERVICE_HOME" - - tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" -touch $GO_ENV -grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile -EOF -} - -morty_is_installed() { - [[ -f $SERVICE_HOME/go-apps/bin/morty ]] -} - -install_morty() { - rst_title "Install morty in user's ~/go-apps" section - echo - go.install github.com/asciimoo/morty@latest "${SERVICE_USER}" -} - -update_morty() { - rst_title "Update morty" section - echo - go.install github.com/asciimoo/morty@latest "${SERVICE_USER}" -} - -set_service_env_debug() { - - # usage: set_service_env_debug [false|true] - - # shellcheck disable=SC2034 - local SERVICE_ENV_DEBUG="${1:-false}" - if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - fi -} - -inspect_service() { - - rst_title "service status & log" - - cat <<EOF - -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} - PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY} - MORTY_LISTEN: : ${MORTY_LISTEN} - -EOF - install_log_searx_instance - - install_check - - if in_container; then - lxc_suite_info - else - info_msg "public URL --> ${PUBLIC_URL_MORTY}" - info_msg "morty URL --> http://${MORTY_LISTEN}" - fi - - local _debug_on - if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then - enable_debug - _debug_on=1 - else - systemctl --no-pager -l status "${SERVICE_NAME}" - fi - echo - - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - while true; do - trap break 2 - journalctl -f -u "${SERVICE_NAME}" - done - - if [[ $_debug_on == 1 ]]; then - FORCE_SELECTION=Y disable_debug - fi - return 0 -} - -enable_debug() { - warn_msg "Do not enable debug in production environments!!" - info_msg "Enabling debug option needs to reinstall systemd service!" - set_service_env_debug true -} - -disable_debug() { - info_msg "Disabling debug option needs to reinstall systemd service!" - set_service_env_debug false -} - - -set_new_key() { - rst_title "Set morty key" - echo - - MORTY_KEY="$(head -c 32 /dev/urandom | base64)" - info_msg "morty key: '${MORTY_KEY}'" - - warn_msg "this will need to reinstall services .." - MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key - - systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" - "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" - "${REPO_ROOT}/utils/searx.sh" option image-proxy-on -} - - -install_apache_site() { - - rst_title "Install Apache site $APACHE_MORTY_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_apache - fi - - apache_install_site "${APACHE_MORTY_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - fi -} remove_apache_site() { @@ -484,35 +97,6 @@ This removes apache site ${APACHE_MORTY_SITE}." apache_remove_site "$APACHE_MORTY_SITE" } -install_nginx_site() { - - rst_title "Install nginx site $NGINX_MORTY_SITE" - - rst_para "\ -This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})" - - ! nginx_is_installed && err_msg "nginx is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_nginx - fi - - "${REPO_ROOT}/utils/searx.sh" install uwsgi - - # shellcheck disable=SC2034 - SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) - # shellcheck disable=SC2034 - SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH) - nginx_install_app "${NGINX_MORTY_SITE}" - - info_msg "testing public url .." - if ! service_is_available "${PUBLIC_URL_MORTY}"; then - err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" - fi -} - remove_nginx_site() { rst_title "Remove nginx site $NGINX_MORTY_SITE" @@ -526,37 +110,10 @@ This removes nginx site ${NGINX_MORTY_SITE}." return fi - nginx_remove_site "$NGINX_MORTY_SITE" - -} - -rst-doc() { - - eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\"" - - echo -e "\n.. START install systemd unit" - cat <<EOF -.. tabs:: - - .. group-tab:: systemd + nginx_remove_app "$NGINX_MORTY_SITE" - .. code:: bash - -EOF - eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " - echo -e "\n.. END install systemd unit" - - # for DIST_NAME in ubuntu-20.04 arch fedora centos; do - # ( - # DIST_ID=${DIST_NAME%-*} - # DIST_VERS=${DIST_NAME#*-} - # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= - # # ... - # ) - # done } - # ---------------------------------------------------------------------------- main "$@" # ---------------------------------------------------------------------------- diff --git a/utils/searx.sh b/utils/searx.sh index ce118ed77..2f5e98c8e 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -5,130 +5,14 @@ # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -# shellcheck source=utils/lib_install.sh -source "${REPO_ROOT}/utils/lib_install.sh" - # ---------------------------------------------------------------------------- # config # ---------------------------------------------------------------------------- -SEARX_INTERNAL_HTTP="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" - -SEARXNG_URL_PATH="${SEARXNG_URL_PATH:-$(echo "${PUBLIC_URL}" \ -| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" -[[ "${SEARXNG_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARXNG_URL_PATH=/ - SERVICE_NAME="searx" SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" -SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" -SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" -# shellcheck disable=SC2034 -SERVICE_GROUP="${SERVICE_USER}" - -GIT_BRANCH="${GIT_BRANCH:-master}" -SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" -SEARX_SRC="${SERVICE_HOME}/searx-src" -SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml" -SEARXNG_UWSGI_APP="searxng.ini" -# shellcheck disable=SC2034 -SEARX_UWSGI_SOCKET="/run/uwsgi/app/searxng/socket" - -# apt packages -SEARX_PACKAGES_debian="\ -python3-dev python3-babel python3-venv -uwsgi uwsgi-plugin-python3 -git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev -shellcheck" - -BUILD_PACKAGES_debian="\ -firefox graphviz imagemagick texlive-xetex librsvg2-bin -texlive-latex-recommended texlive-extra-utils fonts-dejavu -latexmk" - -# pacman packages -SEARX_PACKAGES_arch="\ -python python-pip python-lxml python-babel -uwsgi uwsgi-plugin-python -git base-devel libxml2 -shellcheck" - -BUILD_PACKAGES_arch="\ -firefox graphviz imagemagick texlive-bin extra/librsvg -texlive-core texlive-latexextra ttf-dejavu" - -# dnf packages -SEARX_PACKAGES_fedora="\ -python python-pip python-lxml python-babel python3-devel -uwsgi uwsgi-plugin-python3 -git @development-tools libxml2 openssl -ShellCheck" - -BUILD_PACKAGES_fedora="\ -firefox graphviz graphviz-gd ImageMagick librsvg2-tools -texlive-xetex-bin texlive-collection-fontsrecommended -texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts -dejavu-sans-mono-fonts" - -# yum packages -# -# hint: We do no longer support yum packages, it is to complex to maintain -# automate installation of packages like npm. In the firts step we ignore -# CentOS-7 as developer & build platform (the inital patch which brought -# CentOS-7 supports was not intended to be a developer platform). - -SEARX_PACKAGES_centos="\ -python36 python36-pip python36-lxml python-babel -uwsgi uwsgi-plugin-python3 -git @development-tools libxml2 -ShellCheck" - -BUILD_PACKAGES_centos="\ -firefox graphviz graphviz-gd ImageMagick librsvg2-tools -texlive-xetex-bin texlive-collection-fontsrecommended -texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts -dejavu-sans-mono-fonts" - -case $DIST_ID-$DIST_VERS in - ubuntu-16.04|ubuntu-18.04) - SEARX_PACKAGES="${SEARX_PACKAGES_debian}" - BUILD_PACKAGES="${BUILD_PACKAGES_debian}" - APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi" - ;; - ubuntu-20.04) - # https://askubuntu.com/a/1224710 - SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3" - BUILD_PACKAGES="${BUILD_PACKAGES_debian}" - ;; - ubuntu-*|debian-*) - SEARX_PACKAGES="${SEARX_PACKAGES_debian}" - BUILD_PACKAGES="${BUILD_PACKAGES_debian}" - ;; - arch-*) - SEARX_PACKAGES="${SEARX_PACKAGES_arch}" - BUILD_PACKAGES="${BUILD_PACKAGES_arch}" - ;; - fedora-*) - SEARX_PACKAGES="${SEARX_PACKAGES_fedora}" - BUILD_PACKAGES="${BUILD_PACKAGES_fedora}" - ;; - centos-7) - SEARX_PACKAGES="${SEARX_PACKAGES_centos}" - BUILD_PACKAGES="${BUILD_PACKAGES_centos}" - ;; -esac - -# Apache Settings -APACHE_SEARX_SITE="searxng.conf" - -# shellcheck disable=SC2034 -CONFIG_FILES=( - "${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP}" -) - -# shellcheck disable=SC2034 -CONFIG_BACKUP_ENCRYPTED=( - "${SEARXNG_SETTINGS_PATH}" -) +SEARXNG_SETTINGS_PATH="/etc/searx/settings.yml" +SEARXNG_UWSGI_APP="searx.ini" # ---------------------------------------------------------------------------- usage() { @@ -137,286 +21,30 @@ usage() { # shellcheck disable=SC1117 cat <<EOF usage:: - $(basename "$0") shell - $(basename "$0") install [all|check|init-src|dot-config|user|searx-src|pyenv|uwsgi|packages|settings|buildhost] - $(basename "$0") reinstall all - $(basename "$0") update [searx] - $(basename "$0") remove [all|user|pyenv|searx-src] - $(basename "$0") activate [service] - $(basename "$0") deactivate [service] - $(basename "$0") inspect [service|settings <key>] - $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>] - $(basename "$0") apache [install|remove] + $(basename "$0") remove all -shell - start interactive shell from user ${SERVICE_USER} -install / remove - :all: complete (de-) installation of SearXNG service - :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) - :dot-config: copy ./config.sh to ${SEARX_SRC} - :searx-src: clone $GIT_URL - :init-src: copy files (SEARX_SRC_INIT_FILES) to ${SEARX_SRC} - :pyenv: create/remove virtualenv (python) in $SEARX_PYENV - :uwsgi: install SearXNG uWSGI application - :settings: reinstall settings from ${SEARXNG_SETTINGS_PATH} - :packages: install needed packages from OS package manager - :buildhost: install packages from OS package manager needed by buildhosts -install - :check: check the SearXNG installation -reinstall: - :all: runs 'install/remove all' -update searx - Update SearXNG installation ($SERVICE_HOME) -activate service - activate and start service daemon (systemd unit) -deactivate service - stop and deactivate service daemon (systemd unit) -inspect - :service: run some small tests and inspect service's status and log - :settings: inspect YAML setting <key> from SearXNG instance (${SEARX_SRC}) -option - set one of the available options -apache - :install: apache site with the SearXNG uwsgi app - :remove: apache site ${APACHE_FILTRON_SITE} ----- sourced ${DOT_CONFIG} - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} +remove all: complete uninstall of SearXNG service EOF - install_log_searx_instance [[ -n ${1} ]] && err_msg "$1" } main() { - required_commands \ - sudo systemctl install git wget curl \ - || exit local _usage="unknown or missing $1 command $2" case $1 in - --getenv) var="$2"; echo "${!var}"; exit 0;; - -h|--help) usage; exit 0;; - shell) - sudo_or_exit - interactive_shell "${SERVICE_USER}" - ;; - inspect) - case $2 in - service) - sudo_or_exit - inspect_service - ;; - settings) - prompt_installation_setting "$3" - dump_return $? - ;; - *) usage "$_usage"; exit 42;; - esac ;; - reinstall) - rst_title "re-install $SERVICE_NAME" part - sudo_or_exit - case $2 in - all) - remove_all - install_all - ;; - *) usage "$_usage"; exit 42;; - esac ;; - install) - sudo_or_exit - case $2 in - all) - rst_title "SearXNG (install)" part - install_all - ;; - check) - rst_title "SearXNG (check installation)" part - verify_continue_install - install_check - ;; - user) - rst_title "SearXNG (install user)" - verify_continue_install - assert_user - ;; - pyenv) - rst_title "SearXNG (install pyenv)" - verify_continue_install - create_pyenv - ;; - searx-src) - rst_title "SearXNG (install searx-src)" - verify_continue_install - assert_user - clone_searx - install_DOT_CONFIG - init_SEARX_SRC - ;; - init-src) - init_SEARX_SRC - ;; - dot-config) - install_DOT_CONFIG - ;; - settings) - install_settings - ;; - uwsgi) - rst_title "SearXNG (install uwsgi)" - verify_continue_install - install_searx_uwsgi - if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" - fi - ;; - packages) - rst_title "SearXNG (install packages)" - pkg_install "$SEARX_PACKAGES" - ;; - buildhost) - rst_title "SearXNG (install buildhost)" - pkg_install "$SEARX_PACKAGES" - pkg_install "$BUILD_PACKAGES" - ;; - *) usage "$_usage"; exit 42;; - esac ;; - update) - sudo_or_exit - case $2 in - searx) update_searx;; - *) usage "$_usage"; exit 42;; - esac ;; remove) rst_title "SearXNG (remove)" part sudo_or_exit case $2 in all) remove_all;; - user) drop_service_account "${SERVICE_USER}";; - pyenv) remove_pyenv ;; - searx-src) remove_searx ;; - *) usage "$_usage"; exit 42;; - esac ;; - activate) - sudo_or_exit - case $2 in - service) - activate_service ;; *) usage "$_usage"; exit 42;; esac ;; - deactivate) - sudo_or_exit - case $2 in - service) deactivate_service ;; - *) usage "$_usage"; exit 42;; - esac ;; - option) - sudo_or_exit - case $2 in - debug-on) echo; enable_debug ;; - debug-off) echo; disable_debug ;; - result-proxy) set_result_proxy "$3" "$4" ;; - image-proxy-on) enable_image_proxy ;; - image-proxy-off) disable_image_proxy ;; - *) usage "$_usage"; exit 42;; - esac ;; - apache) - sudo_or_exit - case $2 in - install) install_apache_site ;; - remove) remove_apache_site ;; - *) usage "$_usage"; exit 42;; - esac ;; - doc) rst-doc;; *) usage "unknown or missing command $1"; exit 42;; esac } -_service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} " - -install_all() { - rst_title "Install SearXNG (service)" - verify_continue_install - pkg_install "$SEARX_PACKAGES" - wait_key - assert_user - wait_key - clone_searx - wait_key - install_DOT_CONFIG - wait_key - init_SEARX_SRC - wait_key - create_pyenv - wait_key - install_settings - wait_key - test_local_searx - wait_key - install_searx_uwsgi - if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" - fi - if ask_yn "Do you want to inspect the installation?" Ny; then - inspect_service - fi -} - -install_check() { - if service_account_is_available "$SERVICE_USER"; then - info_msg "Service account $SERVICE_USER exists." - else - err_msg "Service account $SERVICE_USER does not exists!" - fi - - if pyenv_is_available; then - info_msg "~$SERVICE_USER: python environment is available." - else - err_msg "~$SERVICE_USER: python environment is not available!" - fi - - if clone_is_available; then - info_msg "~$SERVICE_USER: SearXNG software is installed." - else - err_msg "~$SERVICE_USER: Missing SearXNG software!" - fi - - if uWSGI_app_enabled "$SEARXNG_UWSGI_APP"; then - info_msg "uWSGI app $SEARXNG_UWSGI_APP is enabled." - else - err_msg "uWSGI app $SEARXNG_UWSGI_APP not enabled!" - fi - - uWSGI_app_available "$SEARXNG_UWSGI_APP" \ - || err_msg "uWSGI app $SEARXNG_UWSGI_APP not available!" - - sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" "utils/searxng_check.py" - - if uWSGI_app_available 'searx.ini'; then - warn_msg "old searx.ini uWSGI app exists" - warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all" - fi -} - -update_searx() { - rst_title "Update SearXNG instance" - - rst_para "fetch from $GIT_URL and reset to origin/$GIT_BRANCH" - tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -git fetch origin "$GIT_BRANCH" -git reset --hard "origin/$GIT_BRANCH" -pip install -U pip -pip install -U setuptools -pip install -U wheel -pip install -U pyyaml -pip install -U -e . -EOF - install_settings - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - remove_all() { rst_title "De-Install SearXNG (service)" @@ -436,250 +64,6 @@ installations that were installed with this script." fi } -assert_user() { - rst_title "user $SERVICE_USER" section - echo - if getent passwd "$SERVICE_USER" > /dev/null; then - echo "user exists" - return 0 - fi - - tee_stderr 1 <<EOF | bash | prefix_stdout -useradd --shell /bin/bash --system \ - --home-dir "$SERVICE_HOME" \ - --comment 'Privacy-respecting metasearch engine' $SERVICE_USER -mkdir "$SERVICE_HOME" -chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" -groups $SERVICE_USER -EOF - #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" - #export SERVICE_HOME - #echo "export SERVICE_HOME=$SERVICE_HOME" -} - -clone_is_available() { - [[ -f "$SEARX_SRC/.git/config" ]] -} - -# shellcheck disable=SC2164 -clone_searx() { - rst_title "Clone SearXNG sources" section - echo - if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then - die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT" - fi - SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" - if [[ ! "${SERVICE_HOME}" ]]; then - err_msg "to clone SearXNG sources, user $SERVICE_USER hast to be created first" - return 42 - fi - if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then - warn_msg "missing local branch ${GIT_BRANCH}" - info_msg "create local branch ${GIT_BRANCH} from start point: origin/${GIT_BRANCH}" - git branch "${GIT_BRANCH}" "origin/${GIT_BRANCH}" - fi - if [[ ! $(git rev-parse --abbrev-ref HEAD) == "${GIT_BRANCH}" ]]; then - warn_msg "take into account, installing branch $GIT_BRANCH while current branch is $(git rev-parse --abbrev-ref HEAD)" - fi - export SERVICE_HOME - git_clone "$REPO_ROOT" "$SEARX_SRC" \ - "$GIT_BRANCH" "$SERVICE_USER" - - pushd "${SEARX_SRC}" > /dev/null - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -cd "${SEARX_SRC}" -git remote set-url origin ${GIT_URL} -git config user.email "$ADMIN_EMAIL" -git config user.name "$ADMIN_NAME" -git config --list -EOF - popd > /dev/null -} - -prompt_installation_status(){ - - # shellcheck disable=SC2034 - local GIT_URL GIT_BRANCH VERSION_STRING VERSION_TAG - local ret_val state branch remote remote_url - state="$(install_searx_get_state)" - - case $state in - missing-searx-clone|missing-searx-pyenv) - info_msg "${_BBlue}(status: $(install_searx_get_state))${_creset}" - return 0 - ;; - *) - info_msg "SearXNG instance already installed at: $SEARX_SRC" - info_msg "status: ${_BBlue}$(install_searx_get_state)${_creset} " - branch="$(git name-rev --name-only HEAD)" - remote="$(git config branch."${branch}".remote)" - remote_url="$(git config remote."${remote}".url)" - eval "$(get_installed_version_variables)" - - ret_val=0 - if ! [ "$GIT_URL" = "$remote_url" ]; then - warn_msg "instance's git URL: '${GIT_URL}'" \ - "differs from local clone's remote URL: ${remote_url}" - ret_val=42 - fi - if ! [ "$GIT_BRANCH" = "$branch" ]; then - warn_msg "instance git branch: ${GIT_BRANCH}" \ - "differs from local clone's branch: ${branch}" - ret_val=42 - fi - return $ret_val - ;; - esac -} - -verify_continue_install(){ - if ! prompt_installation_status; then - MSG="[${_BCyan}KEY${_creset}] to continue installation / [${_BCyan}CTRL-C${_creset}] to exit" \ - wait_key - fi -} - -prompt_installation_setting(){ - - # usage: prompt_installation_setting brand.docs_url - # - # Prompts the value of the (YAML) setting in the SearXNG instance. - - local _state - _state="$(install_searx_get_state)" - case $_state in - python-installed|installer-modified) - sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" <<EOF -import sys -from searx import get_setting -name = "${1}" -unset = object() -value = get_setting(name, unset) -if value is unset: - sys.stderr.write("error: setting '%s' does not exists\n" % name) - sys.exit(42) -print(value) -sys.exit(0) -EOF - ;; - *) - return 42 - ;; - esac -} - -get_installed_version_variables() { - - # usage: eval "$(get_installed_version_variables)" - # - # Set variables VERSION_STRING, VERSION_TAG, GIT_URL, GIT_BRANCH - - local _state - _state="$(install_searx_get_state)" - case $_state in - python-installed|installer-modified) - sudo -H -u "${SERVICE_USER}" "${SEARX_PYENV}/bin/python" -m searx.version;; - *) - return 42 - ;; - esac -} - -init_SEARX_SRC(){ - rst_title "Update instance: ${SEARX_SRC}/" section - - if ! clone_is_available; then - err_msg "you have to install SearXNG first" - return 1 - fi - - init_SEARX_SRC_INIT_FILES - - if [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then - info_msg "no files registered in SEARX_SRC_INIT_FILES" - return 2 - fi - - echo - echo "Update instance with file(s) from: ${REPO_ROOT}" - echo - for i in "${SEARX_SRC_INIT_FILES[@]}"; do - echo "- $i" - done - echo - echo "Be careful when modifying an existing installation." - if ! ask_yn "Do you really want to update these files in the instance?" Yn; then - return 42 - fi - for fname in "${SEARX_SRC_INIT_FILES[@]}"; do - while true; do - choose_one _reply "choose next step with file ${fname}" \ - "replace file" \ - "leave file unchanged" \ - "diff files" \ - "interactive shell" - - case $_reply in - "leave file unchanged") - break - ;; - "replace file") - info_msg "copy: ${REPO_ROOT}/${fname} --> ${SEARX_SRC}/${fname}" - cp "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}" - break - ;; - "diff files") - $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" - ;; - "interactive shell") - backup_file "${SEARX_SRC}/${fname}" - echo -e "// edit ${_Red}${dst}${_creset} to your needs" - echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" - sudo -H -u "${SERVICE_USER}" -i - $DIFF_CMD "${SEARX_SRC}/${fname}" "${REPO_ROOT}/${fname}" - echo - echo -e "// ${_BBlack}did you edit file ...${_creset}" - echo -en "// ${_Red}${dst}${_creset}" - if ask_yn "//${_BBlack}... to your needs?${_creset}"; then - break - fi - ;; - esac - done - done -} - -install_DOT_CONFIG(){ - rst_title "Update instance: ${SEARX_SRC}/.config.sh" section - - if cmp --silent "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh"; then - info_msg "${SEARX_SRC}/.config.sh is up to date" - return 0 - fi - - diff "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" - if ! ask_yn "Do you want to copy file .config.sh into instance?" Yn; then - return 42 - fi - backup_file "${SEARX_SRC}/.config.sh" - cp "${REPO_ROOT}/.config.sh" "${SEARX_SRC}/.config.sh" -} - -install_settings() { - rst_title "${SEARXNG_SETTINGS_PATH}" section - - if ! clone_is_available; then - err_msg "you have to install SearXNG first" - exit 42 - fi - - mkdir -p "$(dirname "${SEARXNG_SETTINGS_PATH}")" - install_template --no-eval \ - "${SEARXNG_SETTINGS_PATH}" \ - "${SERVICE_USER}" "${SERVICE_GROUP}" - configure_searx -} - remove_settings() { rst_title "remove SearXNG settings" section echo @@ -687,419 +71,12 @@ remove_settings() { rm -f "${SEARXNG_SETTINGS_PATH}" } -remove_searx() { - rst_title "Drop SearXNG sources" section - if ask_yn "Do you really want to drop SearXNG sources ($SEARX_SRC)?"; then - rm -rf "$SEARX_SRC" - else - rst_para "Leave SearXNG sources unchanged." - fi -} - -pyenv_is_available() { - [[ -f "${SEARX_PYENV}/bin/activate" ]] -} - -create_pyenv() { - rst_title "Create virtualenv (python)" section - echo - if [[ ! -f "${SEARX_SRC}/manage" ]]; then - err_msg "to create pyenv for SearXNG, SearXNG has to be cloned first" - return 42 - fi - info_msg "create pyenv in ${SEARX_PYENV}" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -rm -rf "${SEARX_PYENV}" -python3 -m venv "${SEARX_PYENV}" -grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ - || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile -EOF - info_msg "inspect python's virtual environment" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -command -v python && python --version -EOF - wait_key - info_msg "install needed python packages" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -pip install -U pip -pip install -U setuptools -pip install -U wheel -pip install -U pyyaml -cd ${SEARX_SRC} -pip install -e . -EOF -} - -remove_pyenv() { - rst_title "Remove virtualenv (python)" section - if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then - return - fi - info_msg "remove pyenv activation from ~/.profile" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## -mv ~/.profile.## ~/.profile -EOF - rm -rf "${SEARX_PYENV}" -} - -configure_searx() { - rst_title "Configure SearXNG" section - rst_para "Setup SearXNG config located at $SEARXNG_SETTINGS_PATH" - echo - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARXNG_SETTINGS_PATH" -EOF -} - -test_local_searx() { - rst_title "Testing SearXNG instance localy" section - echo - - if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then - err_msg "URL/port http://${SEARX_INTERNAL_HTTP} is already in use, you" - err_msg "should stop that service before starting local tests!" - if ! ask_yn "Continue with local tests?"; then - return - fi - fi - sed -i -e "s/debug: false/debug: true/g" "$SEARXNG_SETTINGS_PATH" - tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" -export SEARXNG_SETTINGS_PATH="${SEARXNG_SETTINGS_PATH}" -cd ${SEARX_SRC} -timeout 10 python searx/webapp.py & -sleep 3 -curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP -EOF - sed -i -e "s/debug: true/debug: false/g" "$SEARXNG_SETTINGS_PATH" -} - -install_searx_uwsgi() { - rst_title "Install SearXNG's uWSGI app (searxng.ini)" section - echo - install_uwsgi - uWSGI_install_app "$SEARXNG_UWSGI_APP" -} - remove_searx_uwsgi() { rst_title "Remove SearXNG's uWSGI app (searxng.ini)" section echo uWSGI_remove_app "$SEARXNG_UWSGI_APP" } -activate_service() { - rst_title "Activate SearXNG (service)" section - echo - uWSGI_enable_app "$SEARXNG_UWSGI_APP" - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -deactivate_service() { - rst_title "De-Activate SearXNG (service)" section - echo - uWSGI_disable_app "$SEARXNG_UWSGI_APP" - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -enable_image_proxy() { - info_msg "try to enable image_proxy ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/image_proxy: false/image_proxy: true/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -disable_image_proxy() { - info_msg "try to enable image_proxy ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/image_proxy: true/image_proxy: false/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -enable_debug() { - warn_msg "Do not enable debug in production environments!!" - info_msg "try to enable debug mode ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/debug: false/debug: true/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -disable_debug() { - info_msg "try to disable debug mode ..." - tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" -cd ${SEARX_SRC} -sed -i -e "s/debug: true/debug: false/g" "$SEARXNG_SETTINGS_PATH" -EOF - uWSGI_restart "$SEARXNG_UWSGI_APP" -} - -set_result_proxy() { - - # usage: set_result_proxy <URL> [<key>] - - info_msg "try to set result proxy: '$1' ($2)" - cp "${SEARXNG_SETTINGS_PATH}" "${SEARXNG_SETTINGS_PATH}.bak" - _set_result_proxy "$1" "$2" > "${SEARXNG_SETTINGS_PATH}" -} - -_set_result_proxy() { - local line - local stage=0 - local url=" url: $1" - local key=" key: !!binary \"$2\"" - if [[ -z $2 ]]; then - key= - fi - - while IFS= read -r line - do - if [[ $stage = 0 ]] || [[ $stage = 2 ]] ; then - if [[ $line =~ ^[[:space:]]*#*[[:space:]]*result_proxy[[:space:]]*:[[:space:]]*$ ]]; then - if [[ $stage = 0 ]]; then - stage=1 - echo "result_proxy:" - continue - elif [[ $stage = 2 ]]; then - continue - fi - fi - fi - if [[ $stage = 1 ]] || [[ $stage = 2 ]] ; then - if [[ $line =~ ^[[:space:]]*#*[[:space:]]*url[[:space:]]*:[[:space:]] ]]; then - [[ $stage = 1 ]] && echo "$url" - continue - elif [[ $line =~ ^[[:space:]]*#*[[:space:]]*key[[:space:]]*:[[:space:]] ]]; then - [[ $stage = 1 ]] && [[ -n $key ]] && echo "$key" - continue - elif [[ $line =~ ^[[:space:]]*$ ]]; then - stage=2 - fi - fi - echo "$line" - done < "${SEARXNG_SETTINGS_PATH}.bak" -} - -function has_substring() { - [[ "$1" != "${2/$1/}" ]] -} -inspect_service() { - rst_title "service status & log" - cat <<EOF - -sourced ${DOT_CONFIG} : - SERVICE_USER : ${SERVICE_USER} - SERVICE_HOME : ${SERVICE_HOME} -EOF - install_log_searx_instance - - install_check - if in_container; then - lxc_suite_info - else - info_msg "public URL --> ${PUBLIC_URL}" - info_msg "internal URL --> http://${SEARX_INTERNAL_HTTP}" - fi - - if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then - err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_HTTP} is not available!" - MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue"\ - wait_key - fi - - if ! service_is_available "${PUBLIC_URL}"; then - warn_msg "Public service at ${PUBLIC_URL} is not available!" - if ! in_container; then - warn_msg "Check if public name is correct and routed or use the public IP from above." - fi - fi - - local _debug_on - if ask_yn "Enable SearXNG debug mode?"; then - enable_debug - _debug_on=1 - fi - echo - - case $DIST_ID-$DIST_VERS in - ubuntu-*|debian-*) - systemctl --no-pager -l status "${SERVICE_NAME}" - ;; - arch-*) - systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}" - ;; - fedora-*|centos-7) - systemctl --no-pager -l status uwsgi - ;; - esac - - # shellcheck disable=SC2059 - printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" - read -r -s -n1 -t 5 - echo - - while true; do - trap break 2 - case $DIST_ID-$DIST_VERS in - ubuntu-*|debian-*) tail -f /var/log/uwsgi/app/searx.log ;; - arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;; - fedora-*|centos-7) journalctl -f -u uwsgi ;; - esac - done - - if [[ $_debug_on == 1 ]]; then - disable_debug - fi - return 0 -} - -install_apache_site() { - rst_title "Install Apache site $APACHE_SEARX_SITE" - - rst_para "\ -This installs the SearXNG uwsgi app as apache site. If your server is public to -the internet, you should instead use a reverse proxy (filtron) to block -excessively bot queries." - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - else - install_apache - fi - - apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" - - rst_title "Install SearXNG's uWSGI app (searxng.ini)" section - echo - uWSGI_install_app --variant=socket "$SEARXNG_UWSGI_APP" - - if ! service_is_available "${PUBLIC_URL}"; then - err_msg "Public service at ${PUBLIC_URL} is not available!" - fi -} - -remove_apache_site() { - - rst_title "Remove Apache site ${APACHE_SEARX_SITE}" - - rst_para "\ -This removes apache site ${APACHE_SEARX_SITE}." - - ! apache_is_installed && err_msg "Apache is not installed." - - if ! ask_yn "Do you really want to continue?" Yn; then - return - fi - - apache_remove_site "${APACHE_SEARX_SITE}" - - rst_title "Remove SearXNG's uWSGI app (searxng.ini)" section - echo - uWSGI_remove_app "$SEARXNG_UWSGI_APP" -} - -rst-doc() { - local debian="${SEARX_PACKAGES_debian}" - local arch="${SEARX_PACKAGES_arch}" - local fedora="${SEARX_PACKAGES_fedora}" - local centos="${SEARX_PACKAGES_centos}" - local debian_build="${BUILD_PACKAGES_debian}" - local arch_build="${BUILD_PACKAGES_arch}" - local fedora_build="${BUILD_PACKAGES_fedora}" - local centos_build="${SEARX_PACKAGES_centos}" - debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - centos="$(echo "${centos}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - centos_build="$(echo "${centos_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" - - eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searx.rst")\"" - - # I use ubuntu-20.04 here to demonstrate that versions are also suported, - # normaly debian-* and ubuntu-* are most the same. - - for DIST_NAME in ubuntu-20.04 arch fedora; do - ( - DIST_ID=${DIST_NAME%-*} - DIST_VERS=${DIST_NAME#*-} - [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= - uWSGI_distro_setup - - echo -e "\n.. START searxng uwsgi-description $DIST_NAME" - - case $DIST_ID-$DIST_VERS in - ubuntu-*|debian-*) cat <<EOF - -.. code:: bash - - # init.d --> /usr/share/doc/uwsgi/README.Debian.gz - # For uWSGI debian uses the LSB init process, this might be changed - # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 - - create ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} - enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ - start: sudo -H service uwsgi start ${SEARXNG_UWSGI_APP%.*} - restart: sudo -H service uwsgi restart ${SEARXNG_UWSGI_APP%.*} - stop: sudo -H service uwsgi stop ${SEARXNG_UWSGI_APP%.*} - disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - -EOF - ;; - arch-*) cat <<EOF - -.. code:: bash - - # systemd --> /usr/lib/systemd/system/uwsgi@.service - # For uWSGI archlinux uses systemd template units, see - # - http://0pointer.de/blog/projects/instances.html - # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd - - create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - enable: sudo -H systemctl enable uwsgi@${SEARXNG_UWSGI_APP%.*} - start: sudo -H systemctl start uwsgi@${SEARXNG_UWSGI_APP%.*} - restart: sudo -H systemctl restart uwsgi@${SEARXNG_UWSGI_APP%.*} - stop: sudo -H systemctl stop uwsgi@${SEARXNG_UWSGI_APP%.*} - disable: sudo -H systemctl disable uwsgi@${SEARXNG_UWSGI_APP%.*} - -EOF - ;; - fedora-*|centos-7) cat <<EOF - -.. code:: bash - - # systemd --> /usr/lib/systemd/system/uwsgi.service - # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see - # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html - - create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} - -EOF - ;; - esac - echo -e ".. END searxng uwsgi-description $DIST_NAME" - - echo -e "\n.. START searxng uwsgi-appini $DIST_NAME" - echo ".. code:: bash" - echo - eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP}")\"" | prefix_stdout " " - echo -e "\n.. END searxng uwsgi-appini $DIST_NAME" - - ) - done - -} # ---------------------------------------------------------------------------- main "$@" diff --git a/utils/searxng.sh b/utils/searxng.sh new file mode 100755 index 000000000..d9ecdcca3 --- /dev/null +++ b/utils/searxng.sh @@ -0,0 +1,1017 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2001 + +# Script options from the environment: +SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET:-true}" + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/lib_redis.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib_redis.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +SERVICE_NAME="searxng" +SERVICE_USER="searxng" +SERVICE_HOME="/usr/local/searxng" +SERVICE_GROUP="searxng" + +SEARXNG_SRC="${SERVICE_HOME}/searxng-src" +# shellcheck disable=SC2034 +SEARXNG_STATIC="${SEARXNG_SRC}/searx/static" + +SEARXNG_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml" +SEARXNG_UWSGI_APP="searxng.ini" + +SEARXNG_INTERNAL_HTTP="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" +if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + SEARXNG_UWSGI_SOCKET="${SERVICE_HOME}/run/socket" +else + SEARXNG_UWSGI_SOCKET= +fi + +# SEARXNG_URL: the public URL of the instance (https://example.org/searxng). The +# value is taken from environment ${SEARXNG_URL} in ./utils/brand.env. This +# variable is an empty string if server.base_url in the settings.yml is set to +# 'false'. + +SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)/searxng}" +SEARXNG_URL="${SEARXNG_URL%/}" # if exists, remove trailing slash +if in_container; then + # hint: Linux containers do not have DNS entries, lets use IPs + SEARXNG_URL="http://$(primary_ip)/searxng" +fi +SEARXNG_URL_PATH="$(echo "${SEARXNG_URL}" | sed -e 's,^.*://[^/]*\(/.*\),\1,g')" +[[ "${SEARXNG_URL_PATH}" == "${SEARXNG_URL}" ]] && SEARXNG_URL_PATH=/ + +# Apache settings + +APACHE_SEARXNG_SITE="searxng.conf" + +# nginx settings + +NGINX_SEARXNG_SITE="searxng.conf" + +# apt packages + +SEARXNG_PACKAGES_debian="\ +python3-dev python3-babel python3-venv +uwsgi uwsgi-plugin-python3 +git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev" + +SEARXNG_BUILD_PACKAGES_debian="\ +firefox graphviz imagemagick texlive-xetex librsvg2-bin +texlive-latex-recommended texlive-extra-utils fonts-dejavu +latexmk shellcheck" + +# pacman packages + +SEARXNG_PACKAGES_arch="\ +python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python +git base-devel libxml2" + +SEARXNG_BUILD_PACKAGES_arch="\ +firefox graphviz imagemagick texlive-bin extra/librsvg +texlive-core texlive-latexextra ttf-dejavu shellcheck" + +# dnf packages + +SEARXNG_PACKAGES_fedora="\ +python python-pip python-lxml python-babel python3-devel +uwsgi uwsgi-plugin-python3 +git @development-tools libxml2 openssl" + +SEARXNG_BUILD_PACKAGES_fedora="\ +firefox graphviz graphviz-gd ImageMagick librsvg2-tools +texlive-xetex-bin texlive-collection-fontsrecommended +texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts +dejavu-sans-mono-fonts ShellCheck" + +case $DIST_ID-$DIST_VERS in + ubuntu-18.04) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi" + ;; + ubuntu-20.04) + # https://wiki.ubuntu.com/FocalFossa/ReleaseNotes#Python3_by_default + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian} python-is-python3" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + ;; + ubuntu-*|debian-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + ;; + arch-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_arch}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_arch}" + ;; + fedora-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_fedora}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_fedora}" + ;; +esac + +_service_prefix=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage: + $(basename "$0") install [all|user|pyenv|settings|uwsgi|redis|nginx|apache|searxng-src|packages|buildhost] + $(basename "$0") remove [all|user|pyenv|settings|uwsgi|redis|nginx|apache] + $(basename "$0") instance [cmd|update|check|localtest|inspect] +install|remove: + all : complete (de-) installation of the SearXNG service + user : service user '${SERVICE_USER}' (${SERVICE_HOME}) + pyenv : virtualenv (python) in ${SEARXNG_PYENV} + settings : settings from ${SEARXNG_SETTINGS_PATH} + uwsgi : SearXNG's uWSGI app ${SEARXNG_UWSGI_APP} + redis : build & install or remove a local redis server ${REDIS_HOME}/run/redis.sock + nginx : HTTP site ${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE} + apache : HTTP site ${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE} +install: + searxng-src : clone ${GIT_URL} into ${SEARXNG_SRC} + packages : installs packages from OS package manager required by SearXNG + buildhost : installs packages from OS package manager required by a SearXNG buildhost +instance: + update : update SearXNG instance (git fetch + reset & update settings.yml) + check : run checks from utils/searxng_check.py in the active installation + inspect : run some small tests and inspect SearXNG's server status and log + get_setting : get settings value from running SearXNG instance + cmd : run command in SearXNG instance's environment (e.g. bash) +EOF + searxng.instance.env + [[ -n ${1} ]] && err_msg "$1" +} + +searxng.instance.env() { + echo "uWSGI:" + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + echo " SEARXNG_UWSGI_SOCKET : ${SEARXNG_UWSGI_SOCKET}" + else + echo " SEARXNG_INTERNAL_HTTP: ${SEARXNG_INTERNAL_HTTP}" + fi + cat <<EOF +environment ${SEARXNG_SRC}/utils/brand.env: + GIT_URL : ${GIT_URL} + GIT_BRANCH : ${GIT_BRANCH} + SEARXNG_URL : ${SEARXNG_URL} + SEARXNG_PORT : ${SEARXNG_PORT} + SEARXNG_BIND_ADDRESS : ${SEARXNG_BIND_ADDRESS} +EOF +} + +main() { + required_commands \ + sudo systemctl install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + install) + sudo_or_exit + case $2 in + all) searxng.install.all;; + user) searxng.install.user;; + pyenv) searxng.install.pyenv;; + searxng-src) searxng.install.clone;; + settings) searxng.install.settings;; + uwsgi) searxng.install.uwsgi;; + packages) searxng.install.packages;; + buildhost) searxng.install.buildhost;; + nginx) searxng.nginx.install;; + apache) searxng.apache.install;; + redis) searxng.install.redis;; + *) usage "$_usage"; exit 42;; + esac + ;; + remove) + sudo_or_exit + case $2 in + all) searxng.remove.all;; + user) drop_service_account "${SERVICE_USER}";; + pyenv) searxng.remove.pyenv;; + settings) searxng.remove.settings;; + uwsgi) searxng.remove.uwsgi;; + apache) searxng.apache.remove;; + remove) searxng.nginx.remove;; + redis) searxng.remove.redis;; + *) usage "$_usage"; exit 42;; + esac + ;; + instance) + case $2 in + update) + sudo_or_exit + searxng.instance.update + ;; + check) + sudo_or_exit + searxng.instance.self.call searxng.check + ;; + inspect) + sudo_or_exit + searxng.instance.inspect + ;; + cmd) + sudo_or_exit + shift; shift; searxng.instance.exec "$@" + ;; + get_setting) + shift; shift; searxng.instance.get_setting "$@" + ;; + call) + # call a function in instance's environment + shift; shift; searxng.instance.self.call "$@" + ;; + _call) + shift; shift; "$@" + ;; + *) usage "$_usage"; exit 42;; + esac + ;; + *) + local cmd="$1" + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + usage "unknown or missing command $1" + exit 42 + else + "$cmd" "$@" + fi + ;; + esac +} + +searxng.install.all() { + rst_title "SearXNG installation" part + + local redis_url + + rst_title "SearXNG" + searxng.install.packages + wait_key 10 + searxng.install.user + wait_key 10 + searxng.install.clone + wait_key + searxng.install.pyenv + wait_key + searxng.install.settings + wait_key + searxng.instance.localtest + wait_key + searxng.install.uwsgi + wait_key + + rst_title "Redis DB" + searxng.install.redis.db + + rst_title "HTTP Server" + searxng.install.http.site + + rst_title "Finalize installation" + if ask_yn "Do you want to run some checks?" Yn; then + searxng.instance.self.call searxng.check + fi +} + +searxng.install.redis.db() { + local redis_url + + redis_url=$(searxng.instance.get_setting redis.url) + rst_para "\ +In your instance, redis DB connector is configured at: + + ${redis_url} +" + if searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + info_msg "SearXNG instance is able to connect redis DB." + return + fi + if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then + err_msg "SearXNG instance can't connect redis DB / check redis & your settings" + return + fi + rst_para ".. but this redis DB is not installed yet." + + case $DIST_ID-$DIST_VERS in + fedora-*) + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # + # HINT: without option ``emperor-tyrant-initgroups=true`` in + # ``/etc/uwsgi.ini`` the process won't get the additional groups, + # but this option is not available in 2.0.x branch [2][3] / on + # fedora35 there is v2.0.20 installed --> no way to get additional + # groups on fedora's tyrant mode. + # + # ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... + # ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + # ERROR:searx.plugins.limiter: init limiter DB failed!!! + # + # $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' + # searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini + # searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini + # + # Additional groups: + # + # $ groups searxng + # searxng : searxng searxng-redis + # + # Here you can see that the additional "Groups" of PID 186 are unset + # (missing gid of searxng-redis) + # + # $ cat /proc/186/task/186/status + # ... + # Uid: 993 993 993 993 + # Gid: 993 993 993 993 + # FDSize: 128 + # Groups: + # ... + # + # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting + # [2] https://github.com/unbit/uwsgi/issues/2099 + # [3] https://github.com/unbit/uwsgi/pull/752 + + rst_para "\ +Fedora uses emperor-tyrant mode / in this mode we had a lot of trouble with +sockets and permissions of the vasals. We recommend to setup a redis DB +and using redis:// TCP protocol in the settings.yml configuration." + ;; + *) + if ask_yn "Do you want to install the redis DB now?" Yn; then + searxng.install.redis + uWSGI_restart "$SEARXNG_UWSGI_APP" + fi + ;; + esac +} + +searxng.install.http.site() { + + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy" Yn; then + searxng.apache.install + fi + elif nginx_is_installed; then + info_msg "Nginx is installed on this host." + if ask_yn "Do you want to install a reverse proxy" Yn; then + searxng.nginx.install + fi + else + info_msg "Don't forget to install HTTP site." + fi +} + +searxng.remove.all() { + local redis_url + + rst_title "De-Install SearXNG (service)" + if ! ask_yn "Do you really want to deinstall SearXNG?"; then + return + fi + + redis_url=$(searxng.instance.get_setting redis.url) + if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then + searxng.remove.redis + fi + + searxng.remove.uwsgi + drop_service_account "${SERVICE_USER}" + searxng.remove.settings + wait_key + + if service_is_available "${SEARXNG_URL}"; then + MSG="** Don't forgett to remove your public site! (${SEARXNG_URL}) **" wait_key 10 + fi +} + +searxng.install.user() { + rst_title "SearXNG -- install user" section + echo + if getent passwd "${SERVICE_USER}" > /dev/null; then + echo "user already exists" + return 0 + fi + + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "${SERVICE_HOME}" \ + --comment 'Privacy-respecting metasearch engine' ${SERVICE_USER} +mkdir "${SERVICE_HOME}" +chown -R "${SERVICE_GROUP}:${SERVICE_GROUP}" "${SERVICE_HOME}" +groups ${SERVICE_USER} +EOF +} + +searxng.install.packages() { + TITLE="SearXNG -- install packages" pkg_install "${SEARXNG_PACKAGES}" +} + +searxng.install.buildhost() { + TITLE="SearXNG -- install buildhost packages" pkg_install \ + "${SEARXNG_PACKAGES} ${SEARXNG_BUILD_PACKAGES}" +} + +searxng.install.clone() { + rst_title "Clone SearXNG sources" section + if ! service_account_is_available "${SERVICE_USER}"; then + die 42 "To clone SearXNG, first install user ${SERVICE_USER}." + fi + echo + if ! sudo -i -u "${SERVICE_USER}" ls -d "$REPO_ROOT" > /dev/null; then + die 42 "user '${SERVICE_USER}' missed read permission: $REPO_ROOT" + fi + # SERVICE_HOME="$(sudo -i -u "${SERVICE_USER}" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone SearXNG sources, user ${SERVICE_USER} hast to be created first" + return 42 + fi + if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then + warn_msg "missing local branch ${GIT_BRANCH}" + info_msg "create local branch ${GIT_BRANCH} from start point: origin/${GIT_BRANCH}" + git branch "${GIT_BRANCH}" "origin/${GIT_BRANCH}" + fi + if [[ ! $(git rev-parse --abbrev-ref HEAD) == "${GIT_BRANCH}" ]]; then + warn_msg "take into account, installing branch $GIT_BRANCH while current branch is $(git rev-parse --abbrev-ref HEAD)" + fi + # export SERVICE_HOME + + # clone repo and add a safe.directory entry to git's system config / see + # https://github.com/searxng/searxng/issues/1251 + git_clone "$REPO_ROOT" "${SEARXNG_SRC}" \ + "$GIT_BRANCH" "${SERVICE_USER}" + git config --system --add safe.directory "${SEARXNG_SRC}" + + pushd "${SEARXNG_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARXNG_SRC}" +git remote set-url origin ${GIT_URL} +git config user.email "${ADMIN_EMAIL}" +git config user.name "${ADMIN_NAME}" +git config --list +EOF + popd > /dev/null +} + +searxng.install.pyenv() { + rst_title "Create virtualenv (python)" section + echo + if [[ ! -f "${SEARXNG_SRC}/manage" ]]; then + die 42 "To create pyenv for SearXNG, first install searxng-src." + fi + info_msg "create pyenv in ${SEARXNG_PYENV}" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARXNG_PYENV}" +python3 -m venv "${SEARXNG_PYENV}" +grep -qFs -- 'source ${SEARXNG_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARXNG_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U pyyaml +cd ${SEARXNG_SRC} +pip install -e . +EOF +} + +searxng.remove.pyenv() { + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARXNG_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARXNG_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARXNG_PYENV}" +} + +searxng.install.settings() { + rst_title "install ${SEARXNG_SETTINGS_PATH}" section + + if ! [[ -f "${SEARXNG_SRC}/.git/config" ]]; then + die "Before install settings, first install SearXNG." + exit 42 + fi + + mkdir -p "$(dirname "${SEARXNG_SETTINGS_PATH}")" + + DEFAULT_SELECT=1 \ + install_template --no-eval \ + "${SEARXNG_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "root" +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "${SEARXNG_SETTINGS_PATH}" +EOF +} + +searxng.remove.settings() { + rst_title "remove ${SEARXNG_SETTINGS_PATH}" section + if ask_yn "Do you want to delete the SearXNG settings?" Yn; then + rm -f "${SEARXNG_SETTINGS_PATH}" + fi +} + +searxng.check() { + rst_title "SearXNG checks" section + + for NAME in "searx" "filtron" "morty"; do + if service_account_is_available "${NAME}"; then + err_msg "There exists an old '${NAME}' account from a previous installation." + else + info_msg "[OK] (old) account '${NAME}' does not exists" + fi + done + + "${SEARXNG_PYENV}/bin/python" "${SEARXNG_SRC}/utils/searxng_check.py" +} + +searxng.instance.update() { + rst_title "Update SearXNG instance" + rst_para "fetch from $GIT_URL and reset to origin/$GIT_BRANCH" + tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +git fetch origin "$GIT_BRANCH" +git reset --hard "origin/$GIT_BRANCH" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U pyyaml +pip install -U -e . +EOF + rst_para "update instance's settings.yml from ${SEARXNG_SETTINGS_PATH}" + DEFAULT_SELECT=2 \ + install_template --no-eval \ + "${SEARXNG_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + + sudo -H -i <<EOF +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "${SEARXNG_SETTINGS_PATH}" +EOF + uWSGI_restart "${SEARXNG_UWSGI_APP}" +} + +searxng.install.uwsgi() { + rst_title "SearXNG (install uwsgi)" + install_uwsgi + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + searxng.install.uwsgi.socket + else + searxng.install.uwsgi.http + fi +} + +searxng.install.uwsgi.http() { + rst_para "Install ${SEARXNG_UWSGI_APP} at: http://${SEARXNG_INTERNAL_HTTP}" + uWSGI_install_app "${SEARXNG_UWSGI_APP}" + if ! searxng.uwsgi.available; then + err_msg "URL http://${SEARXNG_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" + fi +} + +searxng.install.uwsgi.socket() { + rst_para "Install ${SEARXNG_UWSGI_APP} using socket at: ${SEARXNG_UWSGI_SOCKET}" + mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})" + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})" + + case $DIST_ID-$DIST_VERS in + fedora-*) + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting + uWSGI_install_app --variant=socket "${SEARXNG_UWSGI_APP}" "${SERVICE_USER}" "${SERVICE_GROUP}" + ;; + *) + uWSGI_install_app --variant=socket "${SEARXNG_UWSGI_APP}" + ;; + esac + sleep 5 + if ! searxng.uwsgi.available; then + err_msg "uWSGI socket not available at: ${SEARXNG_UWSGI_SOCKET}" + fi +} + +searxng.uwsgi.available() { + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + [[ -S "${SEARXNG_UWSGI_SOCKET}" ]] + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "uWSGI socket is located at: ${SEARXNG_UWSGI_SOCKET}" + fi + else + service_is_available "http://${SEARXNG_INTERNAL_HTTP}" + exit_val=$? + fi + return "$exit_val" +} + +searxng.remove.uwsgi() { + rst_title "Remove SearXNG's uWSGI app (${SEARXNG_UWSGI_APP})" section + echo + uWSGI_remove_app "${SEARXNG_UWSGI_APP}" +} + +searxng.install.redis() { + rst_title "SearXNG (install redis)" + redis.build + redis.install + redis.addgrp "${SERVICE_USER}" +} + +searxng.remove.redis() { + rst_title "SearXNG (remove redis)" + redis.rmgrp "${SERVICE_USER}" + redis.remove +} + +searxng.instance.localtest() { + rst_title "Test SearXNG instance localy" section + rst_para "Activate debug mode, start a minimal SearXNG "\ + "service and debug a HTTP request/response cycle." + + if service_is_available "http://${SEARXNG_INTERNAL_HTTP}" &>/dev/null; then + err_msg "URL/port http://${SEARXNG_INTERNAL_HTTP} is already in use, you" + err_msg "should stop that service before starting local tests!" + if ! ask_yn "Continue with local tests?"; then + return + fi + fi + echo + searxng.instance.debug.on + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +export SEARXNG_SETTINGS_PATH="${SEARXNG_SETTINGS_PATH}" +cd ${SEARXNG_SRC} +timeout 10 python searx/webapp.py & +sleep 3 +curl --location --verbose --head --insecure ${SEARXNG_INTERNAL_HTTP} +EOF + echo + searxng.instance.debug.off +} + +searxng.install.http.pre() { + if ! searxng.uwsgi.available; then + rst_para "\ +To install uWSGI use:: + + $(basename "$0") install uwsgi +" + die 42 "SearXNG's uWSGI app not available" + fi + + if ! searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + rst_para "\ +The configured redis DB is not available: If your server is public to the +internet, you should setup a bot protection to block excessively bot queries. +Bot protection requires a redis DB. About bot protection visit the official +SearXNG documentation and query for the word 'limiter'. +" + fi +} + +searxng.apache.install() { + rst_title "Install Apache site ${APACHE_SEARXNG_SITE}" + rst_para "\ +This installs SearXNG's uWSGI app as apache site. The apache site is located at: +${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}." + searxng.install.http.pre + + if ! apache_is_installed; then + err_msg "Apache packages are not installed" + if ! ask_yn "Do you really want to continue and install apache packages?" Yn; then + return + else + FORCE_SELECTION=Y install_apache + fi + else + info_msg "Apache packages are installed [OK]" + fi + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + apache_install_site --variant=socket "${APACHE_SEARXNG_SITE}" + else + apache_install_site "${APACHE_SEARXNG_SITE}" + fi + + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + fi +} + +searxng.apache.remove() { + rst_title "Remove Apache site ${APACHE_SEARXNG_SITE}" + rst_para "\ +This removes apache site ${APACHE_SEARXNG_SITE}:: + + ${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}" + + ! apache_is_installed && err_msg "Apache is not installed." + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + apache_remove_site "${APACHE_SEARXNG_SITE}" +} + +searxng.nginx.install() { + + rst_title "Install nginx site ${NGINX_SEARXNG_SITE}" + rst_para "\ +This installs SearXNG's uWSGI app as Nginx site. The Nginx site is located at: +${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE} and requires a uWSGI." + searxng.install.http.pre + + if ! nginx_is_installed ; then + err_msg "Nginx packages are not installed" + if ! ask_yn "Do you really want to continue and install Nginx packages?" Yn; then + return + else + FORCE_SELECTION=Y install_nginx + fi + else + info_msg "Nginx packages are installed [OK]" + fi + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + nginx_install_app --variant=socket "${NGINX_SEARXNG_SITE}" + else + nginx_install_app "${NGINX_SEARXNG_SITE}" + fi + + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + fi +} + +searxng.nginx.remove() { + rst_title "Remove Nginx site ${NGINX_SEARXNG_SITE}" + rst_para "\ +This removes Nginx site ${NGINX_SEARXNG_SITE}:: + + ${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}" + + ! nginx_is_installed && err_msg "Nginx is not installed." + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + nginx_remove_app "${NGINX_SEARXNG_SITE}" +} + +searxng.instance.exec() { + if ! service_account_is_available "${SERVICE_USER}"; then + die 42 "can't execute: instance does not exists (missed account ${SERVICE_USER})" + fi + sudo -H -i -u "${SERVICE_USER}" \ + SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET}" \ + "$@" +} + +searxng.instance.self.call() { + # wrapper to call a function in instance's environment + info_msg "wrapper: utils/searxng.sh instance _call $*" + searxng.instance.exec "${SEARXNG_SRC}/utils/searxng.sh" instance _call "$@" +} + +searxng.instance.get_setting() { + searxng.instance.exec python <<EOF +from searx import get_setting +print(get_setting('$1')) +EOF +} + +searxng.instance.debug.on() { + warn_msg "Do not enable debug in a production environment!" + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +sed -i -e "s/debug: false/debug: true/g" "$SEARXNG_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARXNG_UWSGI_APP" +} + +searxng.instance.debug.off() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +sed -i -e "s/debug: true/debug: false/g" "$SEARXNG_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARXNG_UWSGI_APP" +} + +searxng.instance.inspect() { + rst_title "Inspect SearXNG instance" + echo + + searxng.instance.self.call _searxng.instance.inspect + + local _debug_on + if ask_yn "Enable SearXNG debug mode?"; then + searxng.instance.debug.on + _debug_on=1 + fi + echo + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # For uWSGI debian uses the LSB init process; for each configuration + # file new uWSGI daemon instance is started with additional option. + service uwsgi status "${SERVICE_NAME}" + ;; + arch-*) + systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}" + ;; + fedora-*) + systemctl --no-pager -l status uwsgi + ;; + esac + + echo -e "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + + while true; do + trap break 2 + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) tail -f "/var/log/uwsgi/app/${SERVICE_NAME%.*}.log" ;; + arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;; + fedora-*) journalctl -f -u uwsgi ;; + esac + done + + if [[ $_debug_on == 1 ]]; then + searxng.instance.debug.off + fi + return 0 +} + +_searxng.instance.inspect() { + searxng.instance.env + + if in_container; then + # shellcheck source=utils/lxc-searxng.env + source "${REPO_ROOT}/utils/lxc-searxng.env" + lxc_suite_info + fi + + MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue${_creset}" + + if ! searxng.uwsgi.available; then + err_msg "SearXNG's uWSGI app not available" + wait_key + fi + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + wait_key + fi +} + +searxng.doc.rst() { + local debian="${SEARXNG_PACKAGES_debian}" + local arch="${SEARXNG_PACKAGES_arch}" + local fedora="${SEARXNG_PACKAGES_fedora}" + local debian_build="${SEARXNG_BUILD_PACKAGES_debian}" + local arch_build="${SEARXNG_BUILD_PACKAGES_arch}" + local fedora_build="${SEARXNG_BUILD_PACKAGES_fedora}" + debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + uwsgi_variant=':socket' + else + uwsgi_variant=':socket' + fi + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searxng.rst")\"" + + # I use ubuntu-20.04 here to demonstrate that versions are also suported, + # normaly debian-* and ubuntu-* are most the same. + + for DIST_NAME in ubuntu-20.04 arch fedora; do + ( + DIST_ID=${DIST_NAME%-*} + DIST_VERS=${DIST_NAME#*-} + [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + uWSGI_distro_setup + + echo -e "\n.. START searxng uwsgi-description $DIST_NAME" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) cat <<EOF + +.. code:: bash + + # init.d --> /usr/share/doc/uwsgi/README.Debian.gz + # For uWSGI debian uses the LSB init process, this might be changed + # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + + create ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} + enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ + start: sudo -H service uwsgi start ${SEARXNG_UWSGI_APP%.*} + restart: sudo -H service uwsgi restart ${SEARXNG_UWSGI_APP%.*} + stop: sudo -H service uwsgi stop ${SEARXNG_UWSGI_APP%.*} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + +EOF + ;; + arch-*) cat <<EOF + +.. code:: bash + + # systemd --> /usr/lib/systemd/system/uwsgi@.service + # For uWSGI archlinux uses systemd template units, see + # - http://0pointer.de/blog/projects/instances.html + # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + + create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + enable: sudo -H systemctl enable uwsgi@${SEARXNG_UWSGI_APP%.*} + start: sudo -H systemctl start uwsgi@${SEARXNG_UWSGI_APP%.*} + restart: sudo -H systemctl restart uwsgi@${SEARXNG_UWSGI_APP%.*} + stop: sudo -H systemctl stop uwsgi@${SEARXNG_UWSGI_APP%.*} + disable: sudo -H systemctl disable uwsgi@${SEARXNG_UWSGI_APP%.*} + +EOF + ;; + fedora-*|centos-7) cat <<EOF + +.. code:: bash + + # systemd --> /usr/lib/systemd/system/uwsgi.service + # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see + # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + + create: ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARXNG_UWSGI_APP} + +EOF + ;; + esac + echo -e ".. END searxng uwsgi-description $DIST_NAME" + + local _show_cursor="" # prevent from prefix_stdout's trailing show-cursor + + echo -e "\n.. START searxng uwsgi-appini $DIST_NAME" + echo ".. code:: bash" + echo + eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARXNG_UWSGI_APP}${uwsgi_variant}")\"" | prefix_stdout " " + echo -e "\n.. END searxng uwsgi-appini $DIST_NAME" + + echo -e "\n.. START nginx socket" + echo ".. code:: nginx" + echo + eval "echo \"$(< "${TEMPLATES}/${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}:socket")\"" | prefix_stdout " " + echo -e "\n.. END nginx socket" + + echo -e "\n.. START nginx http" + echo ".. code:: nginx" + echo + eval "echo \"$(< "${TEMPLATES}/${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}")\"" | prefix_stdout " " + echo -e "\n.. END nginx http" + + echo -e "\n.. START apache socket" + echo ".. code:: apache" + echo + eval "echo \"$(< "${TEMPLATES}/${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}:socket")\"" | prefix_stdout " " + echo -e "\n.. END apache socket" + + echo -e "\n.. START apache http" + echo ".. code:: apache" + echo + eval "echo \"$(< "${TEMPLATES}/${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}")\"" | prefix_stdout " " + echo -e "\n.. END apache http" + ) + done + +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searxng_check.py b/utils/searxng_check.py index afd0c4056..39e774340 100644 --- a/utils/searxng_check.py +++ b/utils/searxng_check.py @@ -25,3 +25,10 @@ if os.path.isfile(OLD_SETTING): os.environ.get('SEARXNG_SETTINGS_PATH', '/etc/searxng/settings.yml') )) warnings.warn(msg, DeprecationWarning) + +from searx.shared import redisdb +from searx import get_setting + +if not redisdb.init(): + warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2) + warnings.warn("--> no bot protection without redis DB", RuntimeWarning, stacklevel=2) diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json deleted file mode 100644 index fff70fa8f..000000000 --- a/utils/templates/etc/filtron/rules.json +++ /dev/null @@ -1,129 +0,0 @@ -[ - { - "name": "roboagent limit", - "filters": [ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)" - ], - "limit": 0, - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "botlimit", - "filters": [ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "limit": 0, - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "suspiciously frequent IP", - "filters": [], - "interval": 600, - "limit": 30, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions":[ - {"name":"log"} - ] - }, - { - "name": "search request", - "filters": [ - "Param:q", - "Path=^(/|/search)$" - ], - "interval": 61, - "limit": 999, - "subrules": [ - { - "name": "missing Accept-Language", - "filters": ["!Header:Accept-Language"], - "limit": 0, - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "suspiciously Connection=close header", - "filters": ["Header:Connection=close"], - "limit": 0, - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "IP limit", - "interval": 61, - "limit": 9, - "stop": true, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "rss/json limit", - "filters": [ - "Param:format=(csv|json|rss)" - ], - "interval": 121, - "limit": 2, - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "useragent limit", - "interval": 61, - "limit": 199, - "aggregations": [ - "Header:User-Agent" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - } - ] - } -] diff --git a/utils/templates/etc/httpd/sites-available/morty.conf b/utils/templates/etc/httpd/sites-available/morty.conf deleted file mode 100644 index daeb3635a..000000000 --- a/utils/templates/etc/httpd/sites-available/morty.conf +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8; mode: apache -*- - -LoadModule headers_module ${APACHE_MODULES}/mod_headers.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so -#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so - -# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog -# CustomLog /dev/null combined env=dontlog - -<Location ${PUBLIC_URL_PATH_MORTY} > - - <IfModule mod_security2.c> - SecRuleEngine Off - </IfModule> - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://${MORTY_LISTEN} - RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} - -</Location> diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf b/utils/templates/etc/httpd/sites-available/searxng.conf new file mode 100644 index 000000000..5278640c3 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searxng.conf @@ -0,0 +1,41 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so +# +# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARXNG_URL_PATH}> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + # add the trailing slash + RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/ + + ProxyPreserveHost On + ProxyPass http://${SEARXNG_INTERNAL_HTTP} + + # see flaskfix.py + RequestHeader set X-Scheme %{REQUEST_SCHEME}s + RequestHeader set X-Script-Name ${SEARXNG_URL_PATH} + + # see limiter.py + RequestHeader set X-Real-IP %{REMOTE_ADDR}s + RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s + +</Location> + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:filtron b/utils/templates/etc/httpd/sites-available/searxng.conf:filtron deleted file mode 100644 index 379d47e24..000000000 --- a/utils/templates/etc/httpd/sites-available/searxng.conf:filtron +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8; mode: apache -*- - -LoadModule headers_module ${APACHE_MODULES}/mod_headers.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so -#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so - -# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog -# CustomLog /dev/null combined env=dontlog - -# SecRuleRemoveById 981054 -# SecRuleRemoveById 981059 -# SecRuleRemoveById 981060 -# SecRuleRemoveById 950907 - -<Location ${FILTRON_URL_PATH} > - - <IfModule mod_security2.c> - SecRuleEngine Off - </IfModule> - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://${FILTRON_LISTEN} - RequestHeader set X-Script-Name ${FILTRON_URL_PATH} - -</Location> diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:socket b/utils/templates/etc/httpd/sites-available/searxng.conf:socket new file mode 100644 index 000000000..b55ea7560 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searxng.conf:socket @@ -0,0 +1,41 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so +# +# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARXNG_URL_PATH}> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + # add the trailing slash + RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/ + + ProxyPreserveHost On + ProxyPass unix:${SEARXNG_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searxng/ + + # see flaskfix.py + RequestHeader set X-Scheme %{REQUEST_SCHEME}s + RequestHeader set X-Script-Name ${SEARXNG_URL_PATH} + + # see limiter.py + RequestHeader set X-Real-IP %{REMOTE_ADDR}s + RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s + +</Location> + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi b/utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi deleted file mode 100644 index aabc125ae..000000000 --- a/utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8; mode: apache -*- - -LoadModule headers_module ${APACHE_MODULES}/mod_headers.so -LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so -LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so -# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so - -# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog -# CustomLog /dev/null combined env=dontlog - -<Location ${SEARXNG_URL_PATH}> - - <IfModule mod_security2.c> - SecRuleEngine Off - </IfModule> - - Require all granted - - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/ - -</Location> diff --git a/utils/templates/etc/nginx/default.apps-available/morty.conf b/utils/templates/etc/nginx/default.apps-available/morty.conf deleted file mode 100644 index 51f083985..000000000 --- a/utils/templates/etc/nginx/default.apps-available/morty.conf +++ /dev/null @@ -1,11 +0,0 @@ -# https://example.org/morty - -location /morty { - proxy_pass http://127.0.0.1:3000/; - - proxy_set_header Host \$host; - proxy_set_header Connection \$http_connection; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Scheme \$scheme; -} diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron b/utils/templates/etc/nginx/default.apps-available/searxng.conf index e25461c47..7225a8f96 100644 --- a/utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf @@ -1,16 +1,29 @@ -# https://example.org/searx - location ${SEARXNG_URL_PATH} { - proxy_pass http://127.0.0.1:4004/; + + proxy_pass http://${SEARXNG_INTERNAL_HTTP}; proxy_set_header Host \$host; proxy_set_header Connection \$http_connection; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + + # see flaskfix.py proxy_set_header X-Scheme \$scheme; proxy_set_header X-Script-Name ${SEARXNG_URL_PATH}; -} -location ${SEARXNG_URL_PATH}/static/ { - alias ${SEARX_SRC}/searx/static/; + # see limiter.py + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + + # proxy_buffering off; + # proxy_request_buffering off; + # proxy_buffer_size 8k; + } + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# location ${SEARXNG_URL_PATH}/static/ { +# alias ${SEARXNG_STATIC}/; +# } diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket new file mode 100644 index 000000000..7a74eab48 --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket @@ -0,0 +1,26 @@ +location ${SEARXNG_URL_PATH} { + + uwsgi_pass unix://${SEARXNG_UWSGI_SOCKET}; + + include uwsgi_params; + + uwsgi_param HTTP_HOST \$host; + uwsgi_param HTTP_CONNECTION \$http_connection; + + # see flaskfix.py + uwsgi_param HTTP_X_SCHEME \$scheme; + uwsgi_param HTTP_X_SCRIPT_NAME ${SEARXNG_URL_PATH}; + + # see limiter.py + uwsgi_param HTTP_X_REAL_IP \$remote_addr; + uwsgi_param HTTP_X_FORWARDED_FOR \$proxy_add_x_forwarded_for; +} + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# location ${SEARXNG_URL_PATH}/static/ { +# alias ${SEARXNG_STATIC}/; +# } diff --git a/utils/templates/etc/searxng/settings.yml b/utils/templates/etc/searxng/settings.yml index 860f4f5e9..aee21474d 100644 --- a/utils/templates/etc/searxng/settings.yml +++ b/utils/templates/etc/searxng/settings.yml @@ -1,46 +1,55 @@ -# SearXNG settings, before editing this file read: -# -# https://docs.searxng.org/admin/engines/settings.html +# SearXNG settings use_default_settings: true general: - # Debug mode, only for development debug: false - # change displayed name - # instance_name: "SearXNG" + instance_name: "SearXNG" search: - # Filter results. 0: None, 1: Moderate, 2: Strict - safe_search: 0 - # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", - # "startpage", "swisscows", "qwant", "wikipedia" - leave blank to turn it off - # by default. - autocomplete: '' - # Default search language - leave blank to detect from browser information or - # use codes from 'languages.py' - default_lang: '' - # remove format to deny access, use lower case. - formats: - - html + safe_search: 2 + autocomplete: 'duckduckgo' server: - secret_key: "ultrasecretkey" # change this! - # Proxying image results through SearXNG - image_proxy: false + secret_key: "ultrasecretkey" + limiter: true + image_proxy: true + +redis: + url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 + +ui: + static_use_hash: true -# result_proxy: -# url: http://127.0.0.1:3000/ -# key: !!binary "your_morty_proxy_key" +# preferences: +# lock: +# - autocomplete +# - method + +enabled_plugins: + - 'Hash plugin' + - 'Search on category select' + - 'Self Informations' + - 'Tracker URL remover' + - 'Ahmia blacklist' + # - 'Hostname replace' # see hostname_replace configuration below + # - 'Infinite scroll' + # - 'Open Access DOI rewrite' + # - 'Vim-like hotkeys' # plugins: # - only_show_green_results -# engines: -# -# - name: duckduckgo -# disabled: false +# hostname_replace: # +# # twitter --> nitter +# '(www\.)?twitter\.com$': 'nitter.net' + +engines: + + - name: google + use_mobile_ui: true + # - name: fdroid # disabled: false # @@ -48,6 +57,13 @@ server: # disabled: false # # - name: mediathekviewweb -# engine: mediathekviewweb -# shortcut: mvw -# categories: general +# categories: TV +# disabled: false +# +# - name: invidious +# disabled: false +# base_url: +# - https://invidious.snopyta.org +# - https://invidious.tiekoetter.com +# - https://invidio.xamh.de +# - https://inv.riverside.rocks diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini index aaf55a807..04c32c662 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini @@ -16,7 +16,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -57,37 +57,27 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html -http = ${SEARX_INTERNAL_HTTP} +http = ${SEARXNG_INTERNAL_HTTP} -# using unix-sockets: +# uWSGI serves the static files and in settings.yml we use:: # -# On some distributions you need to create the app folder for the sockets:: +# ui: +# static_use_hash: true # -# mkdir -p ${SEARX_UWSGI_SOCKET} -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} -# -# socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket index e8facda86..bbfaf63be 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket @@ -16,7 +16,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -57,37 +57,24 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. - -# using IP: -# -# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http -# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html -# http = ${SEARX_INTERNAL_HTTP} +socket = ${SEARXNG_UWSGI_SOCKET} -# using unix-sockets: +# uWSGI serves the static files and in settings.yml we use:: # -# On some distributions you need to create the app folder for the sockets:: +# ui: +# static_use_hash: true # -# mkdir -p ${SEARX_UWSGI_SOCKET} -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} -# -socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini b/utils/templates/etc/uwsgi/apps-available/searxng.ini index 9dad84c16..5ea7d991a 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini @@ -6,7 +6,11 @@ # # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core -# Who will run the code +# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be +# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini). +# +# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting +# uid = ${SERVICE_USER} gid = ${SERVICE_GROUP} @@ -16,7 +20,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -56,37 +60,27 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html -http = ${SEARX_INTERNAL_HTTP} +http = ${SEARXNG_INTERNAL_HTTP} -# using unix-sockets: -# -# On some distributions you need to create the app folder for the sockets:: +# uWSGI serves the static files and in settings.yml we use:: # -# mkdir -p /run/uwsgi/app/searxng -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} +# ui: +# static_use_hash: true # -# socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket index c76d084e1..304ea3500 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket @@ -6,7 +6,11 @@ # # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core -# Who will run the code +# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be +# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini). +# +# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting +# uid = ${SERVICE_USER} gid = ${SERVICE_GROUP} @@ -16,7 +20,7 @@ env = LANGUAGE=C.UTF-8 env = LC_ALL=C.UTF-8 # chdir to specified directory before apps loading -chdir = ${SEARX_SRC}/searx +chdir = ${SEARXNG_SRC}/searx # SearXNG configuration (settings.yml) env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH} @@ -56,37 +60,24 @@ enable-threads = true module = searx.webapp # set PYTHONHOME/virtualenv -virtualenv = ${SEARX_PYENV} +virtualenv = ${SEARXNG_PYENV} # add directory (or glob) to pythonpath -pythonpath = ${SEARX_SRC} +pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# -# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http -# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html +socket = ${SEARXNG_UWSGI_SOCKET} -# http = ${SEARX_INTERNAL_HTTP} - -# using unix-sockets: -# -# On some distributions you need to create the app folder for the sockets:: +# uWSGI serves the static files and in settings.yml we use:: # -# mkdir -p ${SEARX_UWSGI_SOCKET} -# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET} +# ui: +# static_use_hash: true # -socket = ${SEARX_UWSGI_SOCKET} - -# uwsgi serves the static files +static-map = /static=${SEARXNG_STATIC} # expires set to one year since there are hashes -static-map = /static=${SEARX_SRC}/searx/static static-expires = /* 31557600 static-gzip-all = True offload-threads = %k |