Merge branch 'master' into libgen

5 files changed, 622 insertions, 0 deletions

diff --git a/docs/admin/api.rst b/docs/admin/api.rst
new file mode 100644
index 000000000..7804a8664
--- /dev/null
+++ b/docs/admin/api.rst
@@ -0,0 +1,96 @@
+.. _adminapi:
+
+==================
+Administration API
+==================
+
+Get configuration data
+======================
+
+.. code:: http
+
+    GET /config  HTTP/1.1
+
+Sample response
+---------------
+
+.. code:: json
+
+   {
+     "autocomplete": "",
+     "categories": [
+       "map",
+       "it",
+       "images",
+     ],
+     "default_locale": "",
+     "default_theme": "oscar",
+     "engines": [
+       {
+         "categories": [
+           "map"
+         ],
+         "enabled": true,
+         "name": "openstreetmap",
+         "shortcut": "osm"
+       },
+       {
+         "categories": [
+           "it"
+         ],
+         "enabled": true,
+         "name": "arch linux wiki",
+         "shortcut": "al"
+       },
+       {
+         "categories": [
+           "images"
+         ],
+         "enabled": true,
+         "name": "google images",
+         "shortcut": "goi"
+       },
+       {
+         "categories": [
+           "it"
+         ],
+         "enabled": false,
+         "name": "bitbucket",
+         "shortcut": "bb"
+       },
+     ],
+     "instance_name": "searx",
+     "locales": {
+       "de": "Deutsch (German)",
+       "en": "English",
+       "eo": "Esperanto (Esperanto)",
+     },
+     "plugins": [
+       {
+         "enabled": true,
+         "name": "HTTPS rewrite"
+       },
+       {
+         "enabled": false,
+         "name": "Vim-like hotkeys"
+       }
+     ],
+     "safe_search": 0
+   }
+
+
+Embed search bar
+================
+
+The search bar can be embedded into websites.  Just paste the example into the
+HTML of the site.  URL of the searx instance and values are customizable.
+
+.. code:: html
+
+   <form method="post" action="https://searx.me/">
+     <!-- search      --> <input type="text" name="q" />
+     <!-- categories  --> <input type="hidden" name="categories" value="general,social media" />
+     <!-- language    --> <input type="hidden" name="lang" value="all" />
+     <!-- locale      --> <input type="hidden" name="locale" value="en" />
+     <!-- date filter --> <input type="hidden" name="time_range" value="month" />
+   </form>
diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst
new file mode 100644
index 000000000..07dcb9bc5
--- /dev/null
+++ b/docs/admin/filtron.rst
@@ -0,0 +1,148 @@
+==========================
+How to protect an instance
+==========================
+
+Searx depens on external search services.  To avoid the abuse of these services
+it is advised to limit the number of requests processed by searx.
+
+An application firewall, ``filtron`` solves exactly this problem.  Information
+on how to install it can be found at the `project page of filtron
+<https://github.com/asciimoo/filtron>`__.
+
+
+Sample configuration of filtron
+===============================
+
+An example configuration can be find below. This configuration limits the access
+of:
+
+- scripts or applications (roboagent limit)
+- webcrawlers (botlimit)
+- IPs which send too many requests (IP limit)
+- too many json, csv, etc. requests (rss/json limit)
+- the same UserAgent of if too many requests (useragent limit)
+
+.. code:: json
+
+   [{
+      "name":"search request",
+      "filters":[
+         "Param:q",
+         "Path=^(/|/search)$"
+      ],
+      "interval":"<time-interval-in-sec (int)>",
+      "limit":"<max-request-number-in-interval (int)>",
+      "subrules":[
+         {
+            "name":"roboagent limit",
+            "interval":"<time-interval-in-sec (int)>",
+            "limit":"<max-request-number-in-interval (int)>",
+            "filters":[
+               "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
+            ],
+            "actions":[
+               {
+                  "name":"block",
+                  "params":{
+                     "message":"Rate limit exceeded"
+                  }
+               }
+            ]
+         },
+         {
+            "name":"botlimit",
+            "limit":0,
+            "stop":true,
+            "filters":[
+               "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
+            ],
+            "actions":[
+               {
+                  "name":"block",
+                  "params":{
+                     "message":"Rate limit exceeded"
+                  }
+               }
+            ]
+         },
+         {
+            "name":"IP limit",
+            "interval":"<time-interval-in-sec (int)>",
+            "limit":"<max-request-number-in-interval (int)>",
+            "stop":true,
+            "aggregations":[
+               "Header:X-Forwarded-For"
+            ],
+            "actions":[
+               {
+                  "name":"block",
+                  "params":{
+                     "message":"Rate limit exceeded"
+                  }
+               }
+            ]
+         },
+         {
+            "name":"rss/json limit",
+            "interval":"<time-interval-in-sec (int)>",
+            "limit":"<max-request-number-in-interval (int)>",
+            "stop":true,
+            "filters":[
+               "Param:format=(csv|json|rss)"
+            ],
+            "actions":[
+               {
+                  "name":"block",
+                  "params":{
+                     "message":"Rate limit exceeded"
+                  }
+               }
+            ]
+         },
+         {
+            "name":"useragent limit",
+            "interval":"<time-interval-in-sec (int)>",
+            "limit":"<max-request-number-in-interval (int)>",
+            "aggregations":[
+               "Header:User-Agent"
+            ],
+            "actions":[
+               {
+                  "name":"block",
+                  "params":{
+                     "message":"Rate limit exceeded"
+                  }
+               }
+            ]
+         }
+      ]
+   }]
+
+
+
+Route request through filtron
+=============================
+
+Filtron can be started using the following command:
+
+.. code:: sh
+
+   $ filtron -rules rules.json
+
+It listens on ``127.0.0.1:4004`` and forwards filtered requests to
+``127.0.0.1:8888`` by default.
+
+Use it along with ``nginx`` with the following example configuration.
+
+.. code:: nginx
+
+   location / {
+        proxy_set_header   Host    $http_host;
+        proxy_set_header   X-Real-IP $remote_addr;
+        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header   X-Scheme $scheme;
+        proxy_pass         http://127.0.0.1:4004/;
+   }
+
+Requests are coming from port 4004 going through filtron and then forwarded to
+port 8888 where a searx is being run.
diff --git a/docs/admin/index.rst b/docs/admin/index.rst
new file mode 100644
index 000000000..f3a995769
--- /dev/null
+++ b/docs/admin/index.rst
@@ -0,0 +1,11 @@
+===========================
+Administrator documentation
+===========================
+
+.. toctree::
+   :maxdepth: 1
+
+   installation
+   api
+   filtron
+   morty
diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst
new file mode 100644
index 000000000..239ce0704
--- /dev/null
+++ b/docs/admin/installation.rst
@@ -0,0 +1,341 @@
+.. _installation:
+
+============
+Installation
+============
+
+.. contents::
+   :depth: 3
+
+Basic installation
+==================
+
+Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure
+to have enable universe repository.
+
+Install packages:
+
+.. code:: sh
+
+    $ sudo -H apt-get install \
+           git build-essential libxslt-dev \
+	   python-dev python-virtualenv python-babel \
+	   zlib1g-dev libffi-dev libssl-dev
+
+Install searx:
+
+.. code:: sh
+
+    cd /usr/local
+    sudo -H git clone https://github.com/asciimoo/searx.git
+    sudo -H useradd searx -d /usr/local/searx
+    sudo -H chown searx:searx -R /usr/local/searx
+
+Install dependencies in a virtualenv:
+
+.. code:: sh
+
+    cd /usr/local/searx
+    sudo -H -u searx -i
+
+.. code:: sh
+
+    (searx)$ virtualenv searx-ve
+    (searx)$ . ./searx-ve/bin/activate
+    (searx)$ ./manage.sh update_packages
+
+Configuration
+==============
+
+.. code:: sh
+
+    sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml
+
+Edit searx/settings.yml if necessary.
+
+Check
+=====
+
+Start searx:
+
+.. code:: sh
+
+    python searx/webapp.py
+
+Go to http://localhost:8888
+
+If everything works fine, disable the debug option in settings.yml:
+
+.. code:: sh
+
+    sed -i -e "s/debug : True/debug : False/g" searx/settings.yml
+
+At this point searx is not demonized ; uwsgi allows this.
+
+You can exit the virtualenv and the searx user bash (enter exit command
+twice).
+
+uwsgi
+=====
+
+Install packages:
+
+.. code:: sh
+
+    sudo -H apt-get install \
+         uwsgi uwsgi-plugin-python
+
+Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this
+content:
+
+.. code:: ini
+
+    [uwsgi]
+    # Who will run the code
+    uid = searx
+    gid = searx
+
+    # disable logging for privacy
+    disable-logging = true
+
+    # Number of workers (usually CPU count)
+    workers = 4
+
+    # The right granted on the created socket
+    chmod-socket = 666
+
+    # Plugin to use and interpretor config
+    single-interpreter = true
+    master = true
+    plugin = python
+    lazy-apps = true
+    enable-threads = true
+
+    # Module to import
+    module = searx.webapp
+
+    # Virtualenv and python path
+    virtualenv = /usr/local/searx/searx-ve/
+    pythonpath = /usr/local/searx/
+    chdir = /usr/local/searx/searx/
+
+Activate the uwsgi application and restart:
+
+.. code:: sh
+
+    cd /etc/uwsgi/apps-enabled
+    ln -s ../apps-available/searx.ini
+    /etc/init.d/uwsgi restart
+
+Web server
+==========
+
+with nginx
+----------
+
+If nginx is not installed (uwsgi will not work with the package
+nginx-light):
+
+.. code:: sh
+
+    sudo -H apt-get install nginx
+
+Hosted at /
+~~~~~~~~~~~
+
+Create the configuration file ``/etc/nginx/sites-available/searx`` with this
+content:
+
+.. code:: nginx
+
+    server {
+        listen 80;
+        server_name searx.example.com;
+        root /usr/local/searx;
+
+        location / {
+                include uwsgi_params;
+                uwsgi_pass unix:/run/uwsgi/app/searx/socket;
+        }
+    }
+
+Create a symlink to sites-enabled:
+
+.. code:: sh
+
+   sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx
+
+Restart service:
+
+.. code:: sh
+
+    sudo -H service nginx restart
+    sudo -H service uwsgi restart
+
+from subdirectory URL (/searx)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Add this configuration in the server config file
+``/etc/nginx/sites-enabled/default``:
+
+.. code:: nginx
+
+    location = /searx { rewrite ^ /searx/; }
+    location /searx {
+            try_files $uri @searx;
+    }
+    location @searx {
+            uwsgi_param SCRIPT_NAME /searx;
+            include uwsgi_params;
+            uwsgi_modifier1 30;
+            uwsgi_pass unix:/run/uwsgi/app/searx/socket;
+    }
+
+
+**OR** using reverse proxy (Please, note that reverse proxy advised to be used
+in case of single-user or low-traffic instances.)
+
+.. code:: nginx
+
+    location /searx {
+        proxy_pass http://127.0.0.1:8888;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Scheme $scheme;
+        proxy_set_header X-Script-Name /searx;
+        proxy_buffering off;
+    }
+
+
+Enable ``base_url`` in ``searx/settings.yml``
+
+.. code:: yaml
+
+    base_url : http://your.domain.tld/searx/
+
+Restart service:
+
+.. code:: sh
+
+    sudo -H service nginx restart
+    sudo -H service uwsgi restart
+
+disable logs
+^^^^^^^^^^^^
+
+for better privacy you can disable nginx logs about searx.
+
+how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default``
+add:
+
+.. code:: nginx
+
+    access_log /dev/null;
+    error_log /dev/null;
+
+Restart service:
+
+.. code:: sh
+
+    sudo -H service nginx restart
+
+with apache
+-----------
+
+Add wsgi mod:
+
+.. code:: sh
+
+    sudo -H apt-get install libapache2-mod-uwsgi
+    sudo -H a2enmod uwsgi
+
+Add this configuration in the file ``/etc/apache2/apache2.conf``:
+
+.. code:: apache
+
+    <Location />
+        Options FollowSymLinks Indexes
+        SetHandler uwsgi-handler
+        uWSGISocket /run/uwsgi/app/searx/socket
+    </Location>
+
+Note that if your instance of searx is not at the root, you should change
+``<Location />`` by the location of your instance, like ``<Location /searx>``.
+
+Restart Apache:
+
+.. code:: sh
+
+    sudo -H /etc/init.d/apache2 restart
+
+disable logs
+~~~~~~~~~~~~
+
+For better privacy you can disable Apache logs.
+
+.. warning::
+
+   You can only disable logs for the whole (virtual) server not for a specific
+   path.
+
+Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add:
+
+.. code:: apache
+
+    CustomLog /dev/null combined
+
+Restart Apache:
+
+.. code:: sh
+
+    sudo -H /etc/init.d/apache2 restart
+
+How to update
+=============
+
+.. code:: sh
+
+    cd /usr/local/searx
+    sudo -H -u searx -i
+
+.. code:: sh
+
+    (searx)$ . ./searx-ve/bin/activate
+    (searx)$ git stash
+    (searx)$ git pull origin master
+    (searx)$ git stash apply
+    (searx)$ ./manage.sh update_packages
+
+.. code:: sh
+
+    sudo -H service uwsgi restart
+
+Docker
+======
+
+Make sure you have installed Docker. For instance, you can deploy searx like this:
+
+.. code:: sh
+
+    docker pull wonderfall/searx
+    docker run -d --name searx -p $PORT:8888 wonderfall/searx
+
+Go to ``http://localhost:$PORT``.
+
+See https://hub.docker.com/r/wonderfall/searx/ for more informations.  It's also
+possible to build searx from the embedded Dockerfile.
+
+.. code:: sh
+
+   git clone https://github.com/asciimoo/searx.git
+   cd searx
+   docker build -t whatever/searx .
+
+References
+==========
+
+* https://about.okhin.fr/posts/Searx/ with some additions
+
+* How to: `Setup searx in a couple of hours with a free SSL certificate
+  <https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__
+
diff --git a/docs/admin/morty.rst b/docs/admin/morty.rst
new file mode 100644
index 000000000..7d7b34492
--- /dev/null
+++ b/docs/admin/morty.rst
@@ -0,0 +1,26 @@
+=========================
+How to setup result proxy
+=========================
+
+.. _morty: https://github.com/asciimoo/morty
+.. _morty's README: https://github.com/asciimoo/morty
+
+By default searx can only act as an image proxy for result images, but it is
+possible to proxify all the result URLs with an external service, morty_.
+
+To use this feature, morty has to be installed and activated in searx's
+``settings.yml``.
+
+Add the following snippet to your ``settings.yml`` and restart searx:
+
+.. code:: yaml
+
+    result_proxy:
+        url : http://127.0.0.1:3000/
+        key : your_morty_proxy_key
+
+``url``
+  Is the address of the running morty service.
+
+``key``
+  Is an optional argument, see `morty's README`_ for more information.