Merge branch 'master' into docker/opencontainers

8 files changed, 471 insertions, 7 deletions

diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot
new file mode 100644
index 000000000..a46c96de3
--- /dev/null
+++ b/docs/admin/arch_public.dot
@@ -0,0 +1,33 @@
+digraph G {
+
+  node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"];
+  edge [fontname="Sans"];
+
+  browser [label="Browser", shape=Mdiamond];
+  rp      [label="Reverse Proxy", href="url to configure reverse proxy"];
+  filtron [label="Filtron",       href="https://github.com/asciimoo/filtron"];
+  morty   [label="Morty",         href="https://github.com/asciimoo/morty"];
+  static  [label="Static files",  href="url to configure static files"];
+  uwsgi   [label="uwsgi",         href="url to configure uwsgi"]
+  searx1  [label="Searx #1"];
+  searx2  [label="Searx #2"];
+  searx3  [label="Searx #3"];
+  searx4  [label="Searx #4"];
+
+  browser -> rp [label="HTTPS"]
+
+  subgraph cluster_searx {
+      label = "Searx instance" fontname="Sans";
+      bgcolor="#fafafa";
+      { rank=same; static rp };
+      rp -> morty      [label="optional: images and HTML pages proxy"];
+      rp -> static     [label="optional: reverse proxy serves directly static files"];
+      rp -> filtron    [label="HTTP"];
+      filtron -> uwsgi [label="HTTP"];
+      uwsgi -> searx1;
+      uwsgi -> searx2;
+      uwsgi -> searx3;
+      uwsgi -> searx4;
+  }
+
+}
diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst
new file mode 100644
index 000000000..7064a294b
--- /dev/null
+++ b/docs/admin/architecture.rst
@@ -0,0 +1,24 @@
+.. _architecture:
+
+============
+Architecture
+============
+
+.. sidebar:: Needs work!
+
+   This article needs some work / Searx is a collaborative effort.  If you have
+   any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see
+   :ref:`how to contribute`.
+
+Herein you will find some hints and suggestions about typical architectures of
+searx infrastructures.
+
+We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
+It shows a *reference* setup for public searx instances.
+
+.. _arch public:
+
+.. kernel-figure:: arch_public.dot
+   :alt: arch_public.dot
+
+   Reference architecture of a public searx setup.
diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst
new file mode 100644
index 000000000..5260da033
--- /dev/null
+++ b/docs/admin/buildhosts.rst
@@ -0,0 +1,103 @@
+.. _buildhosts:
+
+==========
+Buildhosts
+==========
+
+.. sidebar:: This article needs some work
+
+   If you have any contribution send us your :pull:`PR <../pulls>`, see
+   :ref:`how to contribute`.
+
+To get best results from build, its recommend to install additional packages
+on build hosts.
+
+.. _docs build:
+
+Build docs
+==========
+
+.. _Graphviz: https://graphviz.gitlab.io
+.. _ImageMagick: https://www.imagemagick.org
+.. _XeTeX: https://tug.org/xetex/
+.. _dvisvgm: https://dvisvgm.de/
+
+.. sidebar:: Sphinx build needs
+
+   - ImageMagick_
+   - Graphviz_
+   - XeTeX_
+   - dvisvgm_
+
+Most of the sphinx requirements are installed from :origin:`setup.py` and the
+docs can be build from scratch with ``make docs``.  For better math and image
+processing additional packages are needed.  The XeTeX_ needed not only for PDF
+creation, its also needed for :ref:`math` when HTML output is build.
+
+To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
+as images (``sphinx.ext.imgmath`` extension).  If your docs build (``make
+docs``) shows warnings like this::
+
+   WARNING: dot(1) not found, for better output quality install \
+            graphviz from http://www.graphviz.org
+   ..
+   WARNING: LaTeX command 'latex' cannot be run (needed for math \
+            display), check the imgmath_latex setting
+
+you need to install additional packages on your build host, to get better HTML
+output.
+
+.. _system requirements:
+
+.. tabs::
+
+   .. group-tab:: Ubuntu / debian
+
+      .. code-block:: sh
+
+         $ sudo apt install graphviz imagemagick texlive-xetex librsvg2-bin
+
+   .. group-tab:: Arch Linux
+
+      .. code-block:: sh
+
+         $ sudo pacman -S graphviz imagemagick texlive-bin extra/librsvg
+
+   .. group-tab::  Fedora / RHEL
+
+      .. code-block:: sh
+
+         $ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools
+
+
+For PDF output you also need:
+
+.. tabs::
+
+   .. group-tab:: Ubuntu / debian
+
+      .. code:: sh
+
+         $ sudo apt texlive-latex-recommended texlive-extra-utils ttf-dejavu
+
+   .. group-tab:: Arch Linux
+
+      .. code:: sh
+
+      	 $ sudo pacman -S texlive-core texlive-latexextra ttf-dejavu
+
+   .. group-tab::  Fedora / RHEL
+
+      .. code:: sh
+
+      	 $ sudo dnf install \
+	        texlive-collection-fontsrecommended texlive-collection-latex \
+		dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
+
+.. _system requirements END:
+
+.. literalinclude:: ../conf.py
+   :language: python
+   :start-after: # sphinx.ext.imgmath setup
+   :end-before: # sphinx.ext.imgmath setup END
+
diff --git a/docs/admin/engines.rst b/docs/admin/engines.rst
new file mode 100644
index 000000000..4b78c8c2b
--- /dev/null
+++ b/docs/admin/engines.rst
@@ -0,0 +1,71 @@
+.. _engines generic:
+
+=======
+Engines
+=======
+
+.. sidebar:: Further reading ..
+
+   - :ref:`settings engine`
+   - :ref:`engine settings`
+   - :ref:`engine file`
+
+============= =========== ==================== ============
+:ref:`engine settings`    :ref:`engine file`
+------------------------- ---------------------------------
+Name (cfg)                Categories
+------------------------- ---------------------------------
+Engine        ..          Paging support       **P**
+------------------------- -------------------- ------------
+Shortcut      **S**       Language support     **L**
+Timeout       **TO**      Time range support   **TR**
+Disabled      **D**       Offline              **O**
+------------- ----------- -------------------- ------------
+Safe search   **SS**
+------------- ----------- ---------------------------------
+Weigth        **W**
+------------- ----------- ---------------------------------
+Disabled      **D**
+============= =========== =================================
+
+Configuration defaults (at built time):
+
+.. _configured engines:
+
+.. jinja:: webapp
+
+   .. flat-table:: Engines configured at built time (defaults)
+      :header-rows: 1
+      :stub-columns: 2
+
+      * - Name (cfg)
+        - S
+        - Engine
+        - TO
+        - Categories
+        - P
+        - L
+        - SS
+        - D
+        - TR
+        - O
+	- W
+	- D
+
+      {% for name, mod in engines.items() %}
+
+      * - {{name}}
+        - !{{mod.shortcut}}
+        - {{mod.__name__}}
+        - {{mod.timeout}}
+        - {{", ".join(mod.categories)}}
+        - {{(mod.paging and "y") or ""}}
+        - {{(mod.language_support and "y") or ""}}
+        - {{(mod.safesearch and "y") or ""}}
+        - {{(mod.disabled and "y") or ""}}
+        - {{(mod.time_range_support and "y") or ""}}
+        - {{(mod.offline and "y") or ""}}
+        - {{mod.weight or 1 }}
+        - {{(mod.disabled and "y") or ""}}
+
+     {% endfor %}
diff --git a/docs/admin/index.rst b/docs/admin/index.rst
index f3a995769..b3c7f5119 100644
--- a/docs/admin/index.rst
+++ b/docs/admin/index.rst
@@ -6,6 +6,11 @@ Administrator documentation
    :maxdepth: 1
 
    installation
+   settings
    api
+   architecture
    filtron
    morty
+   engines
+   plugins
+   buildhosts
diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst
index 239ce0704..15800fc01 100644
--- a/docs/admin/installation.rst
+++ b/docs/admin/installation.rst
@@ -114,6 +114,9 @@ content:
     # Module to import
     module = searx.webapp
 
+    # Support running the module from a webserver subdirectory.
+    route-run = fixpathinfo:
+
     # Virtualenv and python path
     virtualenv = /usr/local/searx/searx-ve/
     pythonpath = /usr/local/searx/
@@ -151,7 +154,10 @@ content:
     server {
         listen 80;
         server_name searx.example.com;
-        root /usr/local/searx;
+        root /usr/local/searx/searx;
+
+        location /static {
+        }
 
         location / {
                 include uwsgi_params;
@@ -180,14 +186,13 @@ Add this configuration in the server config file
 
 .. code:: nginx
 
-    location = /searx { rewrite ^ /searx/; }
-    location /searx {
-            try_files $uri @searx;
+    location /searx/static {
+            alias /usr/local/searx/searx/static;
     }
-    location @searx {
+
+    location /searx {
             uwsgi_param SCRIPT_NAME /searx;
             include uwsgi_params;
-            uwsgi_modifier1 30;
             uwsgi_pass unix:/run/uwsgi/app/searx/socket;
     }
 
@@ -197,6 +202,10 @@ in case of single-user or low-traffic instances.)
 
 .. code:: nginx
 
+    location /searx/static {
+            alias /usr/local/searx/searx/static;
+    }
+
     location /searx {
         proxy_pass http://127.0.0.1:8888;
         proxy_set_header Host $host;
@@ -338,4 +347,3 @@ References
 
 * How to: `Setup searx in a couple of hours with a free SSL certificate
   <https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__
-
diff --git a/docs/admin/plugins.rst b/docs/admin/plugins.rst
new file mode 100644
index 000000000..4ed9066fd
--- /dev/null
+++ b/docs/admin/plugins.rst
@@ -0,0 +1,39 @@
+.. _plugins generic:
+
+===============
+Plugins builtin
+===============
+
+.. sidebar:: Further reading ..
+
+   - :ref:`dev plugin`
+
+Configuration defaults (at built time):
+
+:DO: Default on
+
+.. _configured plugins:
+
+.. jinja:: webapp
+
+   .. flat-table:: Plugins configured at built time (defaults)
+      :header-rows: 1
+      :stub-columns: 1
+      :widths: 3 1 9
+
+      * - Name
+        - DO
+        - Description
+
+          JS & CSS dependencies
+
+      {% for plgin in plugins %}
+
+      * - {{plgin.name}}
+        - {{(plgin.default_on and "y") or ""}}
+        - {{plgin.description}}
+
+          {% for dep in (plgin.js_dependencies + plgin.css_dependencies) %}
+          | ``{{dep}}`` {% endfor %}
+
+      {% endfor %}
diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst
new file mode 100644
index 000000000..0bfdcc6cb
--- /dev/null
+++ b/docs/admin/settings.rst
@@ -0,0 +1,181 @@
+.. _settings.yml:
+
+================
+``settings.yml``
+================
+
+.. sidebar:: Further reading ..
+
+   - :ref:`search API`
+
+This page describe the options possibilities of the settings.yml file.
+
+.. _settings global:
+
+Global Settings
+===============
+
+.. code:: yaml
+
+   server:
+       port : 8888
+       secret_key : "ultrasecretkey" # change this!
+       debug : False                 # debug mode, only for development
+       request_timeout : 2.0         # seconds
+       base_url : False              # set custom base_url (or False)
+       themes_path : ""              # custom ui themes path
+       default_theme : oscar         # ui theme
+       useragent_suffix : ""         # suffix of searx_useragent, could contain
+                                     # informations like admins email address
+       image_proxy : False           # proxying image results through searx
+       default_locale : ""           # default interface locale
+
+   # uncomment below section if you want to use a proxy
+
+   #outgoing_proxies :
+   #    http : http://127.0.0.1:8080
+   #    https: http://127.0.0.1:8080
+
+   # uncomment below section only if you have more than one network interface
+   # which can be the source of outgoing search requests
+
+   #source_ips:
+   #  - 1.1.1.1
+   #  - 1.1.1.2
+
+   locales:
+       en : English
+       de : Deutsch
+       he : Hebrew
+       hu : Magyar
+       fr : Français
+       es : Español
+       it : Italiano
+       nl : Nederlands
+       ja : 日本語 (Japanese)
+       tr : Türkçe
+       ru : Russian
+       ro : Romanian
+
+
+``port`` :
+  Port number of the searx web application if you run it directly using ``python
+  searx/webapp.py``.  Doesn't apply to searx running on Apache or Nginx.
+
+``secret_key`` :
+  Used for cryptography purpose.
+
+``debug`` :
+  Allow a more detailed log if you run searx directly. Display *detailed* error
+  messages in the browser too, so this must be deactivated in production.
+
+``request_timeout`` :
+  Global timeout of the requests made to others engines in seconds. A bigger
+  timeout will allow to wait for answers from slow engines, but in consequence
+  will slow searx reactivity (the result page may take the time specified in the
+  timeout to load)
+
+``base_url`` :
+  The base URL where searx is deployed.  Used to create correct inbound links.
+
+``themes_path`` :
+  Path to where the themes are located.  If you didn't develop anything, leave it
+  blank.
+
+``default_theme`` :
+  Name of the theme you want to use by default on you searx instance.
+
+``useragent_suffix`` :
+  Suffix to the user-agent searx uses to send requests to others engines.  If an
+  engine wish to block you, a contact info here may be useful to avoid that.
+
+``image_proxy`` :
+  Allow your instance of searx of being able to proxy images.  Uses memory space.
+
+``default_locale`` :
+  Aearx interface language.  If blank, the locale is detected by using the
+  browser language.  If it doesn't work, or you are deploying a language
+  specific instance of searx, a locale can be defined using an ISO language
+  code, like ``fr``, ``en``, ``de``.
+
+.. _requests proxies: http://docs.python-requests.org/en/latest/user/advanced/#proxies
+.. _PR SOCKS support: https://github.com/kennethreitz/requests/pull/478
+
+``outgoing_proxies`` :
+  Define a proxy you wish to use, see `requests proxies`_.  SOCKS proxies are
+  not supported / see `PR SOCKS support`.
+
+``source_ips`` :
+  If you use multiple nework interfaces, define from which IP the requests must
+  be made.
+
+``locales`` :
+  Locales codes and their names.  Available translations of searx interface.
+
+
+.. _settings engine:
+
+Engine settings
+===============
+
+.. sidebar:: Further reading ..
+
+   - :ref:`engines-dev`
+
+.. code:: yaml
+
+   - name : bing
+     engine : bing
+     shortcut : bi
+     base_url : 'https://{language}.wikipedia.org/'
+     categories : general
+     timeout : 3.0
+     api_key : 'apikey'
+     disabled : True
+     language : en_US
+
+``name`` :
+  Name that will be used accross searx to define this engine.  In settings, on
+  the result page...
+
+``engine`` :
+  Name of the python file used to handle requests and responses to and from this
+  search engine.
+
+``shortcut`` :
+  Code used to execute bang requests (in this case using ``!bi`` or ``?bi``)
+
+``base_url`` : optional
+  Part of the URL that should be stable accross every request.  Can be useful to
+  use multiple sites using only one engine, or updating the site URL without
+  touching at the code.
+
+``categories`` : optional
+  Define in which categories this engine will be active.  Most of the time, it is
+  defined in the code of the engine, but in a few cases it is useful, like when
+  describing multiple search engine using the same code.
+
+``timeout`` : optional
+  Timeout of the search with the current search engine.  **Be careful, it will
+  modify the global timeout of searx.**
+
+``api_key`` : optional
+  In a few cases, using an API needs the use of a secret key.  How to obtain them
+  is described in the file.
+
+``disabled`` : optional
+  To disable by default the engine, but not deleting it. It will allow the user
+  to manually activate it in the settings.
+
+``language`` : optional
+  If you want to use another language for a specific engine, you can define it
+  by using the full ISO code of language and country, like ``fr_FR``, ``en_US``,
+  ``de_DE``.
+
+``weigth`` : default ``1``
+  Weighting of the results of this engine.
+
+.. note::
+
+   A few more options are possible, but they are pretty specific to some
+   engines, and so won't be described here.