From 5720844fcdc8601798e10544e2fd25ce4f2ad099 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 30 Jun 2023 18:07:02 +0200 Subject: [doc] rearranges Settings & Engines docs for better readability We have built up detailed documentation of the *settings* and the *engines* over the past few years. However, this documentation was still spread over various chapters and was difficult to navigate in its entirety. This patch rearranges the Settings & Engines documentation for better readability. To review new ordered docs:: make docs.clean docs.live Signed-off-by: Markus Heiser --- docs/admin/buildhosts.rst | 2 +- docs/admin/engines/command-line-engines.rst | 79 --- docs/admin/engines/index.rst | 26 - docs/admin/engines/nosql-engines.rst | 135 ---- docs/admin/engines/private-engines.rst | 49 -- docs/admin/engines/recoll.rst | 50 -- docs/admin/engines/search-indexer-engines.rst | 136 ---- docs/admin/engines/searx.engines.xpath.rst | 9 - docs/admin/engines/settings.rst | 778 --------------------- docs/admin/engines/sql-engines.rst | 166 ----- docs/admin/index.rst | 3 +- docs/admin/installation-apache.rst | 2 +- docs/admin/installation-nginx.rst | 2 +- docs/admin/installation-searxng.rst | 4 +- docs/admin/installation-uwsgi.rst | 2 +- docs/admin/settings/index.rst | 25 + docs/admin/settings/settings.rst | 117 ++++ docs/admin/settings/settings_brand.rst | 25 + .../admin/settings/settings_categories_as_tabs.rst | 31 + docs/admin/settings/settings_engine.rst | 244 +++++++ docs/admin/settings/settings_general.rst | 34 + docs/admin/settings/settings_outgoing.rst | 110 +++ docs/admin/settings/settings_redis.rst | 43 ++ docs/admin/settings/settings_search.rst | 97 +++ docs/admin/settings/settings_server.rst | 54 ++ docs/admin/settings/settings_ui.rst | 62 ++ docs/admin/update-searxng.rst | 2 +- docs/dev/contribution_guide.rst | 2 +- docs/dev/engine_overview.rst | 432 ------------ docs/dev/engines/demo/demo_offline.rst | 14 + docs/dev/engines/demo/demo_online.rst | 14 + docs/dev/engines/engine_overview.rst | 468 +++++++++++++ docs/dev/engines/enginelib.rst | 22 + docs/dev/engines/engines.rst | 9 + docs/dev/engines/index.rst | 97 +++ docs/dev/engines/offline/command-line-engines.rst | 23 + docs/dev/engines/offline/nosql-engines.rst | 97 +++ .../dev/engines/offline/search-indexer-engines.rst | 62 ++ docs/dev/engines/offline/sql-engines.rst | 121 ++++ docs/dev/engines/offline_concept.rst | 69 ++ docs/dev/engines/online/annas_archive.rst | 13 + docs/dev/engines/online/archlinux.rst | 14 + docs/dev/engines/online/bing.rst | 43 ++ docs/dev/engines/online/dailymotion.rst | 13 + docs/dev/engines/online/duckduckgo.rst | 22 + docs/dev/engines/online/google.rst | 76 ++ docs/dev/engines/online/peertube.rst | 27 + docs/dev/engines/online/recoll.rst | 13 + docs/dev/engines/online/startpage.rst | 13 + docs/dev/engines/online/torznab.rst | 13 + docs/dev/engines/online/wikipedia.rst | 27 + docs/dev/engines/online/yahoo.rst | 13 + docs/dev/engines/online_url_search/tineye.rst | 14 + docs/dev/engines/xpath.rst | 14 + docs/dev/index.rst | 4 +- docs/dev/lxcdev.rst | 2 +- docs/dev/makefile.rst | 6 +- docs/dev/offline_engines.rst | 78 --- docs/dev/reST.rst | 2 +- docs/dev/searxng_extra/index.rst | 1 - docs/index.rst | 1 - docs/own-instance.rst | 2 +- docs/src/index.rst | 1 - docs/src/searx.botdetection.rst | 2 +- docs/src/searx.engine.archlinux.rst | 9 - docs/src/searx.engine.dailymotion.rst | 8 - docs/src/searx.engine.duckduckgo.rst | 22 - docs/src/searx.enginelib.rst | 17 - docs/src/searx.engines.annas_archive.rst | 2 - docs/src/searx.engines.bing.rst | 43 -- docs/src/searx.engines.demo_offline.rst | 9 - docs/src/searx.engines.demo_online.rst | 9 - docs/src/searx.engines.google.rst | 76 -- docs/src/searx.engines.peertube.rst | 27 - docs/src/searx.engines.rst | 8 - docs/src/searx.engines.startpage.rst | 13 - docs/src/searx.engines.tineye.rst | 9 - docs/src/searx.engines.torznab.rst | 2 - docs/src/searx.engines.wikipedia.rst | 27 - docs/src/searx.engines.yahoo.rst | 8 - docs/src/searx.locales.rst | 2 +- docs/src/searx.search.processors.rst | 4 +- docs/user/configured_engines.rst | 2 +- docs/user/index.rst | 2 +- docs/utils/index.rst | 1 - docs/utils/lxc.sh.rst | 2 +- docs/utils/searxng.sh.rst | 2 +- 87 files changed, 2177 insertions(+), 2258 deletions(-) delete mode 100644 docs/admin/engines/command-line-engines.rst delete mode 100644 docs/admin/engines/index.rst delete mode 100644 docs/admin/engines/nosql-engines.rst delete mode 100644 docs/admin/engines/private-engines.rst delete mode 100644 docs/admin/engines/recoll.rst delete mode 100644 docs/admin/engines/search-indexer-engines.rst delete mode 100644 docs/admin/engines/searx.engines.xpath.rst delete mode 100644 docs/admin/engines/settings.rst delete mode 100644 docs/admin/engines/sql-engines.rst create mode 100644 docs/admin/settings/index.rst create mode 100644 docs/admin/settings/settings.rst create mode 100644 docs/admin/settings/settings_brand.rst create mode 100644 docs/admin/settings/settings_categories_as_tabs.rst create mode 100644 docs/admin/settings/settings_engine.rst create mode 100644 docs/admin/settings/settings_general.rst create mode 100644 docs/admin/settings/settings_outgoing.rst create mode 100644 docs/admin/settings/settings_redis.rst create mode 100644 docs/admin/settings/settings_search.rst create mode 100644 docs/admin/settings/settings_server.rst create mode 100644 docs/admin/settings/settings_ui.rst delete mode 100644 docs/dev/engine_overview.rst create mode 100644 docs/dev/engines/demo/demo_offline.rst create mode 100644 docs/dev/engines/demo/demo_online.rst create mode 100644 docs/dev/engines/engine_overview.rst create mode 100644 docs/dev/engines/enginelib.rst create mode 100644 docs/dev/engines/engines.rst create mode 100644 docs/dev/engines/index.rst create mode 100644 docs/dev/engines/offline/command-line-engines.rst create mode 100644 docs/dev/engines/offline/nosql-engines.rst create mode 100644 docs/dev/engines/offline/search-indexer-engines.rst create mode 100644 docs/dev/engines/offline/sql-engines.rst create mode 100644 docs/dev/engines/offline_concept.rst create mode 100644 docs/dev/engines/online/annas_archive.rst create mode 100644 docs/dev/engines/online/archlinux.rst create mode 100644 docs/dev/engines/online/bing.rst create mode 100644 docs/dev/engines/online/dailymotion.rst create mode 100644 docs/dev/engines/online/duckduckgo.rst create mode 100644 docs/dev/engines/online/google.rst create mode 100644 docs/dev/engines/online/peertube.rst create mode 100644 docs/dev/engines/online/recoll.rst create mode 100644 docs/dev/engines/online/startpage.rst create mode 100644 docs/dev/engines/online/torznab.rst create mode 100644 docs/dev/engines/online/wikipedia.rst create mode 100644 docs/dev/engines/online/yahoo.rst create mode 100644 docs/dev/engines/online_url_search/tineye.rst create mode 100644 docs/dev/engines/xpath.rst delete mode 100644 docs/dev/offline_engines.rst delete mode 100644 docs/src/searx.engine.archlinux.rst delete mode 100644 docs/src/searx.engine.dailymotion.rst delete mode 100644 docs/src/searx.engine.duckduckgo.rst delete mode 100644 docs/src/searx.enginelib.rst delete mode 100644 docs/src/searx.engines.annas_archive.rst delete mode 100644 docs/src/searx.engines.bing.rst delete mode 100644 docs/src/searx.engines.demo_offline.rst delete mode 100644 docs/src/searx.engines.demo_online.rst delete mode 100644 docs/src/searx.engines.google.rst delete mode 100644 docs/src/searx.engines.peertube.rst delete mode 100644 docs/src/searx.engines.rst delete mode 100644 docs/src/searx.engines.startpage.rst delete mode 100644 docs/src/searx.engines.tineye.rst delete mode 100644 docs/src/searx.engines.torznab.rst delete mode 100644 docs/src/searx.engines.wikipedia.rst delete mode 100644 docs/src/searx.engines.yahoo.rst (limited to 'docs') diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index d4d5ecb91..d31e4b79b 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -9,7 +9,7 @@ Buildhosts If you have any contribution send us your :pull:`PR <../pulls>`, see :ref:`how to contribute`. -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/admin/engines/command-line-engines.rst b/docs/admin/engines/command-line-engines.rst deleted file mode 100644 index e9535e74f..000000000 --- a/docs/admin/engines/command-line-engines.rst +++ /dev/null @@ -1,79 +0,0 @@ -.. _engine command: - -==================== -Command Line Engines -==================== - -.. sidebar:: info - - - :origin:`command.py ` - - :ref:`offline engines` - -With *command engines* administrators can run engines to integrate arbitrary -shell commands. - -When creating and enabling a ``command`` engine on a public instance, you must -be careful to avoid leaking private data. The easiest solution is to limit the -access by setting ``tokens`` as described in section :ref:`private engines`. - -The engine base is flexible. Only your imagination can limit the power of this -engine (and maybe security concerns). The following options are available: - -``command``: - A comma separated list of the elements of the command. A special token - ``{{QUERY}}`` tells where to put the search terms of the user. Example: - - .. code:: yaml - - ['ls', '-l', '-h', '{{QUERY}}'] - -``delimiter``: - A mapping containing a delimiter ``char`` and the *titles* of each element in - ``keys``. - -``parse_regex``: - A dict containing the regular expressions for each result key. - -``query_type``: - - The expected type of user search terms. Possible values: ``path`` and - ``enum``. - - ``path``: - Checks if the user provided path is inside the working directory. If not, - the query is not executed. - - ``enum``: - Is a list of allowed search terms. If the user submits something which is - not included in the list, the query returns an error. - -``query_enum``: - A list containing allowed search terms if ``query_type`` is set to ``enum``. - -``working_dir``: - - The directory where the command has to be executed. Default: ``./`` - -``result_separator``: - The character that separates results. Default: ``\n`` - -The example engine below can be used to find files with a specific name in the -configured working directory: - -.. code:: yaml - - - name: find - engine: command - command: ['find', '.', '-name', '{{QUERY}}'] - query_type: path - shortcut: fnd - delimiter: - chars: ' ' - keys: ['line'] - - -Acknowledgment -============== - -This development was sponsored by `Search and Discovery Fund -`_ of `NLnet Foundation `_. diff --git a/docs/admin/engines/index.rst b/docs/admin/engines/index.rst deleted file mode 100644 index 92c723817..000000000 --- a/docs/admin/engines/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _engines and settings: - -================== -Engines & Settings -================== - -.. sidebar:: Further reading .. - - - :ref:`settings engine` - - :ref:`engine settings` & :ref:`engine file` - -.. toctree:: - :maxdepth: 3 - - settings - -.. toctree:: - :maxdepth: 1 - - private-engines - recoll - sql-engines - nosql-engines - search-indexer-engines - command-line-engines - searx.engines.xpath diff --git a/docs/admin/engines/nosql-engines.rst b/docs/admin/engines/nosql-engines.rst deleted file mode 100644 index 93f1dd58e..000000000 --- a/docs/admin/engines/nosql-engines.rst +++ /dev/null @@ -1,135 +0,0 @@ -=============== -NoSQL databases -=============== - -.. sidebar:: further read - - - `NoSQL databases `_ - - `redis.io `_ - - `MongoDB `_ - -The following `NoSQL databases`_ are supported: - -- :ref:`engine redis_server` -- :ref:`engine mongodb` - -All of the engines above are just commented out in the :origin:`settings.yml -`, as you have to set various options and install -dependencies before using them. - -By default, the engines use the ``key-value`` template for displaying results / -see :origin:`simple ` -theme. If you are not satisfied with the original result layout, you can use -your own template, set ``result_template`` attribute to ``{template_name}`` and -place the templates at:: - - searx/templates/{theme_name}/result_templates/{template_name} - -Furthermore, if you do not wish to expose these engines on a public instance, you -can still add them and limit the access by setting ``tokens`` as described in -section :ref:`private engines`. - - -Configure the engines -===================== - -`NoSQL databases`_ are used for storing arbitrary data without first defining -their structure. - - -Extra Dependencies ------------------- - -For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to -install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searxng-src`) you can use -:ref:`searxng.sh`:: - - $ sudo utils/searxng.sh instance cmd bash - (searxng-pyenv)$ pip install ... - - -.. _engine redis_server: - -Redis Server ------------- - -.. _redis: https://github.com/andymccurdy/redis-py#installation - -.. sidebar:: info - - - ``pip install`` redis_ - - redis.io_ - - :origin:`redis_server.py ` - - -Redis is an open source (BSD licensed), in-memory data structure (key value -based) store. Before configuring the ``redis_server`` engine, you must install -the dependency redis_. - -Select a database to search in and set its index in the option ``db``. You can -either look for exact matches or use partial keywords to find what you are -looking for by configuring ``exact_match_only``. You find an example -configuration below: - -.. code:: yaml - - # Required dependency: redis - - - name: myredis - shortcut : rds - engine: redis_server - exact_match_only: false - host: '127.0.0.1' - port: 6379 - enable_http: true - password: '' - db: 0 - -.. _engine mongodb: - -MongoDB -------- - -.. _pymongo: https://github.com/mongodb/mongo-python-driver#installation - -.. sidebar:: info - - - ``pip install`` pymongo_ - - MongoDB_ - - :origin:`mongodb.py ` - -MongoDB_ is a document based database program that handles JSON like data. -Before configuring the ``mongodb`` engine, you must install the dependency -redis_. - -In order to query MongoDB_, you have to select a ``database`` and a -``collection``. Furthermore, you have to select a ``key`` that is going to be -searched. MongoDB_ also supports the option ``exact_match_only``, so configure -it as you wish. Below is an example configuration for using a MongoDB -collection: - -.. code:: yaml - - # MongoDB engine - # Required dependency: pymongo - - - name: mymongo - engine: mongodb - shortcut: md - exact_match_only: false - host: '127.0.0.1' - port: 27017 - enable_http: true - results_per_page: 20 - database: 'business' - collection: 'reviews' # name of the db collection - key: 'name' # key in the collection to search for - - -Acknowledgment -============== - -This development was sponsored by `Search and Discovery Fund -`_ of `NLnet Foundation `_. - diff --git a/docs/admin/engines/private-engines.rst b/docs/admin/engines/private-engines.rst deleted file mode 100644 index cc6ab2565..000000000 --- a/docs/admin/engines/private-engines.rst +++ /dev/null @@ -1,49 +0,0 @@ -.. _private engines: - -============================ -Private Engines (``tokens``) -============================ - -Administrators might find themselves wanting to limit access to some of the -enabled engines on their instances. It might be because they do not want to -expose some private information through :ref:`offline engines`. Or they would -rather share engines only with their trusted friends or colleagues. - -To solve this issue the concept of *private engines* exists. - - -A new option was added to engines named `tokens`. It expects a list of -strings. If the user making a request presents one of the tokens of an engine, -they can access information about the engine and make search requests. - -Example configuration to restrict access to the Arch Linux Wiki engine: - -.. code:: yaml - - - name: arch linux wiki - engine: archlinux - shortcut: al - tokens: [ 'my-secret-token' ] - - -Unless a user has configured the right token, the engine is going -to be hidden from him/her. It is not going to be included in the -list of engines on the Preferences page and in the output of -`/config` REST API call. - -Tokens can be added to one's configuration on the Preferences page -under "Engine tokens". The input expects a comma separated list of -strings. - -The distribution of the tokens from the administrator to the users -is not carved in stone. As providing access to such engines -implies that the admin knows and trusts the user, we do not see -necessary to come up with a strict process. Instead, -we would like to add guidelines to the documentation of the feature. - - -Acknowledgment -============== - -This development was sponsored by `Search and Discovery Fund -`_ of `NLnet Foundation `_. diff --git a/docs/admin/engines/recoll.rst b/docs/admin/engines/recoll.rst deleted file mode 100644 index ab8530044..000000000 --- a/docs/admin/engines/recoll.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. _engine recoll: - -============= -Recoll Engine -============= - -.. sidebar:: info - - - `Recoll `_ - - `recoll-webui `_ - - :origin:`searx/engines/recoll.py` - -Recoll_ is a desktop full-text search tool based on Xapian. By itself Recoll_ -does not offer WEB or API access, this can be achieved using recoll-webui_ - - -Configuration -============= - -You must configure the following settings: - -``base_url``: - Location where recoll-webui can be reached. - -``mount_prefix``: - Location where the file hierarchy is mounted on your *local* filesystem. - -``dl_prefix``: - Location where the file hierarchy as indexed by recoll can be reached. - -``search_dir``: - Part of the indexed file hierarchy to be search, if empty the full domain is - searched. - - -Example -======= - -Scenario: - -#. Recoll indexes a local filesystem mounted in ``/export/documents/reference``, -#. the Recoll search interface can be reached at https://recoll.example.org/ and -#. the contents of this filesystem can be reached though https://download.example.org/reference - -.. code:: yaml - - base_url: https://recoll.example.org/ - mount_prefix: /export/documents - dl_prefix: https://download.example.org - search_dir: '' diff --git a/docs/admin/engines/search-indexer-engines.rst b/docs/admin/engines/search-indexer-engines.rst deleted file mode 100644 index 51a661896..000000000 --- a/docs/admin/engines/search-indexer-engines.rst +++ /dev/null @@ -1,136 +0,0 @@ -==================== -Local Search Engines -==================== - -.. sidebar:: further read - - - `Comparison to alternatives - `_ - -Administrators might find themselves wanting to integrate locally running search -engines. The following ones are supported for now: - -* `Elasticsearch`_ -* `Meilisearch`_ -* `Solr`_ - -Each search engine is powerful, capable of full-text search. All of the engines -above are added to ``settings.yml`` just commented out, as you have to -``base_url`` for all them. - -Please note that if you are not using HTTPS to access these engines, you have to enable -HTTP requests by setting ``enable_http`` to ``True``. - -Furthermore, if you do not want to expose these engines on a public instance, you -can still add them and limit the access by setting ``tokens`` as described in -section :ref:`private engines`. - -.. _engine meilisearch: - -MeiliSearch -=========== - -.. sidebar:: info - - - :origin:`meilisearch.py ` - - `MeiliSearch `_ - - `MeiliSearch Documentation `_ - - `Install MeiliSearch - `_ - -MeiliSearch_ is aimed at individuals and small companies. It is designed for -small-scale (less than 10 million documents) data collections. E.g. it is great -for storing web pages you have visited and searching in the contents later. - -The engine supports faceted search, so you can search in a subset of documents -of the collection. Furthermore, you can search in MeiliSearch_ instances that -require authentication by setting ``auth_token``. - -Here is a simple example to query a Meilisearch instance: - -.. code:: yaml - - - name: meilisearch - engine: meilisearch - shortcut: mes - base_url: http://localhost:7700 - index: my-index - enable_http: true - - -.. _engine elasticsearch: - -Elasticsearch -============= - -.. sidebar:: info - - - :origin:`elasticsearch.py ` - - `Elasticsearch `_ - - `Elasticsearch Guide - `_ - - `Install Elasticsearch - `_ - -Elasticsearch_ supports numerous ways to query the data it is storing. At the -moment the engine supports the most popular search methods (``query_type``): - -- ``match``, -- ``simple_query_string``, -- ``term`` and -- ``terms``. - -If none of the methods fit your use case, you can select ``custom`` query type -and provide the JSON payload to submit to Elasticsearch in -``custom_query_json``. - -The following is an example configuration for an Elasticsearch_ instance with -authentication configured to read from ``my-index`` index. - -.. code:: yaml - - - name: elasticsearch - shortcut: es - engine: elasticsearch - base_url: http://localhost:9200 - username: elastic - password: changeme - index: my-index - query_type: match - # custom_query_json: '{ ... }' - enable_http: true - -.. _engine solr: - -Solr -==== - -.. sidebar:: info - - - :origin:`solr.py ` - - `Solr `_ - - `Solr Resources `_ - - `Install Solr `_ - -Solr_ is a popular search engine based on Lucene, just like Elasticsearch_. But -instead of searching in indices, you can search in collections. - -This is an example configuration for searching in the collection -``my-collection`` and get the results in ascending order. - -.. code:: yaml - - - name: solr - engine: solr - shortcut: slr - base_url: http://localhost:8983 - collection: my-collection - sort: asc - enable_http: true - - -Acknowledgment -============== - -This development was sponsored by `Search and Discovery Fund -`_ of `NLnet Foundation `_. diff --git a/docs/admin/engines/searx.engines.xpath.rst b/docs/admin/engines/searx.engines.xpath.rst deleted file mode 100644 index 695aa5224..000000000 --- a/docs/admin/engines/searx.engines.xpath.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _xpath engine: - -============ -XPath Engine -============ - -.. automodule:: searx.engines.xpath - :members: - diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst deleted file mode 100644 index 250a27461..000000000 --- a/docs/admin/engines/settings.rst +++ /dev/null @@ -1,778 +0,0 @@ -.. _settings.yml: - -================ -``settings.yml`` -================ - -This page describe the options possibilities of the :origin:`searx/settings.yml` -file. - -.. sidebar:: Further reading .. - - - :ref:`use_default_settings.yml` - - :ref:`search API` - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - -.. _settings location: - -settings.yml location -===================== - -The initial ``settings.yml`` we be load from these locations: - -1. the full path specified in the ``SEARXNG_SETTINGS_PATH`` environment variable. -2. ``/etc/searxng/settings.yml`` - -If these files don't exist (or are empty or can't be read), SearXNG uses the -:origin:`searx/settings.yml` file. Read :ref:`settings use_default_settings` to -see how you can simplify your *user defined* ``settings.yml``. - - -.. _settings global: - -Global Settings -=============== - -.. _settings brand: - -``brand:`` ----------- - -.. code:: yaml - - brand: - issue_url: https://github.com/searxng/searxng/issues - docs_url: https://docs.searxng.org - public_instances: https://searx.space - wiki_url: https://github.com/searxng/searxng/wiki - -``issue_url`` : - If you host your own issue tracker change this URL. - -``docs_url`` : - If you host your own documentation change this URL. - -``public_instances`` : - If you host your own https://searx.space change this URL. - -``wiki_url`` : - Link to your wiki (or ``false``) - -.. _settings general: - -``general:`` ------------- - -.. code:: yaml - - general: - debug: false - instance_name: "SearXNG" - privacypolicy_url: false - donation_url: false - contact_url: false - enable_metrics: true - -``debug`` : ``$SEARXNG_DEBUG`` - Allow a more detailed log if you run SearXNG directly. Display *detailed* error - messages in the browser too, so this must be deactivated in production. - -``donation_url`` : - Set value to ``true`` to use your own donation page written in the - :ref:`searx/info/en/donate.md ` and use ``false`` to disable - the donation link altogether. - -``privacypolicy_url``: - Link to privacy policy. - -``contact_url``: - Contact ``mailto:`` address or WEB form. - -``enable_metrics``: - Enabled by default. Record various anonymous metrics availabled at ``/stats``, - ``/stats/errors`` and ``/preferences``. - -.. _settings search: - -``search:`` ------------ - -.. code:: yaml - - search: - safe_search: 0 - autocomplete: "" - default_lang: "" - ban_time_on_fail: 5 - max_ban_time_on_fail: 120 - suspended_times: - SearxEngineAccessDenied: 86400 - SearxEngineCaptcha: 86400 - SearxEngineTooManyRequests: 3600 - cf_SearxEngineCaptcha: 1296000 - cf_SearxEngineAccessDenied: 86400 - recaptcha_SearxEngineCaptcha: 604800 - formats: - - html - -``safe_search``: - Filter results. - - - ``0``: None - - ``1``: Moderate - - ``2``: Strict - -``autocomplete``: - Existing autocomplete backends, leave blank to turn it off. - - - ``dbpedia`` - - ``duckduckgo`` - - ``google`` - - ``startpage`` - - ``swisscows`` - - ``qwant`` - - ``wikipedia`` - -``default_lang``: - Default search language - leave blank to detect from browser information or - use codes from :origin:`searx/languages.py`. - -``languages``: - List of available languages - leave unset to use all codes from - :origin:`searx/languages.py`. Otherwise list codes of available languages. - The ``all`` value is shown as the ``Default language`` in the user interface - (in most cases, it is meant to send the query without a language parameter ; - in some cases, it means the English language) Example: - - .. code:: yaml - - languages: - - all - - en - - en-US - - de - - it-IT - - fr - - fr-BE - -``ban_time_on_fail``: - Ban time in seconds after engine errors. - -``max_ban_time_on_fail``: - Max ban time in seconds after engine errors. - -``suspended_times``: - Engine suspension time after error (in seconds; set to 0 to disable) - - ``SearxEngineAccessDenied``: 86400 - For error "Access denied" and "HTTP error [402, 403]" - - ``SearxEngineCaptcha``: 86400 - For error "CAPTCHA" - - ``SearxEngineTooManyRequests``: 3600 - For error "Too many request" and "HTTP error 429" - - Cloudflare CAPTCHA: - - ``cf_SearxEngineCaptcha``: 1296000 - - ``cf_SearxEngineAccessDenied``: 86400 - - Google CAPTCHA: - - ``recaptcha_SearxEngineCaptcha``: 604800 - -``formats``: - Result formats available from web, remove format to deny access (use lower - case). - - - ``html`` - - ``csv`` - - ``json`` - - ``rss`` - - -.. _settings server: - -``server:`` ------------ - -.. code:: yaml - - server: - base_url: http://example.org/location # change this! - port: 8888 - bind_address: "127.0.0.1" - secret_key: "ultrasecretkey" # change this! - limiter: false - image_proxy: false - default_http_headers: - X-Content-Type-Options : nosniff - X-XSS-Protection : 1; mode=block - X-Download-Options : noopen - X-Robots-Tag : noindex, nofollow - Referrer-Policy : no-referrer - - -``base_url`` : ``$SEARXNG_URL`` :ref:`buildenv ` - The base URL where SearXNG is deployed. Used to create correct inbound links. - If you change the value, don't forget to rebuild instance's environment - (:ref:`utils/brand.env `) - -``port`` & ``bind_address``: ``$SEARXNG_PORT`` & ``$SEARXNG_BIND_ADDRESS`` :ref:`buildenv ` - Port number and *bind address* of the SearXNG web application if you run it - directly using ``python searx/webapp.py``. Doesn't apply to a SearXNG - services running behind a proxy and using socket communications. If you - change the value, don't forget to rebuild instance's environment - (:ref:`utils/brand.env `) - -``secret_key`` : ``$SEARXNG_SECRET`` - Used for cryptography purpose. - -.. _limiter: - -``limiter`` : - Rate limit the number of request on the instance, block some bots. The - :ref:`limiter src` requires a :ref:`settings redis` database. - -.. _image_proxy: - -``image_proxy`` : - Allow your instance of SearXNG of being able to proxy images. Uses memory space. - -.. _HTTP headers: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers - -``default_http_headers`` : - Set additional HTTP headers, see `#755 `__ - - -.. _settings ui: - -``ui:`` -------- - -.. _cache busting: - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#caching_static_assets_with_cache_busting - -.. code:: yaml - - ui: - static_use_hash: false - default_locale: "" - query_in_title: false - infinite_scroll: false - center_alignment: false - cache_url: https://web.archive.org/web/ - default_theme: simple - theme_args: - simple_style: auto - -.. _static_use_hash: - -``static_use_hash`` : - Enables `cache busting`_ of static files. - -``default_locale`` : - SearXNG interface language. If blank, the locale is detected by using the - browser language. If it doesn't work, or you are deploying a language - specific instance of searx, a locale can be defined using an ISO language - code, like ``fr``, ``en``, ``de``. - -``query_in_title`` : - When true, the result page's titles contains the query it decreases the - privacy, since the browser can records the page titles. - -``infinite_scroll``: - When true, automatically loads the next page when scrolling to bottom of the current page. - -``center_alignment`` : default ``false`` - When enabled, the results are centered instead of being in the left (or RTL) - side of the screen. This setting only affects the *desktop layout* - (:origin:`min-width: @tablet `) - -.. cache_url: - -``cache_url`` : ``https://web.archive.org/web/`` - URL prefix of the internet archive or cache, don't forgett trailing slash (if - needed). The default is https://web.archive.org/web/ alternatives are: - - - https://webcache.googleusercontent.com/search?q=cache: - - https://archive.today/ - -``default_theme`` : - Name of the theme you want to use by default on your SearXNG instance. - -``theme_args.simple_style``: - Style of simple theme: ``auto``, ``light``, ``dark`` - -``results_on_new_tab``: - Open result links in a new tab by default. - -.. _settings redis: - -``redis:`` ----------- - -.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url - -A redis DB can be connected by an URL, in :py:obj:`searx.redisdb` you -will find a description to test your redis connection in SerXNG. When using -sockets, don't forget to check the access rights on the socket:: - - ls -la /usr/local/searxng-redis/run/redis.sock - srwxrwx--- 1 searxng-redis searxng-redis ... /usr/local/searxng-redis/run/redis.sock - -In this example read/write access is given to the *searxng-redis* group. To get -access rights to redis instance (the socket), your SearXNG (or even your -developer) account needs to be added to the *searxng-redis* group. - -``url`` : ``$SEARXNG_REDIS_URL`` - URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`:: - - redis://[[username]:[password]]@localhost:6379/0 - rediss://[[username]:[password]]@localhost:6379/0 - unix://[[username]:[password]]@/path/to/socket.sock?db=0 - -.. admonition:: Tip for developers - - To set up a local redis instance, first set the socket path of the Redis DB - in your YAML setting: - - .. code:: yaml - - redis: - url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 - - Then use the following commands to install the redis instance :: - - $ ./manage redis.build - $ sudo -H ./manage redis.install - $ sudo -H ./manage redis.addgrp "${USER}" - # don't forget to logout & login to get member of group - - -.. _settings outgoing: - -``outgoing:`` -------------- - -Communication with search engines. - -.. code:: yaml - - outgoing: - request_timeout: 2.0 # default timeout in seconds, can be override by engine - max_request_timeout: 10.0 # the maximum timeout in seconds - useragent_suffix: "" # information like an email address to the administrator - pool_connections: 100 # Maximum number of allowable connections, or null - # for no limits. The default is 100. - pool_maxsize: 10 # Number of allowable keep-alive connections, or null - # to always allow. The default is 10. - enable_http2: true # See https://www.python-httpx.org/http2/ - # uncomment below section if you want to use a custom server certificate - # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults - # and https://www.python-httpx.org/compatibility/#ssl-configuration - # verify: ~/.mitmproxy/mitmproxy-ca-cert.cer - # - # uncomment below section if you want to use a proxyq see: SOCKS proxies - # https://2.python-requests.org/en/latest/user/advanced/#proxies - # are also supported: see - # https://2.python-requests.org/en/latest/user/advanced/#socks - # - # proxies: - # all://: - # - http://proxy1:8080 - # - http://proxy2:8080 - # - # using_tor_proxy: true - # - # Extra seconds to add in order to account for the time taken by the proxy - # - # extra_proxy_timeout: 10.0 - # - -``request_timeout`` : - Global timeout of the requests made to others engines in seconds. A bigger - timeout will allow to wait for answers from slow engines, but in consequence - will slow SearXNG reactivity (the result page may take the time specified in the - timeout to load). Can be override by ``timeout`` in the :ref:`settings engine`. - -``useragent_suffix`` : - Suffix to the user-agent SearXNG uses to send requests to others engines. If an - engine wish to block you, a contact info here may be useful to avoid that. - -.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration - -``pool_maxsize``: - Number of allowable keep-alive connections, or ``null`` to always allow. The - default is 10. See ``max_keepalive_connections`` `Pool limit configuration`_. - -``pool_connections`` : - Maximum number of allowable connections, or ``null`` # for no limits. The - default is 100. See ``max_connections`` `Pool limit configuration`_. - -``keepalive_expiry`` : - Number of seconds to keep a connection in the pool. By default 5.0 seconds. - See ``keepalive_expiry`` `Pool limit configuration`_. - - -.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying - -``proxies`` : - Define one or more proxies you wish to use, see `httpx proxies`_. - If there are more than one proxy for one protocol (http, https), - requests to the engines are distributed in a round-robin fashion. - -``source_ips`` : - If you use multiple network interfaces, define from which IP the requests must - be made. Example: - - * ``0.0.0.0`` any local IPv4 address. - * ``::`` any local IPv6 address. - * ``192.168.0.1`` - * ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses - * ``fe80::60a2:1691:e5a2:ee1f`` - * ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network. - * ``[ 192.168.0.1, fe80::/126 ]`` - -``retries`` : - Number of retry in case of an HTTP error. On each retry, SearXNG uses an - different proxy and source ip. - -``enable_http2`` : - Enable by default. Set to ``false`` to disable HTTP/2. - -.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults -.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration - -``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR`` - Allow to specify a path to certificate. - see `httpx verification defaults`_. - - In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and - ``$SSL_CERT_DIR`` (for a directory) OpenSSL variables. - see `httpx ssl configuration`_. - -``max_redirects`` : - 30 by default. Maximum redirect before it is an error. - -``using_tor_proxy`` : - Using tor proxy (``true``) or not (``false``) for all engines. The default is - ``false`` and can be overwritten in the :ref:`settings engine` - - - -.. _settings categories_as_tabs: - -``categories_as_tabs:`` ------------------------ - -A list of the categories that are displayed as tabs in the user interface. -Categories not listed here can still be searched with the :ref:`search-syntax`. - -.. code-block:: yaml - - categories_as_tabs: - general: - images: - videos: - news: - map: - music: - it: - science: - files: - social media: - -Engines are added to ``categories:`` (compare :ref:`engine categories`), the -categories listed in ``categories_as_tabs`` are shown as tabs in the UI. If -there are no active engines in a category, the tab is not displayed (e.g. if a -user disables all engines in a category). - -On the preferences page (``/preferences``) -- under *engines* -- there is an -additional tab, called *other*. In this tab are all engines listed that are not -in one of the UI tabs (not included in ``categories_as_tabs``). - -.. _settings engine: - -Engine settings -=============== - -.. sidebar:: Further reading .. - - - :ref:`configured engines` - - :ref:`engines-dev` - -In the code example below a *full fledged* example of a YAML setup from a dummy -engine is shown. Most of the options have a default value or even are optional. - -.. code:: yaml - - - name: example engine - engine: example - shortcut: demo - base_url: 'https://{language}.example.com/' - send_accept_language_header: false - categories: general - timeout: 3.0 - api_key: 'apikey' - disabled: false - language: en_US - tokens: [ 'my-secret-token' ] - weight: 1 - display_error_messages: true - about: - website: https://example.com - wikidata_id: Q306656 - official_api_documentation: https://example.com/api-doc - use_official_api: true - require_api_key: true - results: HTML - - # overwrite values from section 'outgoing:' - enable_http2: false - retries: 1 - max_connections: 100 - max_keepalive_connections: 10 - keepalive_expiry: 5.0 - using_tor_proxy: false - proxies: - http: - - http://proxy1:8080 - - http://proxy2:8080 - https: - - http://proxy1:8080 - - http://proxy2:8080 - - socks5://user:password@proxy3:1080 - - socks5h://user:password@proxy4:1080 - - # other network settings - enable_http: false - retry_on_http_error: true # or 403 or [404, 429] - - -``name`` : - Name that will be used across SearXNG to define this engine. In settings, on - the result page... - -``engine`` : - Name of the python file used to handle requests and responses to and from this - search engine. - -``shortcut`` : - Code used to execute bang requests (in this case using ``!bi``) - -``base_url`` : optional - Part of the URL that should be stable across every request. Can be useful to - use multiple sites using only one engine, or updating the site URL without - touching at the code. - -``send_accept_language_header`` : - Several engines that support languages (or regions) deal with the HTTP header - ``Accept-Language`` to build a response that fits to the locale. When this - option is activated, the language (locale) that is selected by the user is used - to build and send a ``Accept-Language`` header in the request to the origin - search engine. - -.. _engine categories: - -``categories`` : optional - Specifies to which categories the engine should be added. Engines can be - assigned to multiple categories. - - Categories can be shown as tabs (:ref:`settings categories_as_tabs`) in the - UI. A search in a tab (in the UI) will query all engines that are active in - this tab. In the preferences page (``/preferences``) -- under *engines* -- - users can select what engine should be active when querying in this tab. - - Alternatively, :ref:`\!bang ` can be used to search all engines - in a category, regardless of whether they are active or not, or whether they - are in a tab of the UI or not. For example, ``!dictionaries`` can be used to - query all search engines in that category (group). - -``timeout`` : optional - Timeout of the search with the current search engine. Overwrites - ``request_timeout`` from :ref:`settings outgoing`. **Be careful, it will - modify the global timeout of SearXNG.** - -``api_key`` : optional - In a few cases, using an API needs the use of a secret key. How to obtain them - is described in the file. - -``disabled`` : optional - To disable by default the engine, but not deleting it. It will allow the user - to manually activate it in the settings. - -``inactive``: optional - Remove the engine from the settings (*disabled & removed*). - -``language`` : optional - If you want to use another language for a specific engine, you can define it - by using the ISO code of language (and region), like ``fr``, ``en-US``, - ``de-DE``. - -``tokens`` : optional - A list of secret tokens to make this engine *private*, more details see - :ref:`private engines`. - -``weight`` : default ``1`` - Weighting of the results of this engine. - -``display_error_messages`` : default ``true`` - When an engine returns an error, the message is displayed on the user interface. - -``network`` : optional - Use the network configuration from another engine. - In addition, there are two default networks: - - - ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses) - - ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses) - -``enable_http`` : optional - Enable HTTP for this engine (by default only HTTPS is enabled). - -``retry_on_http_error`` : optional - Retry request on some HTTP status code. - - Example: - - * ``true`` : on HTTP status code between 400 and 599. - * ``403`` : on HTTP status code 403. - * ``[403, 429]``: on HTTP status code 403 and 429. - -``proxies`` : - Overwrites proxy settings from :ref:`settings outgoing`. - -``using_tor_proxy`` : - Using tor proxy (``true``) or not (``false``) for this engine. The default is - taken from ``using_tor_proxy`` of the :ref:`settings outgoing`. - -``max_keepalive_connection#s`` : - `Pool limit configuration`_, overwrites value ``pool_maxsize`` from - :ref:`settings outgoing` for this engine. - -``max_connections`` : - `Pool limit configuration`_, overwrites value ``pool_connections`` from - :ref:`settings outgoing` for this engine. - -``keepalive_expiry`` : - `Pool limit configuration`_, overwrites value ``keepalive_expiry`` from - :ref:`settings outgoing` for this engine. - -.. note:: - - A few more options are possible, but they are pretty specific to some - engines, and so won't be described here. - - -Example: Multilingual Search ----------------------------- - -SearXNG does not support true multilingual search. You have to use the language -prefix in your search query when searching in a different language. - -But there is a workaround: By adding a new search engine with a different -language, SearXNG will search in your default and other language. - -Example configuration in settings.yml for a German and English speaker: - -.. code-block:: yaml - - search: - default_lang : "de" - ... - - engines: - - name : google english - engine : google - language : en - ... - -When searching, the default google engine will return German results and -"google english" will return English results. - - -.. _settings use_default_settings: - -use_default_settings -==================== - -.. sidebar:: ``use_default_settings: true`` - - - :ref:`settings location` - - :ref:`use_default_settings.yml` - - :origin:`/etc/searxng/settings.yml ` - -The user defined ``settings.yml`` is loaded from the :ref:`settings location` -and can relied on the default configuration :origin:`searx/settings.yml` using: - - ``use_default_settings: true`` - -``server:`` - In the following example, the actual settings are the default settings defined - in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and - the ``bind_address``: - - .. code-block:: yaml - - use_default_settings: true - server: - secret_key: "ultrasecretkey" # change this! - bind_address: "0.0.0.0" - -``engines:`` - With ``use_default_settings: true``, each settings can be override in a - similar way, the ``engines`` section is merged according to the engine - ``name``. In this example, SearXNG will load all the default engines, will - enable the ``bing`` engine and define a :ref:`token ` for - the arch linux engine: - - .. code-block:: yaml - - use_default_settings: true - server: - secret_key: "ultrasecretkey" # change this! - engines: - - name: arch linux wiki - tokens: ['$ecretValue'] - - name: bing - disabled: false - - -``engines:`` / ``remove:`` - It is possible to remove some engines from the default settings. The following - example is similar to the above one, but SearXNG doesn't load the the google - engine: - - .. code-block:: yaml - - use_default_settings: - engines: - remove: - - google - server: - secret_key: "ultrasecretkey" # change this! - engines: - - name: arch linux wiki - tokens: ['$ecretValue'] - -``engines:`` / ``keep_only:`` - As an alternative, it is possible to specify the engines to keep. In the - following example, SearXNG has only two engines: - - .. code-block:: yaml - - use_default_settings: - engines: - keep_only: - - google - - duckduckgo - server: - secret_key: "ultrasecretkey" # change this! - engines: - - name: google - tokens: ['$ecretValue'] - - name: duckduckgo - tokens: ['$ecretValue'] diff --git a/docs/admin/engines/sql-engines.rst b/docs/admin/engines/sql-engines.rst deleted file mode 100644 index 3cd3c8f73..000000000 --- a/docs/admin/engines/sql-engines.rst +++ /dev/null @@ -1,166 +0,0 @@ -.. _sql engines: - -=========== -SQL Engines -=========== - -.. sidebar:: further read - - - `SQLite `_ - - `PostgreSQL `_ - - `MySQL `_ - -With the *SQL engines* you can bind SQL databases into SearXNG. The following -Relational Database Management System (RDBMS) are supported: - -- :ref:`engine sqlite` -- :ref:`engine postgresql` -- :ref:`engine mysql_server` - -All of the engines above are just commented out in the :origin:`settings.yml -`, as you have to set the required attributes for the -engines, e.g. ``database:`` ... - -.. code:: yaml - - - name: ... - engine: {sqlite|postgresql|mysql_server} - database: ... - result_template: {template_name} - query_str: ... - -By default, the engines use the ``key-value`` template for displaying results / -see :origin:`simple ` -theme. If you are not satisfied with the original result layout, you can use -your own template, set ``result_template`` attribute to ``{template_name}`` and -place the templates at:: - - searx/templates/{theme_name}/result_templates/{template_name} - -If you do not wish to expose these engines on a public instance, you can still -add them and limit the access by setting ``tokens`` as described in section -:ref:`private engines`. - - -Configure the engines -===================== - -The configuration of the new database engines are similar. You must put a valid -SQL-SELECT query in ``query_str``. At the moment you can only bind at most one -parameter in your query. By setting the attribute ``limit`` you can define how -many results you want from the SQL server. Basically, it is the same as the -``LIMIT`` keyword in SQL. - -Please, do not include ``LIMIT`` or ``OFFSET`` in your SQL query as the engines -rely on these keywords during paging. If you want to configure the number of -returned results use the option ``limit``. - -.. _engine sqlite: - -SQLite ------- - -.. sidebar:: info - - - :origin:`sqlite.py ` - -.. _MediathekView: https://mediathekview.de/ - -SQLite is a small, fast and reliable SQL database engine. It does not require -any extra dependency. To demonstrate the power of database engines, here is a -more complex example which reads from a MediathekView_ (DE) movie database. For -this example of the SQlite engine download the database: - -- https://liste.mediathekview.de/filmliste-v2.db.bz2 - -and unpack into ``searx/data/filmliste-v2.db``. To search the database use e.g -Query to test: ``!mediathekview concert`` - -.. code:: yaml - - - name: mediathekview - engine: sqlite - disabled: False - categories: general - result_template: default.html - database: searx/data/filmliste-v2.db - query_str: >- - SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title, - COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url, - description AS content - FROM film - WHERE title LIKE :wildcard OR description LIKE :wildcard - ORDER BY duration DESC - - -Extra Dependencies ------------------- - -For using :ref:`engine postgresql` or :ref:`engine mysql_server` you need to -install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searxng-src`) you can use -:ref:`searxng.sh`:: - - $ sudo utils/searxng.sh instance cmd bash - (searxng-pyenv)$ pip install ... - - -.. _engine postgresql: - -PostgreSQL ----------- - -.. _psycopg2: https://www.psycopg.org/install - -.. sidebar:: info - - - :origin:`postgresql.py ` - - ``pip install`` psycopg2_ - -PostgreSQL is a powerful and robust open source database. Before configuring -the PostgreSQL engine, you must install the dependency ``psychopg2``. You can -find an example configuration below: - -.. code:: yaml - - - name: my_database - engine: postgresql - database: my_database - username: searxng - password: password - query_str: 'SELECT * from my_table WHERE my_column = %(query)s' - -.. _engine mysql_server: - -MySQL ------ - -.. sidebar:: info - - - :origin:`mysql_server.py ` - - ``pip install`` :pypi:`mysql-connector-python ` - -MySQL is said to be the most popular open source database. Before enabling MySQL -engine, you must install the package ``mysql-connector-python``. - -The authentication plugin is configurable by setting ``auth_plugin`` in the -attributes. By default it is set to ``caching_sha2_password``. This is an -example configuration for querying a MySQL server: - -.. code:: yaml - - - name: my_database - engine: mysql_server - database: my_database - username: searxng - password: password - limit: 5 - query_str: 'SELECT * from my_table WHERE my_column=%(query)s' - - -Acknowledgment -============== - -This development was sponsored by `Search and Discovery Fund -`_ of `NLnet Foundation `_. - diff --git a/docs/admin/index.rst b/docs/admin/index.rst index 4b41cd2a4..583b6b428 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -4,8 +4,8 @@ Administrator documentation .. toctree:: :maxdepth: 2 - :caption: Contents + settings/index installation installation-docker installation-scripts @@ -15,7 +15,6 @@ Administrator documentation installation-apache update-searxng answer-captcha - engines/index api architecture plugins diff --git a/docs/admin/installation-apache.rst b/docs/admin/installation-apache.rst index 673a37ee4..9fbb187c5 100644 --- a/docs/admin/installation-apache.rst +++ b/docs/admin/installation-apache.rst @@ -61,7 +61,7 @@ section might give you some guidance. - `Apache Fedora`_ - `Apache directives`_ -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst index 8e5299588..f95354b53 100644 --- a/docs/admin/installation-nginx.rst +++ b/docs/admin/installation-nginx.rst @@ -41,7 +41,7 @@ section might give you some guidance. - `uWSGI support from nginx`_ -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/admin/installation-searxng.rst b/docs/admin/installation-searxng.rst index 9152784fc..3efc48e8c 100644 --- a/docs/admin/installation-searxng.rst +++ b/docs/admin/installation-searxng.rst @@ -4,7 +4,7 @@ Step by step installation ========================= -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry @@ -73,7 +73,7 @@ Configuration .. sidebar:: ``use_default_settings: True`` - - :ref:`settings global` + - :ref:`settings.yml` - :ref:`settings location` - :ref:`settings use_default_settings` - :origin:`/etc/searxng/settings.yml ` diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst index e888d067e..78da22f45 100644 --- a/docs/admin/installation-uwsgi.rst +++ b/docs/admin/installation-uwsgi.rst @@ -9,7 +9,7 @@ uWSGI - `systemd.unit`_ - `uWSGI Emperor`_ -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/admin/settings/index.rst b/docs/admin/settings/index.rst new file mode 100644 index 000000000..005ee37e1 --- /dev/null +++ b/docs/admin/settings/index.rst @@ -0,0 +1,25 @@ +======== +Settings +======== + +.. sidebar:: Further reading .. + + - :ref:`engine settings` + - :ref:`engine file` + +.. toctree:: + :maxdepth: 2 + + settings + settings_engine + settings_brand + settings_general + settings_search + settings_server + settings_ui + settings_redis + settings_outgoing + settings_categories_as_tabs + + + diff --git a/docs/admin/settings/settings.rst b/docs/admin/settings/settings.rst new file mode 100644 index 000000000..9c6fb01be --- /dev/null +++ b/docs/admin/settings/settings.rst @@ -0,0 +1,117 @@ +.. _settings.yml: + +================ +``settings.yml`` +================ + +This page describe the options possibilities of the :origin:`searx/settings.yml` +file. + +.. sidebar:: Further reading .. + + - :ref:`use_default_settings.yml` + - :ref:`search API` + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. _settings location: + +settings.yml location +===================== + +The initial ``settings.yml`` we be load from these locations: + +1. the full path specified in the ``SEARXNG_SETTINGS_PATH`` environment variable. +2. ``/etc/searxng/settings.yml`` + +If these files don't exist (or are empty or can't be read), SearXNG uses the +:origin:`searx/settings.yml` file. Read :ref:`settings use_default_settings` to +see how you can simplify your *user defined* ``settings.yml``. + + + +.. _settings use_default_settings: + +use_default_settings +==================== + +.. sidebar:: ``use_default_settings: true`` + + - :ref:`settings location` + - :ref:`use_default_settings.yml` + - :origin:`/etc/searxng/settings.yml ` + +The user defined ``settings.yml`` is loaded from the :ref:`settings location` +and can relied on the default configuration :origin:`searx/settings.yml` using: + + ``use_default_settings: true`` + +``server:`` + In the following example, the actual settings are the default settings defined + in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and + the ``bind_address``: + + .. code:: yaml + + use_default_settings: true + server: + secret_key: "ultrasecretkey" # change this! + bind_address: "0.0.0.0" + +``engines:`` + With ``use_default_settings: true``, each settings can be override in a + similar way, the ``engines`` section is merged according to the engine + ``name``. In this example, SearXNG will load all the default engines, will + enable the ``bing`` engine and define a :ref:`token ` for + the arch linux engine: + + .. code:: yaml + + use_default_settings: true + server: + secret_key: "ultrasecretkey" # change this! + engines: + - name: arch linux wiki + tokens: ['$ecretValue'] + - name: bing + disabled: false + + +``engines:`` / ``remove:`` + It is possible to remove some engines from the default settings. The following + example is similar to the above one, but SearXNG doesn't load the the google + engine: + + .. code:: yaml + + use_default_settings: + engines: + remove: + - google + server: + secret_key: "ultrasecretkey" # change this! + engines: + - name: arch linux wiki + tokens: ['$ecretValue'] + +``engines:`` / ``keep_only:`` + As an alternative, it is possible to specify the engines to keep. In the + following example, SearXNG has only two engines: + + .. code:: yaml + + use_default_settings: + engines: + keep_only: + - google + - duckduckgo + server: + secret_key: "ultrasecretkey" # change this! + engines: + - name: google + tokens: ['$ecretValue'] + - name: duckduckgo + tokens: ['$ecretValue'] diff --git a/docs/admin/settings/settings_brand.rst b/docs/admin/settings/settings_brand.rst new file mode 100644 index 000000000..0f1a0d9a9 --- /dev/null +++ b/docs/admin/settings/settings_brand.rst @@ -0,0 +1,25 @@ +.. _settings brand: + +========== +``brand:`` +========== + +.. code:: yaml + + brand: + issue_url: https://github.com/searxng/searxng/issues + docs_url: https://docs.searxng.org + public_instances: https://searx.space + wiki_url: https://github.com/searxng/searxng/wiki + +``issue_url`` : + If you host your own issue tracker change this URL. + +``docs_url`` : + If you host your own documentation change this URL. + +``public_instances`` : + If you host your own https://searx.space change this URL. + +``wiki_url`` : + Link to your wiki (or ``false``) diff --git a/docs/admin/settings/settings_categories_as_tabs.rst b/docs/admin/settings/settings_categories_as_tabs.rst new file mode 100644 index 000000000..732d04678 --- /dev/null +++ b/docs/admin/settings/settings_categories_as_tabs.rst @@ -0,0 +1,31 @@ +.. _settings categories_as_tabs: + +======================= +``categories_as_tabs:`` +======================= + +A list of the categories that are displayed as tabs in the user interface. +Categories not listed here can still be searched with the :ref:`search-syntax`. + +.. code:: yaml + + categories_as_tabs: + general: + images: + videos: + news: + map: + music: + it: + science: + files: + social media: + +Engines are added to ``categories:`` (compare :ref:`engine categories`), the +categories listed in ``categories_as_tabs`` are shown as tabs in the UI. If +there are no active engines in a category, the tab is not displayed (e.g. if a +user disables all engines in a category). + +On the preferences page (``/preferences``) -- under *engines* -- there is an +additional tab, called *other*. In this tab are all engines listed that are not +in one of the UI tabs (not included in ``categories_as_tabs``). diff --git a/docs/admin/settings/settings_engine.rst b/docs/admin/settings/settings_engine.rst new file mode 100644 index 000000000..78c400ccf --- /dev/null +++ b/docs/admin/settings/settings_engine.rst @@ -0,0 +1,244 @@ +.. _settings engine: + +=========== +``engine:`` +=========== + +.. sidebar:: Further reading .. + + - :ref:`configured engines` + - :ref:`engines-dev` + +In the code example below a *full fledged* example of a YAML setup from a dummy +engine is shown. Most of the options have a default value or even are optional. + +.. hint:: + + A few more options are possible, but they are pretty specific to some + engines (:ref:`engine implementations`). + +.. code:: yaml + + - name: example engine + engine: example + shortcut: demo + base_url: 'https://{language}.example.com/' + send_accept_language_header: false + categories: general + timeout: 3.0 + api_key: 'apikey' + disabled: false + language: en_US + tokens: [ 'my-secret-token' ] + weight: 1 + display_error_messages: true + about: + website: https://example.com + wikidata_id: Q306656 + official_api_documentation: https://example.com/api-doc + use_official_api: true + require_api_key: true + results: HTML + + # overwrite values from section 'outgoing:' + enable_http2: false + retries: 1 + max_connections: 100 + max_keepalive_connections: 10 + keepalive_expiry: 5.0 + using_tor_proxy: false + proxies: + http: + - http://proxy1:8080 + - http://proxy2:8080 + https: + - http://proxy1:8080 + - http://proxy2:8080 + - socks5://user:password@proxy3:1080 + - socks5h://user:password@proxy4:1080 + + # other network settings + enable_http: false + retry_on_http_error: true # or 403 or [404, 429] + + +``name`` : + Name that will be used across SearXNG to define this engine. In settings, on + the result page... + +``engine`` : + Name of the python file used to handle requests and responses to and from this + search engine. + +``shortcut`` : + Code used to execute bang requests (in this case using ``!bi``) + +``base_url`` : optional + Part of the URL that should be stable across every request. Can be useful to + use multiple sites using only one engine, or updating the site URL without + touching at the code. + +``send_accept_language_header`` : + Several engines that support languages (or regions) deal with the HTTP header + ``Accept-Language`` to build a response that fits to the locale. When this + option is activated, the language (locale) that is selected by the user is used + to build and send a ``Accept-Language`` header in the request to the origin + search engine. + +.. _engine categories: + +``categories`` : optional + Specifies to which categories the engine should be added. Engines can be + assigned to multiple categories. + + Categories can be shown as tabs (:ref:`settings categories_as_tabs`) in the + UI. A search in a tab (in the UI) will query all engines that are active in + this tab. In the preferences page (``/preferences``) -- under *engines* -- + users can select what engine should be active when querying in this tab. + + Alternatively, :ref:`\!bang ` can be used to search all engines + in a category, regardless of whether they are active or not, or whether they + are in a tab of the UI or not. For example, ``!dictionaries`` can be used to + query all search engines in that category (group). + +``timeout`` : optional + Timeout of the search with the current search engine. Overwrites + ``request_timeout`` from :ref:`settings outgoing`. **Be careful, it will + modify the global timeout of SearXNG.** + +``api_key`` : optional + In a few cases, using an API needs the use of a secret key. How to obtain them + is described in the file. + +``disabled`` : optional + To disable by default the engine, but not deleting it. It will allow the user + to manually activate it in the settings. + +``inactive``: optional + Remove the engine from the settings (*disabled & removed*). + +``language`` : optional + If you want to use another language for a specific engine, you can define it + by using the ISO code of language (and region), like ``fr``, ``en-US``, + ``de-DE``. + +``tokens`` : optional + A list of secret tokens to make this engine *private*, more details see + :ref:`private engines`. + +``weight`` : default ``1`` + Weighting of the results of this engine. + +``display_error_messages`` : default ``true`` + When an engine returns an error, the message is displayed on the user interface. + +``network`` : optional + Use the network configuration from another engine. + In addition, there are two default networks: + + - ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses) + - ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses) + +``enable_http`` : optional + Enable HTTP for this engine (by default only HTTPS is enabled). + +``retry_on_http_error`` : optional + Retry request on some HTTP status code. + + Example: + + * ``true`` : on HTTP status code between 400 and 599. + * ``403`` : on HTTP status code 403. + * ``[403, 429]``: on HTTP status code 403 and 429. + +``proxies`` : + Overwrites proxy settings from :ref:`settings outgoing`. + +``using_tor_proxy`` : + Using tor proxy (``true``) or not (``false``) for this engine. The default is + taken from ``using_tor_proxy`` of the :ref:`settings outgoing`. + +.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration + +``max_keepalive_connection#s`` : + `Pool limit configuration`_, overwrites value ``pool_maxsize`` from + :ref:`settings outgoing` for this engine. + +``max_connections`` : + `Pool limit configuration`_, overwrites value ``pool_connections`` from + :ref:`settings outgoing` for this engine. + +``keepalive_expiry`` : + `Pool limit configuration`_, overwrites value ``keepalive_expiry`` from + :ref:`settings outgoing` for this engine. + + +.. _private engines: + +Private Engines (``tokens``) +============================ + +Administrators might find themselves wanting to limit access to some of the +enabled engines on their instances. It might be because they do not want to +expose some private information through :ref:`offline engines`. Or they would +rather share engines only with their trusted friends or colleagues. + +.. sidebar:: info + + Initial sponsored by `Search and Discovery Fund + `_ of `NLnet Foundation `_. + +To solve this issue the concept of *private engines* exists. + +A new option was added to engines named `tokens`. It expects a list of strings. +If the user making a request presents one of the tokens of an engine, they can +access information about the engine and make search requests. + +Example configuration to restrict access to the Arch Linux Wiki engine: + +.. code:: yaml + + - name: arch linux wiki + engine: archlinux + shortcut: al + tokens: [ 'my-secret-token' ] + +Unless a user has configured the right token, the engine is going to be hidden +from him/her. It is not going to be included in the list of engines on the +Preferences page and in the output of `/config` REST API call. + +Tokens can be added to one's configuration on the Preferences page under "Engine +tokens". The input expects a comma separated list of strings. + +The distribution of the tokens from the administrator to the users is not carved +in stone. As providing access to such engines implies that the admin knows and +trusts the user, we do not see necessary to come up with a strict process. +Instead, we would like to add guidelines to the documentation of the feature. + + +Example: Multilingual Search +============================ + +SearXNG does not support true multilingual search. You have to use the language +prefix in your search query when searching in a different language. + +But there is a workaround: By adding a new search engine with a different +language, SearXNG will search in your default and other language. + +Example configuration in settings.yml for a German and English speaker: + +.. code-block:: yaml + + search: + default_lang : "de" + ... + + engines: + - name : google english + engine : google + language : en + ... + +When searching, the default google engine will return German results and +"google english" will return English results. + diff --git a/docs/admin/settings/settings_general.rst b/docs/admin/settings/settings_general.rst new file mode 100644 index 000000000..85cc8ef63 --- /dev/null +++ b/docs/admin/settings/settings_general.rst @@ -0,0 +1,34 @@ +.. _settings general: + +============ +``general:`` +============ + +.. code:: yaml + + general: + debug: false + instance_name: "SearXNG" + privacypolicy_url: false + donation_url: false + contact_url: false + enable_metrics: true + +``debug`` : ``$SEARXNG_DEBUG`` + Allow a more detailed log if you run SearXNG directly. Display *detailed* error + messages in the browser too, so this must be deactivated in production. + +``donation_url`` : + Set value to ``true`` to use your own donation page written in the + :ref:`searx/info/en/donate.md ` and use ``false`` to disable + the donation link altogether. + +``privacypolicy_url``: + Link to privacy policy. + +``contact_url``: + Contact ``mailto:`` address or WEB form. + +``enable_metrics``: + Enabled by default. Record various anonymous metrics availabled at ``/stats``, + ``/stats/errors`` and ``/preferences``. diff --git a/docs/admin/settings/settings_outgoing.rst b/docs/admin/settings/settings_outgoing.rst new file mode 100644 index 000000000..7d49ab789 --- /dev/null +++ b/docs/admin/settings/settings_outgoing.rst @@ -0,0 +1,110 @@ +.. _settings outgoing: + +============= +``outgoing:`` +============= + +Communication with search engines. + +.. code:: yaml + + outgoing: + request_timeout: 2.0 # default timeout in seconds, can be override by engine + max_request_timeout: 10.0 # the maximum timeout in seconds + useragent_suffix: "" # information like an email address to the administrator + pool_connections: 100 # Maximum number of allowable connections, or null + # for no limits. The default is 100. + pool_maxsize: 10 # Number of allowable keep-alive connections, or null + # to always allow. The default is 10. + enable_http2: true # See https://www.python-httpx.org/http2/ + # uncomment below section if you want to use a custom server certificate + # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults + # and https://www.python-httpx.org/compatibility/#ssl-configuration + # verify: ~/.mitmproxy/mitmproxy-ca-cert.cer + # + # uncomment below section if you want to use a proxyq see: SOCKS proxies + # https://2.python-requests.org/en/latest/user/advanced/#proxies + # are also supported: see + # https://2.python-requests.org/en/latest/user/advanced/#socks + # + # proxies: + # all://: + # - http://proxy1:8080 + # - http://proxy2:8080 + # + # using_tor_proxy: true + # + # Extra seconds to add in order to account for the time taken by the proxy + # + # extra_proxy_timeout: 10.0 + # + +``request_timeout`` : + Global timeout of the requests made to others engines in seconds. A bigger + timeout will allow to wait for answers from slow engines, but in consequence + will slow SearXNG reactivity (the result page may take the time specified in the + timeout to load). Can be override by ``timeout`` in the :ref:`settings engine`. + +``useragent_suffix`` : + Suffix to the user-agent SearXNG uses to send requests to others engines. If an + engine wish to block you, a contact info here may be useful to avoid that. + +.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration + +``pool_maxsize``: + Number of allowable keep-alive connections, or ``null`` to always allow. The + default is 10. See ``max_keepalive_connections`` `Pool limit configuration`_. + +``pool_connections`` : + Maximum number of allowable connections, or ``null`` # for no limits. The + default is 100. See ``max_connections`` `Pool limit configuration`_. + +``keepalive_expiry`` : + Number of seconds to keep a connection in the pool. By default 5.0 seconds. + See ``keepalive_expiry`` `Pool limit configuration`_. + +.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying + +``proxies`` : + Define one or more proxies you wish to use, see `httpx proxies`_. + If there are more than one proxy for one protocol (http, https), + requests to the engines are distributed in a round-robin fashion. + +``source_ips`` : + If you use multiple network interfaces, define from which IP the requests must + be made. Example: + + * ``0.0.0.0`` any local IPv4 address. + * ``::`` any local IPv6 address. + * ``192.168.0.1`` + * ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses + * ``fe80::60a2:1691:e5a2:ee1f`` + * ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network. + * ``[ 192.168.0.1, fe80::/126 ]`` + +``retries`` : + Number of retry in case of an HTTP error. On each retry, SearXNG uses an + different proxy and source ip. + +``enable_http2`` : + Enable by default. Set to ``false`` to disable HTTP/2. + +.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults +.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration + +``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR`` + Allow to specify a path to certificate. + see `httpx verification defaults`_. + + In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and + ``$SSL_CERT_DIR`` (for a directory) OpenSSL variables. + see `httpx ssl configuration`_. + +``max_redirects`` : + 30 by default. Maximum redirect before it is an error. + +``using_tor_proxy`` : + Using tor proxy (``true``) or not (``false``) for all engines. The default is + ``false`` and can be overwritten in the :ref:`settings engine` + + diff --git a/docs/admin/settings/settings_redis.rst b/docs/admin/settings/settings_redis.rst new file mode 100644 index 000000000..49640f314 --- /dev/null +++ b/docs/admin/settings/settings_redis.rst @@ -0,0 +1,43 @@ +.. _settings redis: + +========== +``redis:`` +========== + +.. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url + +A redis DB can be connected by an URL, in :py:obj:`searx.redisdb` you +will find a description to test your redis connection in SerXNG. When using +sockets, don't forget to check the access rights on the socket:: + + ls -la /usr/local/searxng-redis/run/redis.sock + srwxrwx--- 1 searxng-redis searxng-redis ... /usr/local/searxng-redis/run/redis.sock + +In this example read/write access is given to the *searxng-redis* group. To get +access rights to redis instance (the socket), your SearXNG (or even your +developer) account needs to be added to the *searxng-redis* group. + +``url`` : ``$SEARXNG_REDIS_URL`` + URL to connect redis database, see `Redis.from_url(url)`_ & :ref:`redis db`:: + + redis://[[username]:[password]]@localhost:6379/0 + rediss://[[username]:[password]]@localhost:6379/0 + unix://[[username]:[password]]@/path/to/socket.sock?db=0 + +.. admonition:: Tip for developers + + To set up a local redis instance, first set the socket path of the Redis DB + in your YAML setting: + + .. code:: yaml + + redis: + url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 + + Then use the following commands to install the redis instance :: + + $ ./manage redis.build + $ sudo -H ./manage redis.install + $ sudo -H ./manage redis.addgrp "${USER}" + # don't forget to logout & login to get member of group + diff --git a/docs/admin/settings/settings_search.rst b/docs/admin/settings/settings_search.rst new file mode 100644 index 000000000..dcc0ff198 --- /dev/null +++ b/docs/admin/settings/settings_search.rst @@ -0,0 +1,97 @@ +.. _settings search: + +=========== +``search:`` +=========== + +.. code:: yaml + + search: + safe_search: 0 + autocomplete: "" + default_lang: "" + ban_time_on_fail: 5 + max_ban_time_on_fail: 120 + suspended_times: + SearxEngineAccessDenied: 86400 + SearxEngineCaptcha: 86400 + SearxEngineTooManyRequests: 3600 + cf_SearxEngineCaptcha: 1296000 + cf_SearxEngineAccessDenied: 86400 + recaptcha_SearxEngineCaptcha: 604800 + formats: + - html + +``safe_search``: + Filter results. + + - ``0``: None + - ``1``: Moderate + - ``2``: Strict + +``autocomplete``: + Existing autocomplete backends, leave blank to turn it off. + + - ``dbpedia`` + - ``duckduckgo`` + - ``google`` + - ``startpage`` + - ``swisscows`` + - ``qwant`` + - ``wikipedia`` + +``default_lang``: + Default search language - leave blank to detect from browser information or + use codes from :origin:`searx/languages.py`. + +``languages``: + List of available languages - leave unset to use all codes from + :origin:`searx/languages.py`. Otherwise list codes of available languages. + The ``all`` value is shown as the ``Default language`` in the user interface + (in most cases, it is meant to send the query without a language parameter ; + in some cases, it means the English language) Example: + + .. code:: yaml + + languages: + - all + - en + - en-US + - de + - it-IT + - fr + - fr-BE + +``ban_time_on_fail``: + Ban time in seconds after engine errors. + +``max_ban_time_on_fail``: + Max ban time in seconds after engine errors. + +``suspended_times``: + Engine suspension time after error (in seconds; set to 0 to disable) + + ``SearxEngineAccessDenied``: 86400 + For error "Access denied" and "HTTP error [402, 403]" + + ``SearxEngineCaptcha``: 86400 + For error "CAPTCHA" + + ``SearxEngineTooManyRequests``: 3600 + For error "Too many request" and "HTTP error 429" + + Cloudflare CAPTCHA: + - ``cf_SearxEngineCaptcha``: 1296000 + - ``cf_SearxEngineAccessDenied``: 86400 + + Google CAPTCHA: + - ``recaptcha_SearxEngineCaptcha``: 604800 + +``formats``: + Result formats available from web, remove format to deny access (use lower + case). + + - ``html`` + - ``csv`` + - ``json`` + - ``rss`` diff --git a/docs/admin/settings/settings_server.rst b/docs/admin/settings/settings_server.rst new file mode 100644 index 000000000..e2b4cb67d --- /dev/null +++ b/docs/admin/settings/settings_server.rst @@ -0,0 +1,54 @@ +.. _settings server: + +=========== +``server:`` +=========== + +.. code:: yaml + + server: + base_url: http://example.org/location # change this! + port: 8888 + bind_address: "127.0.0.1" + secret_key: "ultrasecretkey" # change this! + limiter: false + image_proxy: false + default_http_headers: + X-Content-Type-Options : nosniff + X-XSS-Protection : 1; mode=block + X-Download-Options : noopen + X-Robots-Tag : noindex, nofollow + Referrer-Policy : no-referrer + + +``base_url`` : ``$SEARXNG_URL`` :ref:`buildenv ` + The base URL where SearXNG is deployed. Used to create correct inbound links. + If you change the value, don't forget to rebuild instance's environment + (:ref:`utils/brand.env `) + +``port`` & ``bind_address``: ``$SEARXNG_PORT`` & ``$SEARXNG_BIND_ADDRESS`` :ref:`buildenv ` + Port number and *bind address* of the SearXNG web application if you run it + directly using ``python searx/webapp.py``. Doesn't apply to a SearXNG + services running behind a proxy and using socket communications. If you + change the value, don't forget to rebuild instance's environment + (:ref:`utils/brand.env `) + +``secret_key`` : ``$SEARXNG_SECRET`` + Used for cryptography purpose. + +.. _limiter: + +``limiter`` : + Rate limit the number of request on the instance, block some bots. The + :ref:`limiter src` requires a :ref:`settings redis` database. + +.. _image_proxy: + +``image_proxy`` : + Allow your instance of SearXNG of being able to proxy images. Uses memory space. + +.. _HTTP headers: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers + +``default_http_headers`` : + Set additional HTTP headers, see `#755 `__ + diff --git a/docs/admin/settings/settings_ui.rst b/docs/admin/settings/settings_ui.rst new file mode 100644 index 000000000..e355199a2 --- /dev/null +++ b/docs/admin/settings/settings_ui.rst @@ -0,0 +1,62 @@ +.. _settings ui: + +======= +``ui:`` +======= + +.. _cache busting: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#caching_static_assets_with_cache_busting + +.. code:: yaml + + ui: + static_use_hash: false + default_locale: "" + query_in_title: false + infinite_scroll: false + center_alignment: false + cache_url: https://web.archive.org/web/ + default_theme: simple + theme_args: + simple_style: auto + +.. _static_use_hash: + +``static_use_hash`` : + Enables `cache busting`_ of static files. + +``default_locale`` : + SearXNG interface language. If blank, the locale is detected by using the + browser language. If it doesn't work, or you are deploying a language + specific instance of searx, a locale can be defined using an ISO language + code, like ``fr``, ``en``, ``de``. + +``query_in_title`` : + When true, the result page's titles contains the query it decreases the + privacy, since the browser can records the page titles. + +``infinite_scroll``: + When true, automatically loads the next page when scrolling to bottom of the current page. + +``center_alignment`` : default ``false`` + When enabled, the results are centered instead of being in the left (or RTL) + side of the screen. This setting only affects the *desktop layout* + (:origin:`min-width: @tablet `) + +.. cache_url: + +``cache_url`` : ``https://web.archive.org/web/`` + URL prefix of the internet archive or cache, don't forgett trailing slash (if + needed). The default is https://web.archive.org/web/ alternatives are: + + - https://webcache.googleusercontent.com/search?q=cache: + - https://archive.today/ + +``default_theme`` : + Name of the theme you want to use by default on your SearXNG instance. + +``theme_args.simple_style``: + Style of simple theme: ``auto``, ``light``, ``dark`` + +``results_on_new_tab``: + Open result links in a new tab by default. diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index 627ed03e8..27c0108d4 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -9,7 +9,7 @@ SearXNG maintenance - :ref:`toolboxing` - :ref:`uWSGI maintenance` -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 0a2f07e40..67133b6b3 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -4,7 +4,7 @@ How to contribute ================= -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst deleted file mode 100644 index 5fd31c01d..000000000 --- a/docs/dev/engine_overview.rst +++ /dev/null @@ -1,432 +0,0 @@ -.. _engines-dev: - -=============== -Engine Overview -=============== - -.. _metasearch-engine: https://en.wikipedia.org/wiki/Metasearch_engine - -.. sidebar:: Further reading .. - - - :ref:`configured engines` - - :ref:`settings engine` - -.. contents:: - :depth: 3 - :backlinks: entry - -SearXNG is a metasearch-engine_, so it uses different search engines to provide -better results. - -Because there is no general search API which could be used for every search -engine, an adapter has to be built between SearXNG and the external search -engines. Adapters are stored under the folder :origin:`searx/engines`. - -.. _general engine configuration: - -General Engine Configuration -============================ - -It is required to tell SearXNG the type of results the engine provides. The -arguments can be set in the engine file or in the settings file (normally -``settings.yml``). The arguments in the settings file override the ones in the -engine file. - -It does not matter if an option is stored in the engine file or in the settings. -However, the standard way is the following: - -.. _engine file: - -Engine File ------------ - -.. table:: Common options in the engine module - :width: 100% - - ======================= =========== ======================================================== - argument type information - ======================= =========== ======================================================== - categories list categories, in which the engine is working - paging boolean support multiple pages - time_range_support boolean support search time range - engine_type str - ``online`` :ref:`[ref] ` by - default, other possibles values are: - - ``offline`` :ref:`[ref] ` - - ``online_dictionary`` - - ``online_currency`` - - ``online_url_search`` - ======================= =========== ======================================================== - -.. _engine settings: - -Engine ``settings.yml`` ------------------------ - -For a more detailed description, see :ref:`settings engine` in the :ref:`settings.yml`. - -.. table:: Common options in the engine setup (``settings.yml``) - :width: 100% - - ======================= =========== ================================================== - argument type information - ======================= =========== ================================================== - name string name of search-engine - engine string name of searxng-engine (file name without ``.py``) - enable_http bool enable HTTP (by default only HTTPS is enabled). - shortcut string shortcut of search-engine - timeout string specific timeout for search-engine - display_error_messages boolean display error messages on the web UI - proxies dict set proxies for a specific engine - (e.g. ``proxies : {http: socks5://proxy:port, - https: socks5://proxy:port}``) - ======================= =========== ================================================== - -.. _engine overrides: - -Overrides ---------- - -A few of the options have default values in the namespace of engine's python -modul, but are often overwritten by the settings. If ``None`` is assigned to an -option in the engine file, it has to be redefined in the settings, otherwise -SearXNG will not start with that engine (global names with a leading underline can -be ``None``). - -Here is an very simple example of the global names in the namespace of engine's -module: - -.. code:: python - - # engine dependent config - categories = ['general'] - paging = True - _non_overwritten_global = 'foo' - - -.. table:: The naming of overrides is arbitrary / recommended overrides are: - :width: 100% - - ======================= =========== =========================================== - argument type information - ======================= =========== =========================================== - base_url string base-url, can be overwritten to use same - engine on other URL - number_of_results int maximum number of results per request - language string ISO code of language and country like en_US - api_key string api-key if required by engine - ======================= =========== =========================================== - -.. _engine request: - -Making a Request -================ - -To perform a search an URL have to be specified. In addition to specifying an -URL, arguments can be passed to the query. - -.. _engine request arguments: - -Passed Arguments (request) --------------------------- - -These arguments can be used to construct the search query. Furthermore, -parameters with default value can be redefined for special purposes. - -.. _engine request online: - -.. table:: If the ``engine_type`` is :py:obj:`online - ` - :width: 100% - - ====================== ============== ======================================================================== - argument type default-value, information - ====================== ============== ======================================================================== - url str ``''`` - method str ``'GET'`` - headers set ``{}`` - data set ``{}`` - cookies set ``{}`` - verify bool ``True`` - headers.User-Agent str a random User-Agent - category str current category, like ``'general'`` - safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict) - time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year`` - pageno int current pagenumber - searxng_locale str SearXNG's locale selected by user. Specific language code like - ``'en'``, ``'en-US'``, or ``'all'`` if unspecified. - ====================== ============== ======================================================================== - - -.. _engine request online_dictionary: - -.. table:: If the ``engine_type`` is :py:obj:`online_dictionary - `, - in addition to the :ref:`online ` arguments: - :width: 100% - - ====================== ============== ======================================================================== - argument type default-value, information - ====================== ============== ======================================================================== - from_lang str specific language code like ``'en_US'`` - to_lang str specific language code like ``'en_US'`` - query str the text query without the languages - ====================== ============== ======================================================================== - -.. _engine request online_currency: - -.. table:: If the ``engine_type`` is :py:obj:`online_currency - `, - in addition to the :ref:`online ` arguments: - :width: 100% - - ====================== ============== ======================================================================== - argument type default-value, information - ====================== ============== ======================================================================== - amount float the amount to convert - from str ISO 4217 code - to str ISO 4217 code - from_name str currency name - to_name str currency name - ====================== ============== ======================================================================== - -.. _engine request online_url_search: - -.. table:: If the ``engine_type`` is :py:obj:`online_url_search - `, - in addition to the :ref:`online ` arguments: - :width: 100% - - ====================== ============== ======================================================================== - argument type default-value, information - ====================== ============== ======================================================================== - search_url dict URLs from the search query: - - .. code:: python - - { - 'http': str, - 'ftp': str, - 'data:image': str - } - ====================== ============== ======================================================================== - -Specify Request ---------------- - -The function :py:func:`def request(query, params): -` always returns the ``params`` variable, the -following parameters can be used to specify a search request: - -.. table:: - :width: 100% - - =================== =========== ========================================================================== - argument type information - =================== =========== ========================================================================== - url str requested url - method str HTTP request method - headers set HTTP header information - data set HTTP data information - cookies set HTTP cookies - verify bool Performing SSL-Validity check - allow_redirects bool Follow redirects - max_redirects int maximum redirects, hard limit - soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine - raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300 - =================== =========== ========================================================================== - - -.. _engine results: -.. _engine media types: - -Media Types -=========== - -Each result item of an engine can be of different media-types. Currently the -following media-types are supported. To set another media-type as ``default``, -the parameter ``template`` must be set to the desired type. - -.. table:: Parameter of the **default** media type: - :width: 100% - - ========================= ===================================================== - result-parameter information - ========================= ===================================================== - url string, url of the result - title string, title of the result - content string, general result-text - publishedDate :py:class:`datetime.datetime`, time of publish - ========================= ===================================================== - - -.. table:: Parameter of the **images** media type: - :width: 100% - - ========================= ===================================================== - result-parameter information - ------------------------- ----------------------------------------------------- - template is set to ``images.html`` - ========================= ===================================================== - url string, url to the result site - title string, title of the result *(partly implemented)* - content *(partly implemented)* - publishedDate :py:class:`datetime.datetime`, - time of publish *(partly implemented)* - img\_src string, url to the result image - thumbnail\_src string, url to a small-preview image - ========================= ===================================================== - - -.. table:: Parameter of the **videos** media type: - :width: 100% - - ========================= ===================================================== - result-parameter information - ------------------------- ----------------------------------------------------- - template is set to ``videos.html`` - ========================= ===================================================== - url string, url of the result - title string, title of the result - content *(not implemented yet)* - publishedDate :py:class:`datetime.datetime`, time of publish - thumbnail string, url to a small-preview image - ========================= ===================================================== - -.. _magnetlink: https://en.wikipedia.org/wiki/Magnet_URI_scheme - -.. table:: Parameter of the **torrent** media type: - :width: 100% - - ========================= ===================================================== - result-parameter information - ------------------------- ----------------------------------------------------- - template is set to ``torrent.html`` - ========================= ===================================================== - url string, url of the result - title string, title of the result - content string, general result-text - publishedDate :py:class:`datetime.datetime`, - time of publish *(not implemented yet)* - seed int, number of seeder - leech int, number of leecher - filesize int, size of file in bytes - files int, number of files - magnetlink string, magnetlink_ of the result - torrentfile string, torrentfile of the result - ========================= ===================================================== - -.. table:: Parameter of the **map** media type: - :width: 100% - - ========================= ===================================================== - result-parameter information - ------------------------- ----------------------------------------------------- - template is set to ``map.html`` - ========================= ===================================================== - url string, url of the result - title string, title of the result - content string, general result-text - publishedDate :py:class:`datetime.datetime`, time of publish - latitude latitude of result (in decimal format) - longitude longitude of result (in decimal format) - boundingbox boundingbox of result (array of 4. values - ``[lat-min, lat-max, lon-min, lon-max]``) - geojson geojson of result (https://geojson.org/) - osm.type type of osm-object (if OSM-Result) - osm.id id of osm-object (if OSM-Result) - address.name name of object - address.road street name of object - address.house_number house number of object - address.locality city, place of object - address.postcode postcode of object - address.country country of object - ========================= ===================================================== - -.. _BibTeX format: https://www.bibtex.com/g/bibtex-format/ -.. _BibTeX field types: https://en.wikipedia.org/wiki/BibTeX#Field_types - -.. list-table:: Parameter of the **paper** media type / - see `BibTeX field types`_ and `BibTeX format`_ - :header-rows: 2 - :width: 100% - - * - result-parameter - - Python type - - information - - * - template - - :py:class:`str` - - is set to ``paper.html`` - - * - title - - :py:class:`str` - - title of the result - - * - content - - :py:class:`str` - - abstract - - * - comments - - :py:class:`str` - - free text display in italic below the content - - * - tags - - :py:class:`List `\ [\ :py:class:`str`\ ] - - free tag list - - * - publishedDate - - :py:class:`datetime ` - - last publication date - - * - type - - :py:class:`str` - - short description of medium type, e.g. *book*, *pdf* or *html* ... - - * - authors - - :py:class:`List `\ [\ :py:class:`str`\ ] - - list of authors of the work (authors with a "s") - - * - editor - - :py:class:`str` - - list of editors of a book - - * - publisher - - :py:class:`str` - - name of the publisher - - * - journal - - :py:class:`str` - - name of the journal or magazine the article was - published in - - * - volume - - :py:class:`str` - - volume number - - * - pages - - :py:class:`str` - - page range where the article is - - * - number - - :py:class:`str` - - number of the report or the issue number for a journal article - - * - doi - - :py:class:`str` - - DOI number (like ``10.1038/d41586-018-07848-2``) - - * - issn - - :py:class:`List `\ [\ :py:class:`str`\ ] - - ISSN number like ``1476-4687`` - - * - isbn - - :py:class:`List `\ [\ :py:class:`str`\ ] - - ISBN number like ``9780201896831`` - - * - pdf_url - - :py:class:`str` - - URL to the full article, the PDF version - - * - html_url - - :py:class:`str` - - URL to full article, HTML version diff --git a/docs/dev/engines/demo/demo_offline.rst b/docs/dev/engines/demo/demo_offline.rst new file mode 100644 index 000000000..1b4cb887f --- /dev/null +++ b/docs/dev/engines/demo/demo_offline.rst @@ -0,0 +1,14 @@ +.. _demo offline engine: + +=================== +Demo Offline Engine +=================== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.demo_offline + :members: + diff --git a/docs/dev/engines/demo/demo_online.rst b/docs/dev/engines/demo/demo_online.rst new file mode 100644 index 000000000..9b94207f7 --- /dev/null +++ b/docs/dev/engines/demo/demo_online.rst @@ -0,0 +1,14 @@ +.. _demo online engine: + +================== +Demo Online Engine +================== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.demo_online + :members: + diff --git a/docs/dev/engines/engine_overview.rst b/docs/dev/engines/engine_overview.rst new file mode 100644 index 000000000..8052a8bea --- /dev/null +++ b/docs/dev/engines/engine_overview.rst @@ -0,0 +1,468 @@ +.. _engines-dev: + +=============== +Engine Overview +=============== + +.. contents:: + :depth: 3 + :local: + :backlinks: entry + +.. _metasearch-engine: https://en.wikipedia.org/wiki/Metasearch_engine + +.. sidebar:: Further reading .. + + - :ref:`configured engines` + - :ref:`settings engine` + +SearXNG is a metasearch-engine_, so it uses different search engines to provide +better results. + +Because there is no general search API which could be used for every search +engine, an adapter has to be built between SearXNG and the external search +engines. Adapters are stored under the folder :origin:`searx/engines`. + +.. _general engine configuration: + +General Engine Configuration +============================ + +It is required to tell SearXNG the type of results the engine provides. The +arguments can be set in the engine file or in the settings file (normally +``settings.yml``). The arguments in the settings file override the ones in the +engine file. + +It does not matter if an option is stored in the engine file or in the settings. +However, the standard way is the following: + +.. _engine file: + +Engine File +----------- + +.. table:: Common options in the engine module + :width: 100% + + ======================= =========== ======================================================== + argument type information + ======================= =========== ======================================================== + categories list categories, in which the engine is working + paging boolean support multiple pages + time_range_support boolean support search time range + engine_type str - ``online`` :ref:`[ref] ` by + default, other possibles values are: + - ``offline`` :ref:`[ref] ` + - ``online_dictionary`` :ref:`[ref] ` + - ``online_currency`` :ref:`[ref] ` + - ``online_url_search`` :ref:`[ref] ` + ======================= =========== ======================================================== + +.. _engine settings: + +Engine ``settings.yml`` +----------------------- + +For a more detailed description, see :ref:`settings engine` in the :ref:`settings.yml`. + +.. table:: Common options in the engine setup (``settings.yml``) + :width: 100% + + ======================= =========== ================================================== + argument type information + ======================= =========== ================================================== + name string name of search-engine + engine string name of searxng-engine (file name without ``.py``) + enable_http bool enable HTTP (by default only HTTPS is enabled). + shortcut string shortcut of search-engine + timeout string specific timeout for search-engine + display_error_messages boolean display error messages on the web UI + proxies dict set proxies for a specific engine + (e.g. ``proxies : {http: socks5://proxy:port, + https: socks5://proxy:port}``) + ======================= =========== ================================================== + +.. _engine overrides: + +Overrides +--------- + +A few of the options have default values in the namespace of engine's python +modul, but are often overwritten by the settings. If ``None`` is assigned to an +option in the engine file, it has to be redefined in the settings, otherwise +SearXNG will not start with that engine (global names with a leading underline can +be ``None``). + +Here is an very simple example of the global names in the namespace of engine's +module: + +.. code:: python + + # engine dependent config + categories = ['general'] + paging = True + _non_overwritten_global = 'foo' + + +.. table:: The naming of overrides is arbitrary / recommended overrides are: + :width: 100% + + ======================= =========== =========================================== + argument type information + ======================= =========== =========================================== + base_url string base-url, can be overwritten to use same + engine on other URL + number_of_results int maximum number of results per request + language string ISO code of language and country like en_US + api_key string api-key if required by engine + ======================= =========== =========================================== + +.. _engine request: + +Making a Request +================ + +To perform a search an URL have to be specified. In addition to specifying an +URL, arguments can be passed to the query. + +.. _engine request arguments: + +Passed Arguments (request) +-------------------------- + +These arguments can be used to construct the search query. Furthermore, +parameters with default value can be redefined for special purposes. + +.. _engine request online: + +.. table:: If the ``engine_type`` is :py:obj:`online + ` + :width: 100% + + ====================== ============== ======================================================================== + argument type default-value, information + ====================== ============== ======================================================================== + url str ``''`` + method str ``'GET'`` + headers set ``{}`` + data set ``{}`` + cookies set ``{}`` + verify bool ``True`` + headers.User-Agent str a random User-Agent + category str current category, like ``'general'`` + safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict) + time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year`` + pageno int current pagenumber + searxng_locale str SearXNG's locale selected by user. Specific language code like + ``'en'``, ``'en-US'``, or ``'all'`` if unspecified. + ====================== ============== ======================================================================== + + +.. _engine request online_dictionary: + +.. table:: If the ``engine_type`` is :py:obj:`online_dictionary + `, + in addition to the :ref:`online ` arguments: + :width: 100% + + ====================== ============== ======================================================================== + argument type default-value, information + ====================== ============== ======================================================================== + from_lang str specific language code like ``'en_US'`` + to_lang str specific language code like ``'en_US'`` + query str the text query without the languages + ====================== ============== ======================================================================== + +.. _engine request online_currency: + +.. table:: If the ``engine_type`` is :py:obj:`online_currency + `, + in addition to the :ref:`online ` arguments: + :width: 100% + + ====================== ============== ======================================================================== + argument type default-value, information + ====================== ============== ======================================================================== + amount float the amount to convert + from str ISO 4217 code + to str ISO 4217 code + from_name str currency name + to_name str currency name + ====================== ============== ======================================================================== + +.. _engine request online_url_search: + +.. table:: If the ``engine_type`` is :py:obj:`online_url_search + `, + in addition to the :ref:`online ` arguments: + :width: 100% + + ====================== ============== ======================================================================== + argument type default-value, information + ====================== ============== ======================================================================== + search_url dict URLs from the search query: + + .. code:: python + + { + 'http': str, + 'ftp': str, + 'data:image': str + } + ====================== ============== ======================================================================== + +Specify Request +--------------- + +The function :py:func:`def request(query, params): +` always returns the ``params`` variable, the +following parameters can be used to specify a search request: + +.. table:: + :width: 100% + + =================== =========== ========================================================================== + argument type information + =================== =========== ========================================================================== + url str requested url + method str HTTP request method + headers set HTTP header information + data set HTTP data information + cookies set HTTP cookies + verify bool Performing SSL-Validity check + allow_redirects bool Follow redirects + max_redirects int maximum redirects, hard limit + soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine + raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300 + =================== =========== ========================================================================== + + +.. _engine results: +.. _engine media types: + +Result Types (``template``) +=========================== + +Each result item of an engine can be of different media-types. Currently the +following media-types are supported. To set another media-type as +:ref:`template default`, the parameter ``template`` must be set to the desired +type. + +.. _template default: + +``default`` +----------- + +.. table:: Parameter of the **default** media type: + :width: 100% + + ========================= ===================================================== + result-parameter information + ========================= ===================================================== + url string, url of the result + title string, title of the result + content string, general result-text + publishedDate :py:class:`datetime.datetime`, time of publish + ========================= ===================================================== + + +.. _template images: + +``images`` +---------- + +.. table:: Parameter of the **images** media type: + :width: 100% + + ========================= ===================================================== + result-parameter information + ------------------------- ----------------------------------------------------- + template is set to ``images.html`` + ========================= ===================================================== + url string, url to the result site + title string, title of the result *(partly implemented)* + content *(partly implemented)* + publishedDate :py:class:`datetime.datetime`, + time of publish *(partly implemented)* + img\_src string, url to the result image + thumbnail\_src string, url to a small-preview image + ========================= ===================================================== + + +.. _template videos: + +``videos`` +---------- + +.. table:: Parameter of the **videos** media type: + :width: 100% + + ========================= ===================================================== + result-parameter information + ------------------------- ----------------------------------------------------- + template is set to ``videos.html`` + ========================= ===================================================== + url string, url of the result + title string, title of the result + content *(not implemented yet)* + publishedDate :py:class:`datetime.datetime`, time of publish + thumbnail string, url to a small-preview image + ========================= ===================================================== + + +.. _template torrent: + +``torrent`` +----------- + +.. _magnetlink: https://en.wikipedia.org/wiki/Magnet_URI_scheme + +.. table:: Parameter of the **torrent** media type: + :width: 100% + + ========================= ===================================================== + result-parameter information + ------------------------- ----------------------------------------------------- + template is set to ``torrent.html`` + ========================= ===================================================== + url string, url of the result + title string, title of the result + content string, general result-text + publishedDate :py:class:`datetime.datetime`, + time of publish *(not implemented yet)* + seed int, number of seeder + leech int, number of leecher + filesize int, size of file in bytes + files int, number of files + magnetlink string, magnetlink_ of the result + torrentfile string, torrentfile of the result + ========================= ===================================================== + + +.. _template map: + +``map`` +------- + +.. table:: Parameter of the **map** media type: + :width: 100% + + ========================= ===================================================== + result-parameter information + ------------------------- ----------------------------------------------------- + template is set to ``map.html`` + ========================= ===================================================== + url string, url of the result + title string, title of the result + content string, general result-text + publishedDate :py:class:`datetime.datetime`, time of publish + latitude latitude of result (in decimal format) + longitude longitude of result (in decimal format) + boundingbox boundingbox of result (array of 4. values + ``[lat-min, lat-max, lon-min, lon-max]``) + geojson geojson of result (https://geojson.org/) + osm.type type of osm-object (if OSM-Result) + osm.id id of osm-object (if OSM-Result) + address.name name of object + address.road street name of object + address.house_number house number of object + address.locality city, place of object + address.postcode postcode of object + address.country country of object + ========================= ===================================================== + + +.. _template paper: + +``paper`` +--------- + +.. _BibTeX format: https://www.bibtex.com/g/bibtex-format/ +.. _BibTeX field types: https://en.wikipedia.org/wiki/BibTeX#Field_types + +.. list-table:: Parameter of the **paper** media type / + see `BibTeX field types`_ and `BibTeX format`_ + :header-rows: 2 + :width: 100% + + * - result-parameter + - Python type + - information + + * - template + - :py:class:`str` + - is set to ``paper.html`` + + * - title + - :py:class:`str` + - title of the result + + * - content + - :py:class:`str` + - abstract + + * - comments + - :py:class:`str` + - free text display in italic below the content + + * - tags + - :py:class:`List `\ [\ :py:class:`str`\ ] + - free tag list + + * - publishedDate + - :py:class:`datetime ` + - last publication date + + * - type + - :py:class:`str` + - short description of medium type, e.g. *book*, *pdf* or *html* ... + + * - authors + - :py:class:`List `\ [\ :py:class:`str`\ ] + - list of authors of the work (authors with a "s") + + * - editor + - :py:class:`str` + - list of editors of a book + + * - publisher + - :py:class:`str` + - name of the publisher + + * - journal + - :py:class:`str` + - name of the journal or magazine the article was + published in + + * - volume + - :py:class:`str` + - volume number + + * - pages + - :py:class:`str` + - page range where the article is + + * - number + - :py:class:`str` + - number of the report or the issue number for a journal article + + * - doi + - :py:class:`str` + - DOI number (like ``10.1038/d41586-018-07848-2``) + + * - issn + - :py:class:`List `\ [\ :py:class:`str`\ ] + - ISSN number like ``1476-4687`` + + * - isbn + - :py:class:`List `\ [\ :py:class:`str`\ ] + - ISBN number like ``9780201896831`` + + * - pdf_url + - :py:class:`str` + - URL to the full article, the PDF version + + * - html_url + - :py:class:`str` + - URL to full article, HTML version + diff --git a/docs/dev/engines/enginelib.rst b/docs/dev/engines/enginelib.rst new file mode 100644 index 000000000..34e3250da --- /dev/null +++ b/docs/dev/engines/enginelib.rst @@ -0,0 +1,22 @@ +.. _searx.enginelib: + +============== +Engine Library +============== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.enginelib + :members: + +.. _searx.enginelib.traits: + + +Engine traits +============= + +.. automodule:: searx.enginelib.traits + :members: diff --git a/docs/dev/engines/engines.rst b/docs/dev/engines/engines.rst new file mode 100644 index 000000000..0f2cb1f22 --- /dev/null +++ b/docs/dev/engines/engines.rst @@ -0,0 +1,9 @@ +.. _searx.engines loader: + +======================== +SearXNG's engines loader +======================== + +.. automodule:: searx.engines + :members: + diff --git a/docs/dev/engines/index.rst b/docs/dev/engines/index.rst new file mode 100644 index 000000000..ca8f87b3c --- /dev/null +++ b/docs/dev/engines/index.rst @@ -0,0 +1,97 @@ +.. _engine implementations: + +====================== +Engine Implementations +====================== + +Framework Components +==================== + +.. toctree:: + :maxdepth: 2 + + enginelib + engines + engine_overview + + +Engine Types +============ + +The :py:obj:`engine_type ` of an engine +determines which :ref:`search processor ` is used by +the engine. + +In this section a list of the enignes that are documented is given, a complete +list of the engines can be found in the source under: :origin:`searx/engines`. + +.. _online engines: + +Online Engines +-------------- + +.. sidebar:: info + + - :py:obj:`processors.online ` + +.. toctree:: + :maxdepth: 1 + :glob: + + demo/demo_online + xpath + online/* + +.. _offline engines: + +Offline Engines +--------------- + +.. sidebar:: info + + - :py:obj:`processors.offline ` + +.. toctree:: + :maxdepth: 1 + :glob: + + offline_concept + demo/demo_offline + offline/* + +.. _online url search: + +Online URL Search +----------------- + +.. sidebar:: info + + - :py:obj:`processors.online_url_search ` + +.. toctree:: + :maxdepth: 1 + :glob: + + online_url_search/* + +.. _online currency: + +Online Currency +--------------- + +.. sidebar:: info + + - :py:obj:`processors.online_currency ` + +*no engine of this type is documented yet / comming soon* + +.. _online dictionary: + +Online Dictionary +----------------- + +.. sidebar:: info + + - :py:obj:`processors.online_dictionary ` + +*no engine of this type is documented yet / comming soon* diff --git a/docs/dev/engines/offline/command-line-engines.rst b/docs/dev/engines/offline/command-line-engines.rst new file mode 100644 index 000000000..0a80d698e --- /dev/null +++ b/docs/dev/engines/offline/command-line-engines.rst @@ -0,0 +1,23 @@ +.. _engine command: + +==================== +Command Line Engines +==================== + +.. sidebar:: info + + - :origin:`command.py ` + - :ref:`offline engines` + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. sidebar:: info + + Initial sponsored by `Search and Discovery Fund + `_ of `NLnet Foundation `_. + +.. automodule:: searx.engines.command + :members: diff --git a/docs/dev/engines/offline/nosql-engines.rst b/docs/dev/engines/offline/nosql-engines.rst new file mode 100644 index 000000000..76f5cfb61 --- /dev/null +++ b/docs/dev/engines/offline/nosql-engines.rst @@ -0,0 +1,97 @@ +.. _nosql engines: + +=============== +NoSQL databases +=============== + +.. sidebar:: further read + + - `NoSQL databases `_ + - `redis.io `_ + - `MongoDB `_ + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. sidebar:: info + + Initial sponsored by `Search and Discovery Fund + `_ of `NLnet Foundation `_. + +The following `NoSQL databases`_ are supported: + +- :ref:`engine redis_server` +- :ref:`engine mongodb` + +All of the engines above are just commented out in the :origin:`settings.yml +`, as you have to set various options and install +dependencies before using them. + +By default, the engines use the ``key-value`` template for displaying results / +see :origin:`simple ` +theme. If you are not satisfied with the original result layout, you can use +your own template, set ``result_template`` attribute to ``{template_name}`` and +place the templates at:: + + searx/templates/{theme_name}/result_templates/{template_name} + +Furthermore, if you do not wish to expose these engines on a public instance, you +can still add them and limit the access by setting ``tokens`` as described in +section :ref:`private engines`. + + +Extra Dependencies +================== + +For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to +install additional packages in Python's Virtual Environment of your SearXNG +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: + + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... + + +Configure the engines +===================== + +`NoSQL databases`_ are used for storing arbitrary data without first defining +their structure. + + +.. _engine redis_server: + +Redis Server +------------ + +.. _redis: https://github.com/andymccurdy/redis-py#installation + +.. sidebar:: info + + - ``pip install`` redis_ + - redis.io_ + - :origin:`redis_server.py ` + +.. automodule:: searx.engines.redis_server + :members: + + +.. _engine mongodb: + +MongoDB +------- + +.. _pymongo: https://github.com/mongodb/mongo-python-driver#installation + +.. sidebar:: info + + - ``pip install`` pymongo_ + - MongoDB_ + - :origin:`mongodb.py ` + + +.. automodule:: searx.engines.mongodb + :members: + diff --git a/docs/dev/engines/offline/search-indexer-engines.rst b/docs/dev/engines/offline/search-indexer-engines.rst new file mode 100644 index 000000000..fa92798cb --- /dev/null +++ b/docs/dev/engines/offline/search-indexer-engines.rst @@ -0,0 +1,62 @@ +================= +Local Search APIs +================= + +.. sidebar:: further read + + - `Comparison to alternatives + `_ + +.. contents:: + :depth: 1 + :local: + :backlinks: entry + +.. sidebar:: info + + Initial sponsored by `Search and Discovery Fund + `_ of `NLnet Foundation `_. + +Administrators might find themselves wanting to integrate locally running search +engines. The following ones are supported for now: + +* `Elasticsearch`_ +* `Meilisearch`_ +* `Solr`_ + +Each search engine is powerful, capable of full-text search. All of the engines +above are added to ``settings.yml`` just commented out, as you have to +``base_url`` for all them. + +Please note that if you are not using HTTPS to access these engines, you have to +enable HTTP requests by setting ``enable_http`` to ``True``. + +Furthermore, if you do not want to expose these engines on a public instance, +you can still add them and limit the access by setting ``tokens`` as described +in section :ref:`private engines`. + +.. _engine meilisearch: + +MeiliSearch +=========== + +.. automodule:: searx.engines.meilisearch + :members: + + +.. _engine elasticsearch: + +Elasticsearch +============= + +.. automodule:: searx.engines.elasticsearch + :members: + +.. _engine solr: + +Solr +==== + +.. automodule:: searx.engines.solr + :members: + diff --git a/docs/dev/engines/offline/sql-engines.rst b/docs/dev/engines/offline/sql-engines.rst new file mode 100644 index 000000000..f0f5add0b --- /dev/null +++ b/docs/dev/engines/offline/sql-engines.rst @@ -0,0 +1,121 @@ +.. _sql engines: + +=========== +SQL Engines +=========== + +.. sidebar:: further read + + - `SQLite `_ + - `PostgreSQL `_ + - `MySQL `_ + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. sidebar:: info + + Initial sponsored by `Search and Discovery Fund + `_ of `NLnet Foundation `_. + +With the *SQL engines* you can bind SQL databases into SearXNG. The following +Relational Database Management System (RDBMS) are supported: + +- :ref:`engine sqlite` +- :ref:`engine postgresql` +- :ref:`engine mysql_server` + +All of the engines above are just commented out in the :origin:`settings.yml +`, as you have to set the required attributes for the +engines, e.g. ``database:`` ... + +.. code:: yaml + + - name: ... + engine: {sqlite|postgresql|mysql_server} + database: ... + result_template: {template_name} + query_str: ... + +By default, the engines use the ``key-value`` template for displaying results / +see :origin:`simple ` +theme. If you are not satisfied with the original result layout, you can use +your own template, set ``result_template`` attribute to ``{template_name}`` and +place the templates at:: + + searx/templates/{theme_name}/result_templates/{template_name} + +If you do not wish to expose these engines on a public instance, you can still +add them and limit the access by setting ``tokens`` as described in section +:ref:`private engines`. + + +Extra Dependencies +================== + +For using :ref:`engine postgresql` or :ref:`engine mysql_server` you need to +install additional packages in Python's Virtual Environment of your SearXNG +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: + + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... + + +Configure the engines +===================== + +The configuration of the new database engines are similar. You must put a valid +SQL-SELECT query in ``query_str``. At the moment you can only bind at most one +parameter in your query. By setting the attribute ``limit`` you can define how +many results you want from the SQL server. Basically, it is the same as the +``LIMIT`` keyword in SQL. + +Please, do not include ``LIMIT`` or ``OFFSET`` in your SQL query as the engines +rely on these keywords during paging. If you want to configure the number of +returned results use the option ``limit``. + +.. _engine sqlite: + +SQLite +------ + +.. sidebar:: info + + - :origin:`sqlite.py ` + +.. automodule:: searx.engines.sqlite + :members: + + +.. _engine postgresql: + +PostgreSQL +---------- + +.. _psycopg2: https://www.psycopg.org/install + +.. sidebar:: info + + - :origin:`postgresql.py ` + - ``pip install`` `psycopg2-binary `_ + +.. automodule:: searx.engines.postgresql + :members: + +.. _engine mysql_server: + +MySQL +----- + +.. sidebar:: info + + - :origin:`mysql_server.py ` + - ``pip install`` :pypi:`mysql-connector-python ` + + +.. automodule:: searx.engines.mysql_server + :members: + diff --git a/docs/dev/engines/offline_concept.rst b/docs/dev/engines/offline_concept.rst new file mode 100644 index 000000000..ddb34fc60 --- /dev/null +++ b/docs/dev/engines/offline_concept.rst @@ -0,0 +1,69 @@ +=============== +Offline Concept +=============== + +.. sidebar:: offline engines + + - :ref:`demo offline engine` + - :ref:`engine command` + - :ref:`sql engines` + - :ref:`nosql engines` + - :py:obj:`searx.search.processors.offline` + +To extend the functionality of SearXNG, offline engines are going to be +introduced. An offline engine is an engine which does not need Internet +connection to perform a search and does not use HTTP to communicate. + +Offline engines can be configured, by adding those to the `engines` list of +:origin:`settings.yml `. An example skeleton for offline +engines can be found in :ref:`demo offline engine` (:origin:`demo_offline.py +`). + + +Programming Interface +===================== + +:py:func:`init(engine_settings=None) ` + All offline engines can have their own init function to setup the engine before + accepting requests. The function gets the settings from settings.yml as a + parameter. This function can be omitted, if there is no need to setup anything + in advance. + +:py:func:`search(query, params) ` + Each offline engine has a function named ``search``. This function is + responsible to perform a search and return the results in a presentable + format. (Where *presentable* means presentable by the selected result + template.) + + The return value is a list of results retrieved by the engine. + +Engine representation in ``/config`` + If an engine is offline, the attribute ``offline`` is set to ``True``. + +.. _offline requirements: + +Extra Dependencies +================== + +If an offline engine depends on an external tool, SearXNG does not install it by +default. When an administrator configures such engine and starts the instance, +the process returns an error with the list of missing dependencies. Also, +required dependencies will be added to the comment/description of the engine, so +admins can install packages in advance. + +If there is a need to install additional packages in *Python's Virtual +Environment* of your SearXNG instance you need to switch into the environment +(:ref:`searxng-src`) first, for this you can use :ref:`searxng.sh`:: + + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... + + +Private engines (Security) +========================== + +To limit the access to offline engines, if an instance is available publicly, +administrators can set token(s) for each of the :ref:`private engines`. If a +query contains a valid token, then SearXNG performs the requested private +search. If not, requests from an offline engines return errors. + diff --git a/docs/dev/engines/online/annas_archive.rst b/docs/dev/engines/online/annas_archive.rst new file mode 100644 index 000000000..db88e5069 --- /dev/null +++ b/docs/dev/engines/online/annas_archive.rst @@ -0,0 +1,13 @@ +.. _annas_archive engine: + +============== +Anna's Archive +============== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.annas_archive + :members: diff --git a/docs/dev/engines/online/archlinux.rst b/docs/dev/engines/online/archlinux.rst new file mode 100644 index 000000000..834fffa43 --- /dev/null +++ b/docs/dev/engines/online/archlinux.rst @@ -0,0 +1,14 @@ +.. _archlinux engine: + +========== +Arch Linux +========== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.archlinux + :members: + diff --git a/docs/dev/engines/online/bing.rst b/docs/dev/engines/online/bing.rst new file mode 100644 index 000000000..19c31aa80 --- /dev/null +++ b/docs/dev/engines/online/bing.rst @@ -0,0 +1,43 @@ +.. _bing engines: + +============ +Bing Engines +============ + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + + +.. _bing web engine: + +Bing WEB +======== + +.. automodule:: searx.engines.bing + :members: + +.. _bing images engine: + +Bing Images +=========== + +.. automodule:: searx.engines.bing_images + :members: + +.. _bing videos engine: + +Bing Videos +=========== + +.. automodule:: searx.engines.bing_videos + :members: + +.. _bing news engine: + +Bing News +========= + +.. automodule:: searx.engines.bing_news + :members: diff --git a/docs/dev/engines/online/dailymotion.rst b/docs/dev/engines/online/dailymotion.rst new file mode 100644 index 000000000..c661172e5 --- /dev/null +++ b/docs/dev/engines/online/dailymotion.rst @@ -0,0 +1,13 @@ +.. _dailymotion engine: + +=========== +Dailymotion +=========== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.dailymotion + :members: diff --git a/docs/dev/engines/online/duckduckgo.rst b/docs/dev/engines/online/duckduckgo.rst new file mode 100644 index 000000000..d0d18eb8a --- /dev/null +++ b/docs/dev/engines/online/duckduckgo.rst @@ -0,0 +1,22 @@ +.. _duckduckgo engines: + +================= +DukcDukGo Engines +================= + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.duckduckgo + :members: + +.. automodule:: searx.engines.duckduckgo_images + :members: + +.. automodule:: searx.engines.duckduckgo_definitions + :members: + +.. automodule:: searx.engines.duckduckgo_weather + :members: diff --git a/docs/dev/engines/online/google.rst b/docs/dev/engines/online/google.rst new file mode 100644 index 000000000..9085070bd --- /dev/null +++ b/docs/dev/engines/online/google.rst @@ -0,0 +1,76 @@ +.. _google engines: + +============== +Google Engines +============== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + + +.. _google API: + +Google API +========== + +.. _Query Parameter Definitions: + https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions + +SearXNG's implementation of the Google API is mainly done in +:py:obj:`get_google_info `. + +For detailed description of the *REST-full* API see: `Query Parameter +Definitions`_. The linked API documentation can sometimes be helpful during +reverse engineering. However, we cannot use it in the freely accessible WEB +services; not all parameters can be applied and some engines are more *special* +than other (e.g. :ref:`google news engine`). + + +.. _google web engine: + +Google WEB +========== + +.. automodule:: searx.engines.google + :members: + +.. _google autocomplete: + +Google Autocomplete +==================== + +.. autofunction:: searx.autocomplete.google_complete + +.. _google images engine: + +Google Images +============= + +.. automodule:: searx.engines.google_images + :members: + +.. _google videos engine: + +Google Videos +============= + +.. automodule:: searx.engines.google_videos + :members: + +.. _google news engine: + +Google News +=========== + +.. automodule:: searx.engines.google_news + :members: + +.. _google scholar engine: + +Google Scholar +============== + +.. automodule:: searx.engines.google_scholar + :members: diff --git a/docs/dev/engines/online/peertube.rst b/docs/dev/engines/online/peertube.rst new file mode 100644 index 000000000..bedf055fb --- /dev/null +++ b/docs/dev/engines/online/peertube.rst @@ -0,0 +1,27 @@ +.. _peertube engines: + +================ +Peertube Engines +================ + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + + +.. _peertube video engine: + +Peertube Video +============== + +.. automodule:: searx.engines.peertube + :members: + +.. _sepiasearch engine: + +SepiaSearch +=========== + +.. automodule:: searx.engines.sepiasearch + :members: diff --git a/docs/dev/engines/online/recoll.rst b/docs/dev/engines/online/recoll.rst new file mode 100644 index 000000000..2f1a1e4df --- /dev/null +++ b/docs/dev/engines/online/recoll.rst @@ -0,0 +1,13 @@ +.. _engine recoll: + +============= +Recoll Engine +============= + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.recoll + :members: diff --git a/docs/dev/engines/online/startpage.rst b/docs/dev/engines/online/startpage.rst new file mode 100644 index 000000000..89e3ad10b --- /dev/null +++ b/docs/dev/engines/online/startpage.rst @@ -0,0 +1,13 @@ +.. _startpage engines: + +================= +Startpage Engines +================= + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.startpage + :members: diff --git a/docs/dev/engines/online/torznab.rst b/docs/dev/engines/online/torznab.rst new file mode 100644 index 000000000..9056b60d0 --- /dev/null +++ b/docs/dev/engines/online/torznab.rst @@ -0,0 +1,13 @@ +.. _torznab engine: + +============== +Torznab WebAPI +============== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.torznab + :members: diff --git a/docs/dev/engines/online/wikipedia.rst b/docs/dev/engines/online/wikipedia.rst new file mode 100644 index 000000000..d4920f0f6 --- /dev/null +++ b/docs/dev/engines/online/wikipedia.rst @@ -0,0 +1,27 @@ +.. _wikimedia engines: + +========= +Wikimedia +========= + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + + +.. _wikipedia engine: + +Wikipedia +========= + +.. automodule:: searx.engines.wikipedia + :members: + +.. _wikidata engine: + +Wikidata +========= + +.. automodule:: searx.engines.wikidata + :members: diff --git a/docs/dev/engines/online/yahoo.rst b/docs/dev/engines/online/yahoo.rst new file mode 100644 index 000000000..96c1e2774 --- /dev/null +++ b/docs/dev/engines/online/yahoo.rst @@ -0,0 +1,13 @@ +.. _yahoo engine: + +============ +Yahoo Engine +============ + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.yahoo + :members: diff --git a/docs/dev/engines/online_url_search/tineye.rst b/docs/dev/engines/online_url_search/tineye.rst new file mode 100644 index 000000000..3f4db7e0e --- /dev/null +++ b/docs/dev/engines/online_url_search/tineye.rst @@ -0,0 +1,14 @@ +.. _tineye engine: + +====== +Tineye +====== + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.tineye + :members: + diff --git a/docs/dev/engines/xpath.rst b/docs/dev/engines/xpath.rst new file mode 100644 index 000000000..42c4d47b6 --- /dev/null +++ b/docs/dev/engines/xpath.rst @@ -0,0 +1,14 @@ +.. _xpath engine: + +============ +XPath Engine +============ + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.xpath + :members: + diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 39be0885c..aa0667c58 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -4,12 +4,10 @@ Developer documentation .. toctree:: :maxdepth: 2 - :caption: Contents quickstart contribution_guide - engine_overview - offline_engines + engines/index search_api plugins translation diff --git a/docs/dev/lxcdev.rst b/docs/dev/lxcdev.rst index ef603e9fd..854b193ff 100644 --- a/docs/dev/lxcdev.rst +++ b/docs/dev/lxcdev.rst @@ -22,7 +22,7 @@ In this article we will show, how you can make use of Linux Containers (LXC_) in section :ref:`internet connectivity docker`. -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index bdf7f0f49..ad5665de7 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -22,7 +22,7 @@ Calling the ``help`` target gives a first overview (``make help``): .. program-output:: bash -c "cd ..; make --no-print-directory help" -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry @@ -243,14 +243,14 @@ calling ``make clean`` stop all processes using the :ref:`make install` or We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute / Documentation ` section. If you want to edit the documentation read our :ref:`make docs.live` section. If you are working in your own brand, -adjust your :ref:`settings global`. +adjust your :ref:`settings brand`. .. _make docs.gh-pages: ``make docs.gh-pages`` ====================== -To deploy on github.io first adjust your :ref:`settings global`. For any +To deploy on github.io first adjust your :ref:`settings brand`. For any further read :ref:`deploy on github.io`. .. _make test: diff --git a/docs/dev/offline_engines.rst b/docs/dev/offline_engines.rst deleted file mode 100644 index bfb2664f5..000000000 --- a/docs/dev/offline_engines.rst +++ /dev/null @@ -1,78 +0,0 @@ -.. _offline engines: - -=============== -Offline Engines -=============== - -.. sidebar:: offline engines - - - :ref:`demo offline engine` - - :ref:`sql engines` - - :ref:`engine command` - - :origin:`Redis ` - -To extend the functionality of SearXNG, offline engines are going to be -introduced. An offline engine is an engine which does not need Internet -connection to perform a search and does not use HTTP to communicate. - -Offline engines can be configured, by adding those to the `engines` list of -:origin:`settings.yml `. An example skeleton for offline -engines can be found in :ref:`demo offline engine` (:origin:`demo_offline.py -`). - - -Programming Interface -===================== - -:py:func:`init(engine_settings=None) ` - All offline engines can have their own init function to setup the engine before - accepting requests. The function gets the settings from settings.yml as a - parameter. This function can be omitted, if there is no need to setup anything - in advance. - -:py:func:`search(query, params) ` - - Each offline engine has a function named ``search``. This function is - responsible to perform a search and return the results in a presentable - format. (Where *presentable* means presentable by the selected result - template.) - - The return value is a list of results retrieved by the engine. - -Engine representation in ``/config`` - If an engine is offline, the attribute ``offline`` is set to ``True``. - -.. _offline requirements: - -Extra Dependencies -================== - -If an offline engine depends on an external tool, SearXNG does not install it by -default. When an administrator configures such engine and starts the instance, -the process returns an error with the list of missing dependencies. Also, -required dependencies will be added to the comment/description of the engine, so -admins can install packages in advance. - -If there is a need to install additional packages in *Python's Virtual -Environment* of your SearXNG instance you need to switch into the environment -(:ref:`searxng-src`) first, for this you can use :ref:`searxng.sh`:: - - $ sudo utils/searxng.sh instance cmd bash - (searxng-pyenv)$ pip install ... - - -Private engines (Security) -========================== - -To limit the access to offline engines, if an instance is available publicly, -administrators can set token(s) for each of the :ref:`private engines`. If a -query contains a valid token, then SearXNG performs the requested private -search. If not, requests from an offline engines return errors. - - -Acknowledgement -=============== - -This development was sponsored by `Search and Discovery Fund -`_ of `NLnet Foundation `_ . - diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index b19a7096e..47af0b130 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -37,7 +37,7 @@ docs.live ` to build HTML while editing. - DOT_, `Graphviz's dot`_, Graphviz_ -.. contents:: Contents +.. contents:: :depth: 3 :local: :backlinks: entry diff --git a/docs/dev/searxng_extra/index.rst b/docs/dev/searxng_extra/index.rst index c2b5c312b..7bca3c0d4 100644 --- a/docs/dev/searxng_extra/index.rst +++ b/docs/dev/searxng_extra/index.rst @@ -9,7 +9,6 @@ developers. .. toctree:: :maxdepth: 2 - :caption: Contents update standalone_searx.py diff --git a/docs/index.rst b/docs/index.rst index 331591c7a..3db54734a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,7 +43,6 @@ If you don't trust anyone, you can set up your own, see :ref:`installation`. .. toctree:: :maxdepth: 2 - :caption: Contents user/index own-instance diff --git a/docs/own-instance.rst b/docs/own-instance.rst index d1124c389..62099ce23 100644 --- a/docs/own-instance.rst +++ b/docs/own-instance.rst @@ -7,7 +7,7 @@ Why use a private instance? \.\. is a common question among SearXNG users. Before answering this question, see what options a SearXNG user has. -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/src/index.rst b/docs/src/index.rst index efa7a509b..fd16e7635 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -8,7 +8,6 @@ every item from the source code, but we will add documentation when requested. .. toctree:: :maxdepth: 2 - :caption: Contents :glob: searx.* diff --git a/docs/src/searx.botdetection.rst b/docs/src/searx.botdetection.rst index 093414ec8..48261ceac 100644 --- a/docs/src/searx.botdetection.rst +++ b/docs/src/searx.botdetection.rst @@ -4,7 +4,7 @@ Bot Detection ============= -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/src/searx.engine.archlinux.rst b/docs/src/searx.engine.archlinux.rst deleted file mode 100644 index be48b1859..000000000 --- a/docs/src/searx.engine.archlinux.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _archlinux engine: - -========== -Arch Linux -========== - -.. automodule:: searx.engines.archlinux - :members: - diff --git a/docs/src/searx.engine.dailymotion.rst b/docs/src/searx.engine.dailymotion.rst deleted file mode 100644 index 84348e2d0..000000000 --- a/docs/src/searx.engine.dailymotion.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _dailymotion engine: - -=========== -Dailymotion -=========== - -.. automodule:: searx.engines.dailymotion - :members: diff --git a/docs/src/searx.engine.duckduckgo.rst b/docs/src/searx.engine.duckduckgo.rst deleted file mode 100644 index 1646d4984..000000000 --- a/docs/src/searx.engine.duckduckgo.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. _duckduckgo engines: - -================= -DukcDukGo engines -================= - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - -.. automodule:: searx.engines.duckduckgo - :members: - -.. automodule:: searx.engines.duckduckgo_images - :members: - -.. automodule:: searx.engines.duckduckgo_definitions - :members: - -.. automodule:: searx.engines.duckduckgo_weather - :members: diff --git a/docs/src/searx.enginelib.rst b/docs/src/searx.enginelib.rst deleted file mode 100644 index 651a04e68..000000000 --- a/docs/src/searx.enginelib.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _searx.enginelib: - -============ -Engine model -============ - -.. automodule:: searx.enginelib - :members: - -.. _searx.enginelib.traits: - -============= -Engine traits -============= - -.. automodule:: searx.enginelib.traits - :members: diff --git a/docs/src/searx.engines.annas_archive.rst b/docs/src/searx.engines.annas_archive.rst deleted file mode 100644 index 377f42082..000000000 --- a/docs/src/searx.engines.annas_archive.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. automodule:: searx.engines.annas_archive - :members: diff --git a/docs/src/searx.engines.bing.rst b/docs/src/searx.engines.bing.rst deleted file mode 100644 index 6b7bba8f2..000000000 --- a/docs/src/searx.engines.bing.rst +++ /dev/null @@ -1,43 +0,0 @@ -.. _bing engines: - -============ -Bing Engines -============ - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - - -.. _bing web engine: - -Bing WEB -======== - -.. automodule:: searx.engines.bing - :members: - -.. _bing images engine: - -Bing Images -=========== - -.. automodule:: searx.engines.bing_images - :members: - -.. _bing videos engine: - -Bing Videos -=========== - -.. automodule:: searx.engines.bing_videos - :members: - -.. _bing news engine: - -Bing News -========= - -.. automodule:: searx.engines.bing_news - :members: diff --git a/docs/src/searx.engines.demo_offline.rst b/docs/src/searx.engines.demo_offline.rst deleted file mode 100644 index 9424244fd..000000000 --- a/docs/src/searx.engines.demo_offline.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _demo offline engine: - -=================== -Demo Offline Engine -=================== - -.. automodule:: searx.engines.demo_offline - :members: - diff --git a/docs/src/searx.engines.demo_online.rst b/docs/src/searx.engines.demo_online.rst deleted file mode 100644 index 0a8c8e985..000000000 --- a/docs/src/searx.engines.demo_online.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _demo online engine: - -================== -Demo Online Engine -================== - -.. automodule:: searx.engines.demo_online - :members: - diff --git a/docs/src/searx.engines.google.rst b/docs/src/searx.engines.google.rst deleted file mode 100644 index 9c15325f8..000000000 --- a/docs/src/searx.engines.google.rst +++ /dev/null @@ -1,76 +0,0 @@ -.. _google engines: - -============== -Google Engines -============== - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - - -.. _google API: - -Google API -========== - -.. _Query Parameter Definitions: - https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions - -SearXNG's implementation of the Google API is mainly done in -:py:obj:`get_google_info `. - -For detailed description of the *REST-full* API see: `Query Parameter -Definitions`_. The linked API documentation can sometimes be helpful during -reverse engineering. However, we cannot use it in the freely accessible WEB -services; not all parameters can be applied and some engines are more *special* -than other (e.g. :ref:`google news engine`). - - -.. _google web engine: - -Google WEB -========== - -.. automodule:: searx.engines.google - :members: - -.. _google autocomplete: - -Google Autocomplete -==================== - -.. autofunction:: searx.autocomplete.google_complete - -.. _google images engine: - -Google Images -============= - -.. automodule:: searx.engines.google_images - :members: - -.. _google videos engine: - -Google Videos -============= - -.. automodule:: searx.engines.google_videos - :members: - -.. _google news engine: - -Google News -=========== - -.. automodule:: searx.engines.google_news - :members: - -.. _google scholar engine: - -Google Scholar -============== - -.. automodule:: searx.engines.google_scholar - :members: diff --git a/docs/src/searx.engines.peertube.rst b/docs/src/searx.engines.peertube.rst deleted file mode 100644 index 8e1576ea0..000000000 --- a/docs/src/searx.engines.peertube.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _peertube engines: - -================ -Peertube Engines -================ - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - - -.. _peertube video engine: - -Peertube Video -============== - -.. automodule:: searx.engines.peertube - :members: - -.. _sepiasearch engine: - -SepiaSearch -=========== - -.. automodule:: searx.engines.sepiasearch - :members: diff --git a/docs/src/searx.engines.rst b/docs/src/searx.engines.rst deleted file mode 100644 index 4ce96b27e..000000000 --- a/docs/src/searx.engines.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _searx.engines: - -================= -SearXNG's engines -================= - -.. automodule:: searx.engines - :members: diff --git a/docs/src/searx.engines.startpage.rst b/docs/src/searx.engines.startpage.rst deleted file mode 100644 index c885d8f1b..000000000 --- a/docs/src/searx.engines.startpage.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _startpage engines: - -================= -Startpage engines -================= - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - -.. automodule:: searx.engines.startpage - :members: diff --git a/docs/src/searx.engines.tineye.rst b/docs/src/searx.engines.tineye.rst deleted file mode 100644 index 79e24cfb8..000000000 --- a/docs/src/searx.engines.tineye.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _tineye engine: - -====== -Tineye -====== - -.. automodule:: searx.engines.tineye - :members: - diff --git a/docs/src/searx.engines.torznab.rst b/docs/src/searx.engines.torznab.rst deleted file mode 100644 index 0b96e18ec..000000000 --- a/docs/src/searx.engines.torznab.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. automodule:: searx.engines.torznab - :members: diff --git a/docs/src/searx.engines.wikipedia.rst b/docs/src/searx.engines.wikipedia.rst deleted file mode 100644 index e644cd645..000000000 --- a/docs/src/searx.engines.wikipedia.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _wikimedia engines: - -========= -Wikimedia -========= - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - - -.. _wikipedia engine: - -Wikipedia -========= - -.. automodule:: searx.engines.wikipedia - :members: - -.. _wikidata engine: - -Wikidata -========= - -.. automodule:: searx.engines.wikidata - :members: diff --git a/docs/src/searx.engines.yahoo.rst b/docs/src/searx.engines.yahoo.rst deleted file mode 100644 index df08550ca..000000000 --- a/docs/src/searx.engines.yahoo.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _yahoo engine: - -============ -Yahoo Engine -============ - -.. automodule:: searx.engines.yahoo - :members: diff --git a/docs/src/searx.locales.rst b/docs/src/searx.locales.rst index 2f13bfca1..0de49a5e1 100644 --- a/docs/src/searx.locales.rst +++ b/docs/src/searx.locales.rst @@ -4,7 +4,7 @@ Locales ======= -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/src/searx.search.processors.rst b/docs/src/searx.search.processors.rst index 390680657..3bb4c44bc 100644 --- a/docs/src/searx.search.processors.rst +++ b/docs/src/searx.search.processors.rst @@ -4,7 +4,7 @@ Search processors ================= -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry @@ -34,7 +34,7 @@ Online currency processor .. automodule:: searx.search.processors.online_currency :members: -Online Dictionary processor +Online dictionary processor =========================== .. automodule:: searx.search.processors.online_dictionary diff --git a/docs/user/configured_engines.rst b/docs/user/configured_engines.rst index 9507c77df..c32a264ed 100644 --- a/docs/user/configured_engines.rst +++ b/docs/user/configured_engines.rst @@ -22,7 +22,7 @@ Configured Engines called *tabs*), engines can be queried by their name or the categories they belong to, by using a :ref:`\!bing syntax `. -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/user/index.rst b/docs/user/index.rst index 4603b4716..705fd3f9e 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -2,7 +2,7 @@ User information ================ -.. contents:: Contents +.. contents:: :depth: 3 :local: :backlinks: entry diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 1ac77e285..b570b07e6 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -10,7 +10,6 @@ and developers. .. toctree:: :maxdepth: 2 - :caption: Contents searxng.sh lxc.sh diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst index cb0fe9361..43dc915ef 100644 --- a/docs/utils/lxc.sh.rst +++ b/docs/utils/lxc.sh.rst @@ -26,7 +26,7 @@ to care about*). - `LXC/LXD Image Server`_ - `LXD@github`_ -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry diff --git a/docs/utils/searxng.sh.rst b/docs/utils/searxng.sh.rst index 3a6ac6d43..bedc1ba4c 100644 --- a/docs/utils/searxng.sh.rst +++ b/docs/utils/searxng.sh.rst @@ -15,7 +15,7 @@ script :origin:`utils/searxng.sh`. - :ref:`installation nginx` - :ref:`installation apache` -.. contents:: Contents +.. contents:: :depth: 2 :local: :backlinks: entry -- cgit v1.2.3