Merge pull request #2326 from return42/ungrouped

[mod] clarify the difference of the default category and subgrouping

11 files changed, 56 insertions, 36 deletions

diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst
index 0d9e14e57..fa61f6212 100644
--- a/docs/admin/engines/settings.rst
+++ b/docs/admin/engines/settings.rst
@@ -311,7 +311,6 @@ Global Settings
 ``results_on_new_tab``:
   Open result links in a new tab by default.
 
-
 .. _settings redis:
 
 ``redis:``
@@ -451,12 +450,15 @@ Communication with search engines.
   see `httpx verification defaults`_.
 
   In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and
-  ``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.  
+  ``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.
   see `httpx ssl configuration`_.
 
 ``max_redirects`` :
   30 by default. Maximum redirect before it is an error.
 
+
+.. _settings categories_as_tabs:
+
 ``categories_as_tabs:``
 -----------------------
 
@@ -477,6 +479,15 @@ Categories not listed here can still be searched with the :ref:`search-syntax`.
     files:
     social media:
 
+Engines are added to ``categories:`` (compare :ref:`engine categories`), the
+categories listed in ``categories_as_tabs`` are shown as tabs in the UI.  If
+there are no active engines in a category, the tab is not displayed (e.g. if a
+user disables all engines in a category).
+
+On the preferences page (``/preferences``) -- under *engines* -- there is an
+additional tab, called *other*.  In this tab are all engines listed that are not
+in one of the UI tabs (not included in ``categories_as_tabs``).
+
 .. _settings engine:
 
 Engine settings
@@ -552,10 +563,21 @@ engine is shown.  Most of the options have a default value or even are optional.
   to build and send a ``Accept-Language`` header in the request to the origin
   search engine.
 
+.. _engine categories:
+
 ``categories`` : optional
-  Define in which categories this engine will be active.  Most of the time, it is
-  defined in the code of the engine, but in a few cases it is useful, like when
-  describing multiple search engine using the same code.
+  Specifies to which categories the engine should be added.  Engines can be
+  assigned to multiple categories.
+
+  Categories can be shown as tabs (:ref:`settings categories_as_tabs`) in the
+  UI.  A search in a tab (in the UI) will query all engines that are active in
+  this tab.  In the preferences page (``/preferences``) -- under *engines* --
+  users can select what engine should be active when querying in this tab.
+
+  Alternatively, :ref:`\!bang <search-syntax>` can be used to search all engines
+  in a category, regardless of whether they are active or not, or whether they
+  are in a tab of the UI or not.  For example, ``!dictionaries`` can be used to
+  query all search engines in that category (group).
 
 ``timeout`` : optional
   Timeout of the search with the current search engine.  **Be careful, it will
@@ -658,7 +680,7 @@ and can relied on the default configuration :origin:`searx/settings.yml` using:
 ``engines:``
   With ``use_default_settings: true``, each settings can be override in a
   similar way, the ``engines`` section is merged according to the engine
-  ``name``.  In this example, SearXNG will load all the default engines, will 
+  ``name``.  In this example, SearXNG will load all the default engines, will
   enable the ``bing`` engine and define a :ref:`token <private engines>` for
   the arch linux engine:
 
diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst
index 930fd0813..5fd31c01d 100644
--- a/docs/dev/engine_overview.rst
+++ b/docs/dev/engine_overview.rst
@@ -46,7 +46,7 @@ Engine File
    ======================= =========== ========================================================
    argument                type        information
    ======================= =========== ========================================================
-   categories              list        pages, in which the engine is working
+   categories              list        categories, in which the engine is working
    paging                  boolean     support multiple pages
    time_range_support      boolean     support search time range
    engine_type             str         - ``online`` :ref:`[ref] <demo online engine>` by
diff --git a/searx/enginelib/__init__.py b/searx/enginelib/__init__.py
index 461791b18..00962e215 100644
--- a/searx/enginelib/__init__.py
+++ b/searx/enginelib/__init__.py
@@ -81,7 +81,7 @@ class Engine:  # pylint: disable=too-few-public-methods
     # settings.yml
 
     categories: List[str]
-    """Tabs, in which the engine is working."""
+    """Specifies to which :ref:`engine categories` the engine should be added."""
 
     name: str
     """Name that will be used across SearXNG to define this engine.  In settings, on
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index c8e8e7241..c9026f96d 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -45,7 +45,7 @@ ENGINE_DEFAULT_ARGS = {
     "about": {},
 }
 # set automatically when an engine does not have any tab category
-OTHER_CATEGORY = 'other'
+DEFAULT_CATEGORY = 'other'
 
 
 # Defaults for the namespace of an engine module, see :py:func:`load_engine`
@@ -132,7 +132,7 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
     set_loggers(engine, engine_name)
 
     if not any(cat in settings['categories_as_tabs'] for cat in engine.categories):
-        engine.categories.append(OTHER_CATEGORY)
+        engine.categories.append(DEFAULT_CATEGORY)
 
     return engine
 
diff --git a/searx/engines/duckduckgo_weather.py b/searx/engines/duckduckgo_weather.py
index 4f0ce1b49..f239ce8f9 100644
--- a/searx/engines/duckduckgo_weather.py
+++ b/searx/engines/duckduckgo_weather.py
@@ -36,7 +36,7 @@ about = {
 send_accept_language_header = True
 
 # engine dependent config
-categories = ["others"]
+categories = ["weather"]
 URL = "https://duckduckgo.com/js/spice/forecast/{query}/{lang}"
 
 
diff --git a/searx/engines/wttr.py b/searx/engines/wttr.py
index d3da2748a..2eaee62ae 100644
--- a/searx/engines/wttr.py
+++ b/searx/engines/wttr.py
@@ -15,7 +15,7 @@ about = {
     "results": "JSON",
 }
 
-categories = ["others"]
+categories = ["weather"]
 
 url = "https://wttr.in/{query}?format=j1&lang={lang}"
 
diff --git a/searx/preferences.py b/searx/preferences.py
index 5cee83a03..3da6d5d16 100644
--- a/searx/preferences.py
+++ b/searx/preferences.py
@@ -17,7 +17,7 @@ from searx.enginelib import Engine
 from searx.plugins import Plugin
 from searx.locales import LOCALE_NAMES
 from searx.webutils import VALID_LANGUAGE_CODE
-from searx.engines import OTHER_CATEGORY
+from searx.engines import DEFAULT_CATEGORY
 
 
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
@@ -259,7 +259,7 @@ class EnginesSetting(BooleanChoices):
         choices = {}
         for engine in engines:
             for category in engine.categories:
-                if not category in list(settings['categories_as_tabs'].keys()) + [OTHER_CATEGORY]:
+                if not category in list(settings['categories_as_tabs'].keys()) + [DEFAULT_CATEGORY]:
                     continue
                 choices['{}__{}'.format(engine.name, category)] = not engine.disabled
         super().__init__(default_value, choices)
diff --git a/searx/searxng.msg b/searx/searxng.msg
index c37240f83..17e49367c 100644
--- a/searx/searxng.msg
+++ b/searx/searxng.msg
@@ -15,8 +15,8 @@ __all__ = [
 
 CONSTANT_NAMES = {
     # Constants defined in other modules
-    'DEFAULT_GROUP_NAME': webutils.DEFAULT_GROUP_NAME,
-    'OTHER_CATEGORY': engines.OTHER_CATEGORY,
+    'NO_SUBGROUPING': webutils.NO_SUBGROUPING,
+    'DEFAULT_CATEGORY': engines.DEFAULT_CATEGORY,
 }
 
 CATEGORY_NAMES = {
diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html
index a0cc8efc2..70c150581 100644
--- a/searx/templates/simple/preferences.html
+++ b/searx/templates/simple/preferences.html
@@ -298,18 +298,18 @@
     <p>{{ _('Currently used search engines') }}</p>
     {{ tabs_open() }}
      {% set ns = namespace(checked=true) %}
-     {% for categ in categories_as_tabs + [OTHER_CATEGORY] %}
+     {% for categ in categories_as_tabs + [DEFAULT_CATEGORY] %}
      {{ tab_header('enginetab', 'category' + categ, _(categ), ns.checked )}}
      {% set ns.checked = false %}
-   {% if categ == OTHER_CATEGORY %}
-      <p>{{_('This tab does not show up for search results, but you can search the engines listed here via bangs.')}}</p>
-    {% endif %}
+     {% if categ == DEFAULT_CATEGORY %}
+      <p>{{_('This tab dues not exists in the user interface, but you can search in these engines by its !bangs.')}} <a href="{{ url_for('info', pagename='search-syntax') }}">&#9432;</a></p>
+     {% endif %}
     <div class="scrollx">
     <table class="striped table_engines">
       <tr>{{- "" -}}
         <th class="engine_checkbox">{{ _("Allow") }}</th>{{- "" -}}
         <th class="name">{{ _("Engine name") }}</th>{{- "" -}}
-        <th class="shortcut">{{ _("Shortcut") }}</th>{{- "" -}}
+        <th class="shortcut">{{ _("!bang") }}</th>{{- "" -}}
         <th>{{ _("Supports selected language") }}</th>{{- "" -}}
         <th>{{ _("SafeSearch") }}</th>{{- "" -}}
         <th>{{ _("Time range") }}</th>{{- "" -}}
diff --git a/searx/webapp.py b/searx/webapp.py
index 4ed6c2eb7..64a5c8452 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -63,7 +63,7 @@ from searx.settings_defaults import OUTPUT_FORMATS
 from searx.settings_loader import get_default_settings_path
 from searx.exceptions import SearxParameterException
 from searx.engines import (
-    OTHER_CATEGORY,
+    DEFAULT_CATEGORY,
     categories,
     engines,
     engine_shortcuts,
@@ -435,7 +435,7 @@ def render(template_name: str, **kwargs):
     kwargs['method'] = request.preferences.get_value('method')
     kwargs['categories_as_tabs'] = list(settings['categories_as_tabs'].keys())
     kwargs['categories'] = _get_enable_categories(categories.keys())
-    kwargs['OTHER_CATEGORY'] = OTHER_CATEGORY
+    kwargs['DEFAULT_CATEGORY'] = DEFAULT_CATEGORY
 
     # i18n
     kwargs['sxng_locales'] = [l for l in sxng_locales if l[0] in settings['search']['languages']]
diff --git a/searx/webutils.py b/searx/webutils.py
index e62b0d695..4b11a15af 100644
--- a/searx/webutils.py
+++ b/searx/webutils.py
@@ -18,7 +18,7 @@ from codecs import getincrementalencoder
 from flask_babel import gettext, format_date
 
 from searx import logger, settings
-from searx.engines import OTHER_CATEGORY
+from searx.engines import DEFAULT_CATEGORY
 
 if TYPE_CHECKING:
     from searx.enginelib import Engine
@@ -222,26 +222,24 @@ def is_flask_run_cmdline():
     return frames[-2].filename.endswith('flask/cli.py')
 
 
-DEFAULT_GROUP_NAME = 'others'
+NO_SUBGROUPING = 'without further subgrouping'
 
 
 def group_engines_in_tab(engines: Iterable[Engine]) -> List[Tuple[str, Iterable[Engine]]]:
-    """Groups an Iterable of engines by their first non tab category"""
+    """Groups an Iterable of engines by their first non tab category (first subgroup)"""
 
-    def get_group(eng):
-        non_tab_categories = [
-            c for c in eng.categories if c not in list(settings['categories_as_tabs'].keys()) + [OTHER_CATEGORY]
-        ]
-        return non_tab_categories[0] if len(non_tab_categories) > 0 else DEFAULT_GROUP_NAME
-
-    groups = itertools.groupby(sorted(engines, key=get_group), get_group)
+    def get_subgroup(eng):
+        non_tab_categories = [c for c in eng.categories if c not in tabs + [DEFAULT_CATEGORY]]
+        return non_tab_categories[0] if len(non_tab_categories) > 0 else NO_SUBGROUPING
 
     def group_sort_key(group):
-        return (group[0] == DEFAULT_GROUP_NAME, group[0].lower())
-
-    sorted_groups = sorted(((name, list(engines)) for name, engines in groups), key=group_sort_key)
+        return (group[0] == NO_SUBGROUPING, group[0].lower())
 
     def engine_sort_key(engine):
         return (engine.about.get('language', ''), engine.name)
 
+    tabs = list(settings['categories_as_tabs'].keys())
+    subgroups = itertools.groupby(sorted(engines, key=get_subgroup), get_subgroup)
+    sorted_groups = sorted(((name, list(engines)) for name, engines in subgroups), key=group_sort_key)
+
     return [(groupname, sorted(engines, key=engine_sort_key)) for groupname, engines in sorted_groups]