[enh] settings.yml: add use_default_settings option

This change is backward compatible with the existing configurations.

If a settings.yml loaded from an user defined location (SEARX_SETTINGS_PATH or /etc/searx/settings.yml),
then this settings can relied on the default settings.yml with this option:
user_default_settings:True

3 files changed, 102 insertions, 28 deletions

diff --git a/searx/__init__.py b/searx/__init__.py
index 887ef806d..214e554d4 100644
--- a/searx/__init__.py
+++ b/searx/__init__.py
@@ -16,39 +16,15 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 '''
 
 import logging
+import searx.settings
 from os import environ
 from os.path import realpath, dirname, join, abspath, isfile
-from io import open
-from yaml import safe_load
 
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
 static_path = abspath(join(dirname(__file__), 'static'))
-
-
-def check_settings_yml(file_name):
-    if isfile(file_name):
-        return file_name
-    else:
-        return None
-
-
-# find location of settings.yml
-if 'SEARX_SETTINGS_PATH' in environ:
-    # if possible set path to settings using the
-    # enviroment variable SEARX_SETTINGS_PATH
-    settings_path = check_settings_yml(environ['SEARX_SETTINGS_PATH'])
-else:
-    # if not, get it from searx code base or last solution from /etc/searx
-    settings_path = check_settings_yml(join(searx_dir, 'settings.yml')) or check_settings_yml('/etc/searx/settings.yml')
-
-if not settings_path:
-    raise Exception('settings.yml not found')
-
-# load settings
-with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
-    settings = safe_load(settings_yaml)
+settings, settings_load_message = searx.settings.load_settings()
 
 if settings['ui']['static_path']:
     static_path = settings['ui']['static_path']
@@ -58,7 +34,6 @@ enable debug if
 the environnement variable SEARX_DEBUG is 1 or true
 (whatever the value in settings.yml)
 or general.debug=True in settings.yml
-
 disable debug if
 the environnement variable SEARX_DEBUG is 0 or false
 (whatever the value in settings.yml)
@@ -78,7 +53,7 @@ else:
     logging.basicConfig(level=logging.WARNING)
 
 logger = logging.getLogger('searx')
-logger.debug('read configuration from %s', settings_path)
+logger.info(settings_load_message)
 logger.info('Initialisation done')
 
 if 'SEARX_SECRET' in environ:
diff --git a/searx/exceptions.py b/searx/exceptions.py
index 4af816272..2d1b1167e 100644
--- a/searx/exceptions.py
+++ b/searx/exceptions.py
@@ -31,3 +31,11 @@ class SearxParameterException(SearxException):
         self.message = message
         self.parameter_name = name
         self.parameter_value = value
+
+
+class SearxSettingsException(SearxException):
+
+    def __init__(self, message, filename):
+        super().__init__(message)
+        self.message = message
+        self.filename = filename
diff --git a/searx/settings.py b/searx/settings.py
new file mode 100644
index 000000000..cdddff589
--- /dev/null
+++ b/searx/settings.py
@@ -0,0 +1,91 @@
+import collections.abc
+
+import yaml
+from searx.exceptions import SearxSettingsException
+from os import environ
+from os.path import dirname, join, abspath, isfile
+
+
+searx_dir = abspath(dirname(__file__))
+
+
+def check_settings_yml(file_name):
+    if isfile(file_name):
+        return file_name
+    else:
+        return None
+
+
+def load_yaml(file_name):
+    try:
+        with open(file_name, 'r', encoding='utf-8') as settings_yaml:
+            settings = yaml.safe_load(settings_yaml)
+            if not isinstance(settings, dict) or len(settings) == 0:
+                raise SearxSettingsException('Empty file', file_name)
+            return settings
+    except IOError as e:
+        raise SearxSettingsException(e, file_name)
+    except yaml.YAMLError as e:
+        raise SearxSettingsException(e, file_name)
+
+
+def get_default_settings_path():
+    return check_settings_yml(join(searx_dir, 'settings.yml'))
+
+
+def get_user_settings_path():
+    # find location of settings.yml
+    if 'SEARX_SETTINGS_PATH' in environ:
+        # if possible set path to settings using the
+        # enviroment variable SEARX_SETTINGS_PATH
+        return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
+    else:
+        # if not, get it from searx code base or last solution from /etc/searx
+        return check_settings_yml('/etc/searx/settings.yml')
+
+
+def update_dict(d, u):
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = update_dict(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+
+def update_settings(default_settings, user_settings):
+    for k, v in user_settings.items():
+        if k == 'use_default_settings':
+            continue
+        elif k == 'engines':
+            default_engines = default_settings[k]
+            default_engines_dict = dict((definition['name'], definition) for definition in default_engines)
+            default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition)
+                                   for definition in v]
+        else:
+            update_dict(default_settings[k], v)
+
+    return default_settings
+
+
+def load_settings(load_user_setttings=True):
+    default_settings_path = get_default_settings_path()
+    user_settings_path = get_user_settings_path()
+    if user_settings_path is None or not load_user_setttings:
+        # no user settings
+        return (load_yaml(default_settings_path),
+                'load the default settings from {}'.format(default_settings_path))
+
+    # user settings
+    user_settings = load_yaml(user_settings_path)
+    if user_settings.get('use_default_settings'):
+        # the user settings are merged with the default configuration
+        default_settings = load_yaml(default_settings_path)
+        update_settings(default_settings, user_settings)
+        return (default_settings,
+                'merge the default settings ( {} ) and the user setttings ( {} )'
+                .format(default_settings_path, user_settings_path))
+
+    # the user settings, fully replace the default configuration
+    return (user_settings,
+            'load the user settings from {}'.format(user_settings_path))