Merge pull request #2074 from asciimoo/external-plugins

This is a second proposal to accomplish plugin decoupling. I think #1938 is highly complicated and does much more than this feature requires, so here is an alternative implementation for the same feature. Please review it and let me know your opinion.

This solution supports the use of any kind of standard python modules which implements the required attributes of a plugin, so new plugins can be installed by standard python tools (pip/setup.py).

Downsides:
 - Localization of plugins name/description isn't possible
 - Plugins have to be updated manually

## What does this PR do?

Implements external plugin extensibility.

## Why is this change important?

Makes us able to decouple plugins from searx.

## Related issues

#1938 #1716 #1878

5 files changed, 110 insertions, 4 deletions

diff --git a/searx/__init__.py b/searx/__init__.py
index 2f3ebfcfe..1ba03ad63 100644
--- a/searx/__init__.py
+++ b/searx/__init__.py
@@ -30,6 +30,7 @@ except:
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
+static_path = abspath(join(dirname(__file__), 'static'))
 
 
 def check_settings_yml(file_name):
@@ -55,6 +56,9 @@ if not settings_path:
 with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
     settings = safe_load(settings_yaml)
 
+if settings['ui']['static_path']:
+    static_path = settings['ui']['static_path']
+
 '''
 enable debug if
 the environnement variable SEARX_DEBUG is 1 or true
diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py
index 4dbcbbd28..c701df640 100644
--- a/searx/plugins/__init__.py
+++ b/searx/plugins/__init__.py
@@ -14,8 +14,16 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 (C) 2015 by Adam Tauber, <asciimoo@gmail.com>
 '''
-from sys import exit, version_info
-from searx import logger
+
+from hashlib import sha256
+from importlib import import_module
+from os import listdir, makedirs, remove, stat, utime
+from os.path import abspath, basename, dirname, exists, join
+from shutil import copyfile
+from sys import version_info
+from traceback import print_exc
+
+from searx import logger, settings, static_path
 
 if version_info[0] == 3:
     unicode = str
@@ -54,7 +62,9 @@ class PluginStore():
         for plugin in self.plugins:
             yield plugin
 
-    def register(self, *plugins):
+    def register(self, *plugins, external=False):
+        if external:
+            plugins = load_external_plugins(plugins)
         for plugin in plugins:
             for plugin_attr, plugin_attr_type in required_attrs:
                 if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
@@ -77,6 +87,84 @@ class PluginStore():
         return ret
 
 
+def load_external_plugins(plugin_names):
+    plugins = []
+    for name in plugin_names:
+        logger.debug('loading plugin: {0}'.format(name))
+        try:
+            pkg = import_module(name)
+        except Exception as e:
+            logger.critical('failed to load plugin module {0}: {1}'.format(name, e))
+            exit(3)
+
+        pkg.__base_path = dirname(abspath(pkg.__file__))
+
+        prepare_package_resources(pkg, name)
+
+        plugins.append(pkg)
+        logger.debug('plugin "{0}" loaded'.format(name))
+    return plugins
+
+
+def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
+    dep_path = join(base_path, resource_path)
+    file_name = basename(dep_path)
+    resource_path = join(target_dir, file_name)
+    if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path):
+        try:
+            copyfile(dep_path, resource_path)
+            # copy atime_ns and mtime_ns, so the weak ETags (generated by
+            # the HTTP server) do not change
+            dep_stat = stat(dep_path)
+            utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns))
+        except:
+            logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name))
+            exit(3)
+
+    # returning with the web path of the resource
+    return join('plugins/external_plugins', plugin_dir, file_name)
+
+
+def prepare_package_resources(pkg, name):
+    plugin_dir = 'plugin_' + name
+    target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)
+    try:
+        makedirs(target_dir, exist_ok=True)
+    except:
+        logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name))
+        exit(3)
+
+    resources = []
+
+    if hasattr(pkg, 'js_dependencies'):
+        resources.extend(map(basename, pkg.js_dependencies))
+        pkg.js_dependencies = tuple([
+            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+            for x in pkg.js_dependencies
+        ])
+    if hasattr(pkg, 'css_dependencies'):
+        resources.extend(map(basename, pkg.css_dependencies))
+        pkg.css_dependencies = tuple([
+            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+            for x in pkg.css_dependencies
+        ])
+
+    for f in listdir(target_dir):
+        if basename(f) not in resources:
+            resource_path = join(target_dir, basename(f))
+            try:
+                remove(resource_path)
+            except:
+                logger.critical('failed to remove unused resource file {0} for plugin {1}'.format(resource_path, name))
+                exit(3)
+
+
+def sha_sum(filename):
+    with open(filename, "rb") as f:
+        bytes = f.read()
+        return sha256(bytes).hexdigest()
+
+
 plugins = PluginStore()
 plugins.register(oa_doi_rewrite)
 plugins.register(https_rewrite)
@@ -86,3 +174,6 @@ plugins.register(self_info)
 plugins.register(search_on_category_select)
 plugins.register(tracker_url_remover)
 plugins.register(vim_hotkeys)
+# load external plugins
+if 'plugins' in settings:
+    plugins.register(*settings['plugins'], external=True)
diff --git a/searx/settings.yml b/searx/settings.yml
index 8df151b14..bee6e3e7b 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -57,6 +57,14 @@ outgoing: # communication with search engines
 #        - 1.1.1.1
 #        - 1.1.1.2
 
+# External plugin configuration
+# See http://asciimoo.github.io/searx/dev/plugins.html for more details
+#
+# plugins:
+#   - plugin1
+#   - plugin2
+#   - ...
+
 engines:
   - name: apk mirror
     engine: apkmirror
diff --git a/searx/static/plugins/external_plugins/.gitignore b/searx/static/plugins/external_plugins/.gitignore
new file mode 100644
index 000000000..94548af5b
--- /dev/null
+++ b/searx/static/plugins/external_plugins/.gitignore
@@ -0,0 +1,3 @@
+*
+*/
+!.gitignore
diff --git a/searx/webapp.py b/searx/webapp.py
index 4b52c0cb3..2df96e198 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -58,7 +58,7 @@ import flask_babel
 from flask_babel import Babel, gettext, format_date, format_decimal
 from flask.ctx import has_request_context
 from flask.json import jsonify
-from searx import brand
+from searx import brand, static_path
 from searx import settings, searx_dir, searx_debug
 from searx.exceptions import SearxParameterException
 from searx.engines import (