Merge pull request #2074 from asciimoo/external-plugins

This is a second proposal to accomplish plugin decoupling. I think #1938 is highly complicated and does much more than this feature requires, so here is an alternative implementation for the same feature. Please review it and let me know your opinion.

This solution supports the use of any kind of standard python modules which implements the required attributes of a plugin, so new plugins can be installed by standard python tools (pip/setup.py).

Downsides:
 - Localization of plugins name/description isn't possible
 - Plugins have to be updated manually

## What does this PR do?

Implements external plugin extensibility.

## Why is this change important?

Makes us able to decouple plugins from searx.

## Related issues

#1938 #1716 #1878

1 files changed, 94 insertions, 3 deletions

diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py
index 4dbcbbd28..c701df640 100644
--- a/searx/plugins/__init__.py
+++ b/searx/plugins/__init__.py
@@ -14,8 +14,16 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 (C) 2015 by Adam Tauber, <asciimoo@gmail.com>
 '''
-from sys import exit, version_info
-from searx import logger
+
+from hashlib import sha256
+from importlib import import_module
+from os import listdir, makedirs, remove, stat, utime
+from os.path import abspath, basename, dirname, exists, join
+from shutil import copyfile
+from sys import version_info
+from traceback import print_exc
+
+from searx import logger, settings, static_path
 
 if version_info[0] == 3:
     unicode = str
@@ -54,7 +62,9 @@ class PluginStore():
         for plugin in self.plugins:
             yield plugin
 
-    def register(self, *plugins):
+    def register(self, *plugins, external=False):
+        if external:
+            plugins = load_external_plugins(plugins)
         for plugin in plugins:
             for plugin_attr, plugin_attr_type in required_attrs:
                 if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
@@ -77,6 +87,84 @@ class PluginStore():
         return ret
 
 
+def load_external_plugins(plugin_names):
+    plugins = []
+    for name in plugin_names:
+        logger.debug('loading plugin: {0}'.format(name))
+        try:
+            pkg = import_module(name)
+        except Exception as e:
+            logger.critical('failed to load plugin module {0}: {1}'.format(name, e))
+            exit(3)
+
+        pkg.__base_path = dirname(abspath(pkg.__file__))
+
+        prepare_package_resources(pkg, name)
+
+        plugins.append(pkg)
+        logger.debug('plugin "{0}" loaded'.format(name))
+    return plugins
+
+
+def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
+    dep_path = join(base_path, resource_path)
+    file_name = basename(dep_path)
+    resource_path = join(target_dir, file_name)
+    if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path):
+        try:
+            copyfile(dep_path, resource_path)
+            # copy atime_ns and mtime_ns, so the weak ETags (generated by
+            # the HTTP server) do not change
+            dep_stat = stat(dep_path)
+            utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns))
+        except:
+            logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name))
+            exit(3)
+
+    # returning with the web path of the resource
+    return join('plugins/external_plugins', plugin_dir, file_name)
+
+
+def prepare_package_resources(pkg, name):
+    plugin_dir = 'plugin_' + name
+    target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)
+    try:
+        makedirs(target_dir, exist_ok=True)
+    except:
+        logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name))
+        exit(3)
+
+    resources = []
+
+    if hasattr(pkg, 'js_dependencies'):
+        resources.extend(map(basename, pkg.js_dependencies))
+        pkg.js_dependencies = tuple([
+            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+            for x in pkg.js_dependencies
+        ])
+    if hasattr(pkg, 'css_dependencies'):
+        resources.extend(map(basename, pkg.css_dependencies))
+        pkg.css_dependencies = tuple([
+            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+            for x in pkg.css_dependencies
+        ])
+
+    for f in listdir(target_dir):
+        if basename(f) not in resources:
+            resource_path = join(target_dir, basename(f))
+            try:
+                remove(resource_path)
+            except:
+                logger.critical('failed to remove unused resource file {0} for plugin {1}'.format(resource_path, name))
+                exit(3)
+
+
+def sha_sum(filename):
+    with open(filename, "rb") as f:
+        bytes = f.read()
+        return sha256(bytes).hexdigest()
+
+
 plugins = PluginStore()
 plugins.register(oa_doi_rewrite)
 plugins.register(https_rewrite)
@@ -86,3 +174,6 @@ plugins.register(self_info)
 plugins.register(search_on_category_select)
 plugins.register(tracker_url_remover)
 plugins.register(vim_hotkeys)
+# load external plugins
+if 'plugins' in settings:
+    plugins.register(*settings['plugins'], external=True)