From 1f2dc6c64785ab962efbeed0cfc093fdd3f41bf0 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Tue, 14 Jul 2020 18:56:57 +0200 Subject: [enh] add external plugin support --- searx/plugins/__init__.py | 63 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 4dbcbbd28..66ff93c2a 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -14,8 +14,14 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2015 by Adam Tauber, ''' + +from importlib import import_module +from os.path import abspath, basename, dirname, exists, join +from shutil import copyfile from sys import exit, version_info -from searx import logger +from traceback import print_exc + +from searx import logger, settings, static_path if version_info[0] == 3: unicode = str @@ -54,7 +60,9 @@ class PluginStore(): for plugin in self.plugins: yield plugin - def register(self, *plugins): + def register(self, *plugins, external=False): + if external: + plugins = load_external_plugins(plugins) for plugin in plugins: for plugin_attr, plugin_attr_type in required_attrs: if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type): @@ -77,6 +85,54 @@ class PluginStore(): return ret +def load_external_plugins(plugin_names): + plugins = [] + for name in plugin_names: + logger.debug('loading plugin: {0}'.format(name)) + try: + pkg = import_module(name) + except Exception as e: + logger.critical('failed to load plugin module {0}: {1}'.format(name, e)) + exit(3) + + pkg.__base_path = dirname(abspath(pkg.__file__)) + + fix_package_resources(pkg, name) + + plugins.append(pkg) + logger.debug('plugin "{0}" loaded'.format(name)) + return plugins + + +def check_resource(base_path, resource_path, name, dir_prefix): + dep_path = join(base_path, resource_path) + file_name = basename(dep_path) + resource_name = '{0}_{1}'.format('_'.join(name.split()), file_name) + resource_path = join(static_path, 'plugins', dir_prefix, resource_name) + if not exists(resource_path): + try: + copyfile(dep_path, resource_path) + except: + logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(resource_name, name)) + exit(3) + + # returning with the web path of the resource + return join('plugins', dir_prefix, resource_name) + + +def fix_package_resources(pkg, name): + if hasattr(pkg, 'js_dependencies'): + pkg.js_dependencies = tuple([ + check_resource(pkg.__base_path, x, name, 'js') + for x in pkg.js_dependencies + ]) + if hasattr(pkg, 'css_dependencies'): + pkg.css_dependencies = tuple([ + check_resource(pkg.__base_path, x, name, 'css') + for x in pkg.css_dependencies + ]) + + plugins = PluginStore() plugins.register(oa_doi_rewrite) plugins.register(https_rewrite) @@ -86,3 +142,6 @@ plugins.register(self_info) plugins.register(search_on_category_select) plugins.register(tracker_url_remover) plugins.register(vim_hotkeys) +# load external plugins +if 'plugins' in settings: + plugins.register(*settings['plugins'], external=True) -- cgit v1.2.3 From 7b93d11d841aee8da7825d7608ac1308944d88d9 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Tue, 14 Jul 2020 21:51:38 +0200 Subject: [fix] add plugin sha sum check to be able to copy updated resources --- searx/plugins/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 66ff93c2a..8742dc2d6 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -15,6 +15,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2015 by Adam Tauber, ''' +from hashlib import sha256 from importlib import import_module from os.path import abspath, basename, dirname, exists, join from shutil import copyfile @@ -109,7 +110,7 @@ def check_resource(base_path, resource_path, name, dir_prefix): file_name = basename(dep_path) resource_name = '{0}_{1}'.format('_'.join(name.split()), file_name) resource_path = join(static_path, 'plugins', dir_prefix, resource_name) - if not exists(resource_path): + if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path): try: copyfile(dep_path, resource_path) except: @@ -133,6 +134,12 @@ def fix_package_resources(pkg, name): ]) +def sha_sum(filename): + with open(filename,"rb") as f: + bytes = f.read() # read entire file as bytes + return sha256(bytes).hexdigest() + + plugins = PluginStore() plugins.register(oa_doi_rewrite) plugins.register(https_rewrite) -- cgit v1.2.3 From 81e9c755345600e474dbffb758f76dc85357c7ab Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Tue, 14 Jul 2020 22:00:24 +0200 Subject: [fix] pep8 --- searx/plugins/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 8742dc2d6..379c69e2e 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -135,8 +135,8 @@ def fix_package_resources(pkg, name): def sha_sum(filename): - with open(filename,"rb") as f: - bytes = f.read() # read entire file as bytes + with open(filename, "rb") as f: + bytes = f.read() return sha256(bytes).hexdigest() -- cgit v1.2.3 From e45a269ecbb3a80b61d706e5dc7075e250f5afa9 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Wed, 22 Jul 2020 19:02:34 +0200 Subject: [mod] create static directory for each external plugin --- searx/plugins/__init__.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 379c69e2e..a3916d964 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -17,7 +17,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from hashlib import sha256 from importlib import import_module -from os.path import abspath, basename, dirname, exists, join +from os import listdir, mkdir, remove +from os.path import abspath, basename, dirname, exists, isdir, join from shutil import copyfile from sys import exit, version_info from traceback import print_exc @@ -105,34 +106,55 @@ def load_external_plugins(plugin_names): return plugins -def check_resource(base_path, resource_path, name, dir_prefix): +def check_resource(base_path, resource_path, name, target_dir, plugin_dir): dep_path = join(base_path, resource_path) file_name = basename(dep_path) - resource_name = '{0}_{1}'.format('_'.join(name.split()), file_name) - resource_path = join(static_path, 'plugins', dir_prefix, resource_name) + resource_path = join(target_dir, file_name) if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path): try: copyfile(dep_path, resource_path) except: - logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(resource_name, name)) + logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name)) exit(3) # returning with the web path of the resource - return join('plugins', dir_prefix, resource_name) + return join('plugins', plugin_dir, file_name) def fix_package_resources(pkg, name): + plugin_dir = 'plugin_' + name + target_dir = join(static_path, 'plugins', plugin_dir) + if not isdir(target_dir): + try: + mkdir(target_dir) + except: + logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name)) + exit(3) + + resources = [] + if hasattr(pkg, 'js_dependencies'): + resources.extend(map(basename, pkg.js_dependencies)) pkg.js_dependencies = tuple([ - check_resource(pkg.__base_path, x, name, 'js') + check_resource(pkg.__base_path, x, name, target_dir, plugin_dir) for x in pkg.js_dependencies ]) if hasattr(pkg, 'css_dependencies'): + resources.extend(map(basename, pkg.css_dependencies)) pkg.css_dependencies = tuple([ - check_resource(pkg.__base_path, x, name, 'css') + check_resource(pkg.__base_path, x, name, target_dir, plugin_dir) for x in pkg.css_dependencies ]) + for f in listdir(target_dir): + if basename(f) not in resources: + resource_path = join(target_dir, basename(f)) + try: + remove(resource_path) + except: + logger.critical('failed to remove unused resource file {0} for plugin {1}'.format(resource_path, name)) + exit(3) + def sha_sum(filename): with open(filename, "rb") as f: -- cgit v1.2.3 From dff879d07d3cd937526804dda4f07d2e84249c59 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 25 Jul 2020 03:20:29 +0200 Subject: [fix] resolve minor review issues --- searx/plugins/__init__.py | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index a3916d964..3d652e9ca 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -17,7 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from hashlib import sha256 from importlib import import_module -from os import listdir, mkdir, remove +from os import makedirs, remove from os.path import abspath, basename, dirname, exists, isdir, join from shutil import copyfile from sys import exit, version_info @@ -99,50 +99,49 @@ def load_external_plugins(plugin_names): pkg.__base_path = dirname(abspath(pkg.__file__)) - fix_package_resources(pkg, name) + prepare_package_resources(pkg, name) plugins.append(pkg) logger.debug('plugin "{0}" loaded'.format(name)) return plugins -def check_resource(base_path, resource_path, name, target_dir, plugin_dir): - dep_path = join(base_path, resource_path) - file_name = basename(dep_path) - resource_path = join(target_dir, file_name) - if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path): - try: - copyfile(dep_path, resource_path) - except: - logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name)) - exit(3) +def sync_resource(base_path, resource_path, name, target_dir, plugin_dir): + dep_path = join(base_path, resource_path) + file_name = basename(dep_path) + resource_path = join(target_dir, file_name) + if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path): + try: + copyfile(dep_path, resource_path) + except: + logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name)) + exit(3) - # returning with the web path of the resource - return join('plugins', plugin_dir, file_name) + # returning with the web path of the resource + return join('plugins', plugin_dir, file_name) -def fix_package_resources(pkg, name): +def prepare_package_resources(pkg, name): plugin_dir = 'plugin_' + name target_dir = join(static_path, 'plugins', plugin_dir) - if not isdir(target_dir): - try: - mkdir(target_dir) - except: - logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name)) - exit(3) + try: + makedirs(target_dir, exist_ok=True) + except: + logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name)) + exit(3) resources = [] if hasattr(pkg, 'js_dependencies'): resources.extend(map(basename, pkg.js_dependencies)) pkg.js_dependencies = tuple([ - check_resource(pkg.__base_path, x, name, target_dir, plugin_dir) + sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir) for x in pkg.js_dependencies ]) if hasattr(pkg, 'css_dependencies'): resources.extend(map(basename, pkg.css_dependencies)) pkg.css_dependencies = tuple([ - check_resource(pkg.__base_path, x, name, target_dir, plugin_dir) + sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir) for x in pkg.css_dependencies ]) -- cgit v1.2.3 From 3f60bb470a0cd1f4c158ac4d621a51017ecadb54 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sat, 25 Jul 2020 21:05:23 +0200 Subject: [fix] correct imported modules --- searx/plugins/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 3d652e9ca..1abf02026 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -17,10 +17,10 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from hashlib import sha256 from importlib import import_module -from os import makedirs, remove -from os.path import abspath, basename, dirname, exists, isdir, join +from os import listdir, makedirs, remove +from os.path import abspath, basename, dirname, exists, join from shutil import copyfile -from sys import exit, version_info +from sys import version_info from traceback import print_exc from searx import logger, settings, static_path -- cgit v1.2.3 From 164e4725f3ab4adcc0c1527d0ae01965109cfd0d Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Tue, 28 Jul 2020 13:10:09 +0200 Subject: [mod] add dedicated directory to external plugins with gitignore file --- searx/plugins/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 1abf02026..457183a1e 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -118,12 +118,12 @@ def sync_resource(base_path, resource_path, name, target_dir, plugin_dir): exit(3) # returning with the web path of the resource - return join('plugins', plugin_dir, file_name) + return join('plugins/external_plugins', plugin_dir, file_name) def prepare_package_resources(pkg, name): plugin_dir = 'plugin_' + name - target_dir = join(static_path, 'plugins', plugin_dir) + target_dir = join(static_path, 'plugins/external_plugins', plugin_dir) try: makedirs(target_dir, exist_ok=True) except: -- cgit v1.2.3 From 93ac4db312f0ec04bc631714a8fbc2054fe7578b Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Tue, 28 Jul 2020 13:57:57 +0200 Subject: [enh] copy atime_ns and mtime_ns of external plugin resources --- searx/plugins/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'searx/plugins/__init__.py') diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 457183a1e..c701df640 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -17,7 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from hashlib import sha256 from importlib import import_module -from os import listdir, makedirs, remove +from os import listdir, makedirs, remove, stat, utime from os.path import abspath, basename, dirname, exists, join from shutil import copyfile from sys import version_info @@ -113,6 +113,10 @@ def sync_resource(base_path, resource_path, name, target_dir, plugin_dir): if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path): try: copyfile(dep_path, resource_path) + # copy atime_ns and mtime_ns, so the weak ETags (generated by + # the HTTP server) do not change + dep_stat = stat(dep_path) + utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns)) except: logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name)) exit(3) -- cgit v1.2.3