Merge pull request #2074 from asciimoo/external-plugins

This is a second proposal to accomplish plugin decoupling. I think #1938 is highly complicated and does much more than this feature requires, so here is an alternative implementation for the same feature. Please review it and let me know your opinion.

This solution supports the use of any kind of standard python modules which implements the required attributes of a plugin, so new plugins can be installed by standard python tools (pip/setup.py).

Downsides:
 - Localization of plugins name/description isn't possible
 - Plugins have to be updated manually

## What does this PR do?

Implements external plugin extensibility.

## Why is this change important?

Makes us able to decouple plugins from searx.

## Related issues

#1938 #1716 #1878

docs/dev/plugins.rst

@@ -30,6 +30,13 @@ Example plugin
        ctx['search'].suggestions.add('example')
        return True
 
+External plugins
+================
+
+External plugins are standard python modules implementing all the requirements of the standard plugins.
+Plugins can be enabled by adding them to :ref:`settings.yml`'s ``plugins`` section.
+Example external plugin can be found `here <https://github.com/asciimoo/searx_external_plugin_example>`_.
+
 Register your plugin
 ====================
 

searx/__init__.py

@@ -30,6 +30,7 @@ except:
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
+static_path = abspath(join(dirname(__file__), 'static'))
 
 
 def check_settings_yml(file_name):
@@ -55,6 +56,9 @@ if not settings_path:
 with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
     settings = safe_load(settings_yaml)
 
+if settings['ui']['static_path']:
+    static_path = settings['ui']['static_path']
+
 '''
 enable debug if
 the environnement variable SEARX_DEBUG is 1 or true

searx/plugins/__init__.py

@@ -14,8 +14,16 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 (C) 2015 by Adam Tauber, <asciimoo@gmail.com>
 '''
-from sys import exit, version_info
-from searx import logger
+
+from hashlib import sha256
+from importlib import import_module
+from os import listdir, makedirs, remove, stat, utime
+from os.path import abspath, basename, dirname, exists, join
+from shutil import copyfile
+from sys import version_info
+from traceback import print_exc
+
+from searx import logger, settings, static_path
 
 if version_info[0] == 3:
     unicode = str
@@ -54,7 +62,9 @@ class PluginStore():
         for plugin in self.plugins:
             yield plugin
 
-    def register(self, *plugins):
+    def register(self, *plugins, external=False):
+        if external:
+            plugins = load_external_plugins(plugins)
         for plugin in plugins:
             for plugin_attr, plugin_attr_type in required_attrs:
                 if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
@@ -77,6 +87,84 @@ class PluginStore():
         return ret
 
 
+def load_external_plugins(plugin_names):
+    plugins = []
+    for name in plugin_names:
+        logger.debug('loading plugin: {0}'.format(name))
+        try:
+            pkg = import_module(name)
+        except Exception as e:
+            logger.critical('failed to load plugin module {0}: {1}'.format(name, e))
+            exit(3)
+
+        pkg.__base_path = dirname(abspath(pkg.__file__))
+
+        prepare_package_resources(pkg, name)
+
+        plugins.append(pkg)
+        logger.debug('plugin "{0}" loaded'.format(name))
+    return plugins
+
+
+def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
+    dep_path = join(base_path, resource_path)
+    file_name = basename(dep_path)
+    resource_path = join(target_dir, file_name)
+    if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path):
+        try:
+            copyfile(dep_path, resource_path)
+            # copy atime_ns and mtime_ns, so the weak ETags (generated by
+            # the HTTP server) do not change
+            dep_stat = stat(dep_path)
+            utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns))
+        except:
+            logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name))
+            exit(3)
+
+    # returning with the web path of the resource
+    return join('plugins/external_plugins', plugin_dir, file_name)
+
+
+def prepare_package_resources(pkg, name):
+    plugin_dir = 'plugin_' + name
+    target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)
+    try:
+        makedirs(target_dir, exist_ok=True)
+    except:
+        logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name))
+        exit(3)
+
+    resources = []
+
+    if hasattr(pkg, 'js_dependencies'):
+        resources.extend(map(basename, pkg.js_dependencies))
+        pkg.js_dependencies = tuple([
+            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+            for x in pkg.js_dependencies
+        ])
+    if hasattr(pkg, 'css_dependencies'):
+        resources.extend(map(basename, pkg.css_dependencies))
+        pkg.css_dependencies = tuple([
+            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+            for x in pkg.css_dependencies
+        ])
+
+    for f in listdir(target_dir):
+        if basename(f) not in resources:
+            resource_path = join(target_dir, basename(f))
+            try:
+                remove(resource_path)
+            except:
+                logger.critical('failed to remove unused resource file {0} for plugin {1}'.format(resource_path, name))
+                exit(3)
+
+
+def sha_sum(filename):
+    with open(filename, "rb") as f:
+        bytes = f.read()
+        return sha256(bytes).hexdigest()
+
+
 plugins = PluginStore()
 plugins.register(oa_doi_rewrite)
 plugins.register(https_rewrite)
@@ -86,3 +174,6 @@ plugins.register(self_info)
 plugins.register(search_on_category_select)
 plugins.register(tracker_url_remover)
 plugins.register(vim_hotkeys)
+# load external plugins
+if 'plugins' in settings:
+    plugins.register(*settings['plugins'], external=True)

searx/settings.yml

@@ -57,6 +57,14 @@ outgoing: # communication with search engines
 #        - 1.1.1.1
 #        - 1.1.1.2
 
+# External plugin configuration
+# See http://asciimoo.github.io/searx/dev/plugins.html for more details
+#
+# plugins:
+#   - plugin1
+#   - plugin2
+#   - ...
+
 engines:
   - name: apk mirror
     engine: apkmirror

searx/static/plugins/external_plugins/.gitignore

@@ -0,0 +1,3 @@
+*
+*/
+!.gitignore

searx/webapp.py

@@ -58,7 +58,7 @@ import flask_babel
 from flask_babel import Babel, gettext, format_date, format_decimal
 from flask.ctx import has_request_context
 from flask.json import jsonify
-from searx import brand
+from searx import brand, static_path
 from searx import settings, searx_dir, searx_debug
 from searx.exceptions import SearxParameterException
 from searx.engines import (