Merge pull request #319 from dalf/mod-plugins

plugins: refactor initialization

searx/plugins/__init__.py

@@ -1,122 +1,38 @@
-'''
+# SPDX-License-Identifier: AGPL-3.0-or-later
-searx is free software: you can redistribute it and/or modify
+# lint: pylint
-it under the terms of the GNU Affero General Public License as published by
+# pylint: disable=missing-module-docstring, missing-class-docstring
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-searx is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with searx. If not, see < http://www.gnu.org/licenses/ >.
-
-(C) 2015 by Adam Tauber, <asciimoo@gmail.com>
-'''
 
+import sys
 from hashlib import sha256
 from importlib import import_module
 from os import listdir, makedirs, remove, stat, utime
 from os.path import abspath, basename, dirname, exists, join
 from shutil import copyfile
+from pkgutil import iter_modules
+from logging import getLogger
 
 from searx import logger, settings
 
 
-logger = logger.getChild('plugins')
+logger = logger.getChild("plugins")
 
-from searx.plugins import (oa_doi_rewrite,
+required_attrs = (
-                           ahmia_filter,
+    ("name", str),
-                           hash_plugin,
+    ("description", str),
-                           infinite_scroll,
+    ("default_on", bool)
-                           self_info,
+)
-                           hostname_replace,
-                           search_on_category_select,
-                           tracker_url_remover,
-                           vim_hotkeys)
 
-required_attrs = (('name', str),
+optional_attrs = (
-                  ('description', str),
+    ("js_dependencies", tuple),
-                  ('default_on', bool))
+    ("css_dependencies", tuple),
-
+    ("preference_section", str),
-optional_attrs = (('js_dependencies', tuple),
+)
-                  ('css_dependencies', tuple),
-                  ('preference_section', str))
 
 
-class Plugin():
+def sha_sum(filename):
-    default_on = False
+    with open(filename, "rb") as f:
-    name = 'Default plugin'
+        file_content_bytes = f.read()
-    description = 'Default plugin description'
+        return sha256(file_content_bytes).hexdigest()
-
-
-class PluginStore():
-
-    def __init__(self):
-        self.plugins = []
-
-    def __iter__(self):
-        for plugin in self.plugins:
-            yield plugin
-
-    def register(self, *plugins, external=False):
-        if external:
-            plugins = load_external_plugins(plugins)
-        for plugin in plugins:
-            for plugin_attr, plugin_attr_type in required_attrs:
-                if not hasattr(plugin, plugin_attr):
-                    logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))
-                    exit(3)
-                attr = getattr(plugin, plugin_attr)
-                if not isinstance(attr, plugin_attr_type):
-                    type_attr = str(type(attr))
-                    logger.critical(
-                        'attribute "{0}" is of type {2}, must be of type {3}, cannot load plugin: {1}'
-                        .format(plugin_attr, plugin, type_attr, plugin_attr_type)
-                    )
-                    exit(3)
-            for plugin_attr, plugin_attr_type in optional_attrs:
-                if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
-                    setattr(plugin, plugin_attr, plugin_attr_type())
-            plugin.id = plugin.name.replace(' ', '_')
-            if not hasattr(plugin, 'preference_section'):
-                plugin.preference_section = 'general'
-            if plugin.preference_section == 'query':
-                for plugin_attr in ('query_keywords', 'query_examples'):
-                    if not hasattr(plugin, plugin_attr):
-                        logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))
-                        exit(3)
-            self.plugins.append(plugin)
-
-    def call(self, ordered_plugin_list, plugin_type, request, *args, **kwargs):
-        ret = True
-        for plugin in ordered_plugin_list:
-            if hasattr(plugin, plugin_type):
-                ret = getattr(plugin, plugin_type)(request, *args, **kwargs)
-                if not ret:
-                    break
-
-        return ret
-
-
-def load_external_plugins(plugin_names):
-    plugins = []
-    for name in plugin_names:
-        logger.debug('loading plugin: {0}'.format(name))
-        try:
-            pkg = import_module(name)
-        except Exception as e:
-            logger.critical('failed to load plugin module {0}: {1}'.format(name, e))
-            exit(3)
-
-        pkg.__base_path = dirname(abspath(pkg.__file__))
-
-        prepare_package_resources(pkg, name)
-
-        plugins.append(pkg)
-        logger.debug('plugin "{0}" loaded'.format(name))
-    return plugins
 
 
 def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
@@ -130,74 +46,200 @@ def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
             # the HTTP server) do not change
             dep_stat = stat(dep_path)
             utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns))
-        except:
+        except IOError:
-            logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name))
+            logger.critical(
-            exit(3)
+                "failed to copy plugin resource {0} for plugin {1}".format(
+                    file_name, name
+                )
+            )
+            sys.exit(3)
 
     # returning with the web path of the resource
-    return join('plugins/external_plugins', plugin_dir, file_name)
+    return join("plugins/external_plugins", plugin_dir, file_name)
 
 
-def prepare_package_resources(pkg, name):
+def prepare_package_resources(plugin, plugin_module_name):
-    plugin_dir = 'plugin_' + name
+    plugin_base_path = dirname(abspath(plugin.__file__))
-    target_dir = join(settings['ui']['static_path'], 'plugins/external_plugins', plugin_dir)
+
+    plugin_dir = plugin_module_name
+    target_dir = join(
+        settings["ui"]["static_path"], "plugins/external_plugins", plugin_dir
+    )
     try:
         makedirs(target_dir, exist_ok=True)
-    except:
+    except IOError:
-        logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name))
+        logger.critical(
-        exit(3)
+            "failed to create resource directory {0} for plugin {1}".format(
+                target_dir, plugin_module_name
+            )
+        )
+        sys.exit(3)
 
     resources = []
 
-    if hasattr(pkg, 'js_dependencies'):
+    if hasattr(plugin, "js_dependencies"):
-        resources.extend(map(basename, pkg.js_dependencies))
+        resources.extend(map(basename, plugin.js_dependencies))
-        pkg.js_dependencies = tuple([
+        plugin.js_dependencies = ([
-            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+            sync_resource(
-            for x in pkg.js_dependencies
+                plugin_base_path, x, plugin_module_name, target_dir, plugin_dir
-        ])
+            ) for x in plugin.js_dependencies
-    if hasattr(pkg, 'css_dependencies'):
+            ])
-        resources.extend(map(basename, pkg.css_dependencies))
+
-        pkg.css_dependencies = tuple([
+    if hasattr(plugin, "css_dependencies"):
-            sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
+        resources.extend(map(basename, plugin.css_dependencies))
-            for x in pkg.css_dependencies
+        plugin.css_dependencies = ([
-        ])
+            sync_resource(
+                plugin_base_path, x, plugin_module_name, target_dir, plugin_dir
+            ) for x in plugin.css_dependencies
+            ])
 
     for f in listdir(target_dir):
         if basename(f) not in resources:
             resource_path = join(target_dir, basename(f))
             try:
                 remove(resource_path)
-            except:
+            except IOError:
-                logger.critical('failed to remove unused resource file {0} for plugin {1}'.format(resource_path, name))
+                logger.critical(
-                exit(3)
+                    "failed to remove unused resource file {0} for plugin {1}".format(
+                        resource_path, plugin_module_name
+                    )
+                )
+                sys.exit(3)
 
 
-def sha_sum(filename):
+def load_plugin(plugin_module_name, external):
-    with open(filename, "rb") as f:
+    # pylint: disable=too-many-branches
-        file_content_bytes = f.read()
+    try:
-        return sha256(file_content_bytes).hexdigest()
+        plugin = import_module(plugin_module_name)
+    except (
+        SyntaxError,
+        KeyboardInterrupt,
+        SystemExit,
+        SystemError,
+        ImportError,
+        RuntimeError,
+    ) as e:
+        logger.critical("%s: fatal exception", plugin_module_name, exc_info=e)
+        sys.exit(3)
+    except BaseException:
+        logger.exception("%s: exception while loading, the plugin is disabled", plugin_module_name)
+        return None
+
+    # difference with searx: use module name instead of the user name
+    plugin.id = plugin_module_name
+
+    #
+    plugin.logger = getLogger(plugin_module_name)
+
+    for plugin_attr, plugin_attr_type in required_attrs:
+        if not hasattr(plugin, plugin_attr):
+            logger.critical(
+                '%s: missing attribute "%s", cannot load plugin', plugin, plugin_attr
+            )
+            sys.exit(3)
+        attr = getattr(plugin, plugin_attr)
+        if not isinstance(attr, plugin_attr_type):
+            type_attr = str(type(attr))
+            logger.critical(
+                '{1}: attribute "{0}" is of type {2}, must be of type {3}, cannot load plugin'.format(
+                    plugin, plugin_attr, type_attr, plugin_attr_type
+                )
+            )
+            sys.exit(3)
+
+    for plugin_attr, plugin_attr_type in optional_attrs:
+        if not hasattr(plugin, plugin_attr) or not isinstance(
+            getattr(plugin, plugin_attr), plugin_attr_type
+        ):
+            setattr(plugin, plugin_attr, plugin_attr_type())
+
+    if not hasattr(plugin, "preference_section"):
+        plugin.preference_section = "general"
+
+    # query plugin
+    if plugin.preference_section == "query":
+        for plugin_attr in ("query_keywords", "query_examples"):
+            if not hasattr(plugin, plugin_attr):
+                logger.critical(
+                    'missing attribute "{0}", cannot load plugin: {1}'.format(
+                        plugin_attr, plugin
+                    )
+                )
+                sys.exit(3)
+
+    if settings.get("enabled_plugins"):
+        # searx compatibility: plugin.name in settings['enabled_plugins']
+        plugin.default_on = (
+            plugin.name in settings["enabled_plugins"]
+            or plugin.id in settings["enabled_plugins"]
+        )
+
+    # copy ressources if this is an external plugin
+    if external:
+        prepare_package_resources(plugin, plugin_module_name)
+
+    logger.debug("%s: loaded", plugin_module_name)
+
+    return plugin
+
+
+def load_and_initialize_plugin(plugin_module_name, external, init_args):
+    plugin = load_plugin(plugin_module_name, external)
+    if plugin and hasattr(plugin, 'init'):
+        try:
+            return plugin if plugin.init(*init_args) else None
+        except Exception:  # pylint: disable=broad-except
+            plugin.logger.exception(
+                "Exception while calling init, the plugin is disabled"
+            )
+            return None
+    return plugin
+
+
+class PluginStore:
+    def __init__(self):
+        self.plugins = []
+
+    def __iter__(self):
+        for plugin in self.plugins:
+            yield plugin
+
+    def register(self, plugin):
+        self.plugins.append(plugin)
+
+    def call(self, ordered_plugin_list, plugin_type, *args, **kwargs):
+        # pylint: disable=no-self-use
+        ret = True
+        for plugin in ordered_plugin_list:
+            if hasattr(plugin, plugin_type):
+                try:
+                    ret = getattr(plugin, plugin_type)(*args, **kwargs)
+                    if not ret:
+                        break
+                except Exception:  # pylint: disable=broad-except
+                    plugin.logger.exception("Exception while calling %s", plugin_type)
+        return ret
 
 
 plugins = PluginStore()
-plugins.register(oa_doi_rewrite)
-plugins.register(hash_plugin)
-plugins.register(infinite_scroll)
-plugins.register(self_info)
-plugins.register(hostname_replace)
-plugins.register(search_on_category_select)
-plugins.register(tracker_url_remover)
-plugins.register(vim_hotkeys)
-# load external plugins
-if settings['plugins']:
-    plugins.register(*settings['plugins'], external=True)
 
-if settings['enabled_plugins']:
-    for plugin in plugins:
-        if plugin.name in settings['enabled_plugins']:
-            plugin.default_on = True
-        else:
-            plugin.default_on = False
 
-# load tor specific plugins
+def plugin_module_names():
-if settings['outgoing']['using_tor_proxy']:
+    yield_plugins = set()
-    plugins.register(ahmia_filter)
+
+    # embedded plugins
+    for module in iter_modules(path=[dirname(__file__)]):
+        yield (__name__ + "." + module.name, False)
+        yield_plugins.add(module.name)
+    # external plugins
+    for module_name in settings['plugins']:
+        if module_name not in yield_plugins:
+            yield (module_name, True)
+            yield_plugins.add(module_name)
+
+
+def initialize(app):
+    for module_name, external in plugin_module_names():
+        plugin = load_and_initialize_plugin(module_name, external, (app, settings))
+        if plugin:
+            plugins.register(plugin)

searx/plugins/ahmia_filter.py

@@ -13,15 +13,17 @@ preference_section = 'onions'
 ahmia_blacklist = None
 
 
-def get_ahmia_blacklist():
-    global ahmia_blacklist
-    if not ahmia_blacklist:
-        ahmia_blacklist = ahmia_blacklist_loader()
-    return ahmia_blacklist
-
-
 def on_result(request, search, result):
     if not result.get('is_onion') or not result.get('parsed_url'):
         return True
     result_hash = md5(result['parsed_url'].hostname.encode()).hexdigest()
-    return result_hash not in get_ahmia_blacklist()
+    return result_hash not in ahmia_blacklist
+
+
+def init(app, settings):
+    global ahmia_blacklist  # pylint: disable=global-statement
+    if not settings['outgoing']['using_tor_proxy']:
+        # disable the plugin
+        return False
+    ahmia_blacklist = ahmia_blacklist_loader()
+    return True

searx/settings_defaults.py

@@ -200,8 +200,8 @@ SCHEMA = {
         'networks': {
         },
     },
-    'plugins': SettingsValue((None, list), None),
+    'plugins': SettingsValue(list, []),
-    'enabled_plugins': SettingsValue(list, []),
+    'enabled_plugins': SettingsValue((None, list), None),
     'checker': {
         'off_when_debug': SettingsValue(bool, True),
     },

searx/webapp.py

@@ -85,7 +85,7 @@ from searx.utils import (
 )
 from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
 from searx.query import RawTextQuery
-from searx.plugins import plugins
+from searx.plugins import plugins, initialize as plugin_initialize
 from searx.plugins.oa_doi_rewrite import get_doi_resolver
 from searx.preferences import (
     Preferences,
@@ -158,25 +158,6 @@ app.jinja_env.lstrip_blocks = True
 app.jinja_env.add_extension('jinja2.ext.loopcontrols')  # pylint: disable=no-member
 app.secret_key = settings['server']['secret_key']
 
-# see https://flask.palletsprojects.com/en/1.1.x/cli/
-# True if "FLASK_APP=searx/webapp.py FLASK_ENV=development flask run"
-flask_run_development = (
-    os.environ.get("FLASK_APP") is not None
-    and os.environ.get("FLASK_ENV") == 'development'
-    and is_flask_run_cmdline()
-)
-
-# True if reload feature is activated of werkzeug, False otherwise (including uwsgi, etc..)
-#  __name__ != "__main__" if searx.webapp is imported (make test, make docs, uwsgi...)
-# see run() at the end of this file : searx_debug activates the reload feature.
-werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__main__")
-
-# initialize the engines except on the first run of the werkzeug server.
-if (not werkzeug_reloader
-    or (werkzeug_reloader
-        and os.environ.get("WERKZEUG_RUN_MAIN") == "true") ):
-    search_initialize(enable_checker=True)
-
 babel = Babel(app)
 
 # used when translating category names
@@ -1351,6 +1332,27 @@ def page_not_found(_e):
     return render('404.html'), 404
 
 
+# see https://flask.palletsprojects.com/en/1.1.x/cli/
+# True if "FLASK_APP=searx/webapp.py FLASK_ENV=development flask run"
+flask_run_development = (
+    os.environ.get("FLASK_APP") is not None
+    and os.environ.get("FLASK_ENV") == 'development'
+    and is_flask_run_cmdline()
+)
+
+# True if reload feature is activated of werkzeug, False otherwise (including uwsgi, etc..)
+#  __name__ != "__main__" if searx.webapp is imported (make test, make docs, uwsgi...)
+# see run() at the end of this file : searx_debug activates the reload feature.
+werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__main__")
+
+# initialize the engines except on the first run of the werkzeug server.
+if (not werkzeug_reloader
+    or (werkzeug_reloader
+        and os.environ.get("WERKZEUG_RUN_MAIN") == "true") ):
+    plugin_initialize(app)
+    search_initialize(enable_checker=True)
+
+
 def run():
     logger.debug(
         'starting webserver on %s:%s',

tests/unit/test_plugins.py

@@ -10,6 +10,12 @@ def get_search_mock(query, **kwargs):
                 result_container=Mock(answers=dict()))
 
 
+class PluginMock():
+    default_on = False
+    name = 'Default plugin'
+    description = 'Default plugin description'
+
+
 class PluginStoreTest(SearxTestCase):
 
     def test_PluginStore_init(self):
@@ -18,14 +24,14 @@ class PluginStoreTest(SearxTestCase):
 
     def test_PluginStore_register(self):
         store = plugins.PluginStore()
-        testplugin = plugins.Plugin()
+        testplugin = PluginMock()
         store.register(testplugin)
 
         self.assertTrue(len(store.plugins) == 1)
 
     def test_PluginStore_call(self):
         store = plugins.PluginStore()
-        testplugin = plugins.Plugin()
+        testplugin = PluginMock()
         store.register(testplugin)
         setattr(testplugin, 'asdf', Mock())
         request = Mock()
@@ -40,8 +46,9 @@ class PluginStoreTest(SearxTestCase):
 class SelfIPTest(SearxTestCase):
 
     def test_PluginStore_init(self):
+        plugin = plugins.load_and_initialize_plugin('searx.plugins.self_info', False, (None, {}))
         store = plugins.PluginStore()
-        store.register(plugins.self_info)
+        store.register(plugin)
 
         self.assertTrue(len(store.plugins) == 1)
 
@@ -89,7 +96,8 @@ class HashPluginTest(SearxTestCase):
 
     def test_PluginStore_init(self):
         store = plugins.PluginStore()
-        store.register(plugins.hash_plugin)
+        plugin = plugins.load_and_initialize_plugin('searx.plugins.hash_plugin', False, (None, {}))
+        store.register(plugin)
 
         self.assertTrue(len(store.plugins) == 1)