Merge pull request #2074 from asciimoo/external-plugins

This is a second proposal to accomplish plugin decoupling. I think #1938 is highly complicated and does much more than this feature requires, so here is an alternative implementation for the same feature. Please review it and let me know your opinion.

This solution supports the use of any kind of standard python modules which implements the required attributes of a plugin, so new plugins can be installed by standard python tools (pip/setup.py).

Downsides:
 - Localization of plugins name/description isn't possible
 - Plugins have to be updated manually

## What does this PR do?

Implements external plugin extensibility.

## Why is this change important?

Makes us able to decouple plugins from searx.

## Related issues

#1938 #1716 #1878
This commit is contained in:
Noémi Ványi 2020-07-28 21:28:55 +02:00 committed by GitHub
commit cdc2f33972
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 117 additions and 4 deletions

View File

@ -30,6 +30,13 @@ Example plugin
ctx['search'].suggestions.add('example')
return True
External plugins
================
External plugins are standard python modules implementing all the requirements of the standard plugins.
Plugins can be enabled by adding them to :ref:`settings.yml`'s ``plugins`` section.
Example external plugin can be found `here <https://github.com/asciimoo/searx_external_plugin_example>`_.
Register your plugin
====================

View File

@ -30,6 +30,7 @@ except:
searx_dir = abspath(dirname(__file__))
engine_dir = dirname(realpath(__file__))
static_path = abspath(join(dirname(__file__), 'static'))
def check_settings_yml(file_name):
@ -55,6 +56,9 @@ if not settings_path:
with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
settings = safe_load(settings_yaml)
if settings['ui']['static_path']:
static_path = settings['ui']['static_path']
'''
enable debug if
the environnement variable SEARX_DEBUG is 1 or true

View File

@ -14,8 +14,16 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2015 by Adam Tauber, <asciimoo@gmail.com>
'''
from sys import exit, version_info
from searx import logger
from hashlib import sha256
from importlib import import_module
from os import listdir, makedirs, remove, stat, utime
from os.path import abspath, basename, dirname, exists, join
from shutil import copyfile
from sys import version_info
from traceback import print_exc
from searx import logger, settings, static_path
if version_info[0] == 3:
unicode = str
@ -54,7 +62,9 @@ class PluginStore():
for plugin in self.plugins:
yield plugin
def register(self, *plugins):
def register(self, *plugins, external=False):
if external:
plugins = load_external_plugins(plugins)
for plugin in plugins:
for plugin_attr, plugin_attr_type in required_attrs:
if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):
@ -77,6 +87,84 @@ class PluginStore():
return ret
def load_external_plugins(plugin_names):
plugins = []
for name in plugin_names:
logger.debug('loading plugin: {0}'.format(name))
try:
pkg = import_module(name)
except Exception as e:
logger.critical('failed to load plugin module {0}: {1}'.format(name, e))
exit(3)
pkg.__base_path = dirname(abspath(pkg.__file__))
prepare_package_resources(pkg, name)
plugins.append(pkg)
logger.debug('plugin "{0}" loaded'.format(name))
return plugins
def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
dep_path = join(base_path, resource_path)
file_name = basename(dep_path)
resource_path = join(target_dir, file_name)
if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path):
try:
copyfile(dep_path, resource_path)
# copy atime_ns and mtime_ns, so the weak ETags (generated by
# the HTTP server) do not change
dep_stat = stat(dep_path)
utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns))
except:
logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name))
exit(3)
# returning with the web path of the resource
return join('plugins/external_plugins', plugin_dir, file_name)
def prepare_package_resources(pkg, name):
plugin_dir = 'plugin_' + name
target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)
try:
makedirs(target_dir, exist_ok=True)
except:
logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name))
exit(3)
resources = []
if hasattr(pkg, 'js_dependencies'):
resources.extend(map(basename, pkg.js_dependencies))
pkg.js_dependencies = tuple([
sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
for x in pkg.js_dependencies
])
if hasattr(pkg, 'css_dependencies'):
resources.extend(map(basename, pkg.css_dependencies))
pkg.css_dependencies = tuple([
sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)
for x in pkg.css_dependencies
])
for f in listdir(target_dir):
if basename(f) not in resources:
resource_path = join(target_dir, basename(f))
try:
remove(resource_path)
except:
logger.critical('failed to remove unused resource file {0} for plugin {1}'.format(resource_path, name))
exit(3)
def sha_sum(filename):
with open(filename, "rb") as f:
bytes = f.read()
return sha256(bytes).hexdigest()
plugins = PluginStore()
plugins.register(oa_doi_rewrite)
plugins.register(https_rewrite)
@ -86,3 +174,6 @@ plugins.register(self_info)
plugins.register(search_on_category_select)
plugins.register(tracker_url_remover)
plugins.register(vim_hotkeys)
# load external plugins
if 'plugins' in settings:
plugins.register(*settings['plugins'], external=True)

View File

@ -57,6 +57,14 @@ outgoing: # communication with search engines
# - 1.1.1.1
# - 1.1.1.2
# External plugin configuration
# See http://asciimoo.github.io/searx/dev/plugins.html for more details
#
# plugins:
# - plugin1
# - plugin2
# - ...
engines:
- name: apk mirror
engine: apkmirror

View File

@ -0,0 +1,3 @@
*
*/
!.gitignore

View File

@ -58,7 +58,7 @@ import flask_babel
from flask_babel import Babel, gettext, format_date, format_decimal
from flask.ctx import has_request_context
from flask.json import jsonify
from searx import brand
from searx import brand, static_path
from searx import settings, searx_dir, searx_debug
from searx.exceptions import SearxParameterException
from searx.engines import (