[enh] settings.yml: add use_default_settings option (2nd version)

This commit is contained in:
Alexandre Flament 2020-11-27 19:32:45 +01:00
parent 1cfe7f2a75
commit b4b81a5e1a
14 changed files with 441 additions and 253 deletions

View File

@ -266,19 +266,4 @@ test.clean:
travis.codecov:
$(Q)$(PY_ENV_BIN)/python -m pip install codecov
# user-settings
# -------------
PHONY += user-settings.create user-settings.update
user-settings.update: pyenvinstall
$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH}
user-settings.update.engines: pyenvinstall
$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH}
.PHONY: $(PHONY)
.PHONY: $(PHONY)

View File

@ -235,68 +235,51 @@ In the following example, the actual settings are the default settings defined i
.. code-block:: yaml
use_default_settings: true
use_default_settings: True
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
server:
bind_address: "0.0.0.0"
With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section:
With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example).
* If the ``engines`` section is defined then:
* searx loads only the engines declare in the user setttings.
* searx merges the configuration according to the engine name.
In the following example, only three engines are available. Each engine configuration is merged with the default configuration.
In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
.. code-block:: yaml
use_default_settings: true
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
engines:
- name: wikipedia
- name: wikidata
- name: ddg definitions
Another example where four engines are available. The arch linux wiki engine has a :ref:`token<private engines>`.
.. code-block:: yaml
use_default_settings: true
use_default_settings: True
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
engines:
- name: arch linux wiki
tokens: ['$ecretValue']
- name: wikipedia
- name: wikidata
- name: ddg definitions
automatic update
----------------
It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case):
.. code-block:: yaml
.. code-block:: sh
use_default_settings:
engines:
remove:
- google
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
engines:
- name: arch linux wiki
tokens: ['$ecretValue']
make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update
As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
Set ``SEARX_SETTINGS_PATH`` to your user settings path.
.. code-block:: yaml
As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated.
The following command creates or updates the user settings including the ``engines`` section:
.. code-block:: sh
make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines
After that ``/etc/searx/settings.yml``
* has a ``secret key``
* has a ``engine`` section if it is not already the case, moreover the command:
* has deleted engines that do not exist in the default settings.
* has added engines that exist in the default settings but are not declare in the user settings.
use_default_settings:
engines:
keep_only:
- google
- duckduckgo
server:
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
engines:
- name: google
tokens: ['$ecretValue']
- name: duckduckgo
tokens: ['$ecretValue']

View File

@ -16,7 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
'''
import logging
import searx.settings
import searx.settings_loader
from os import environ
from os.path import realpath, dirname, join, abspath, isfile
@ -24,7 +24,7 @@ from os.path import realpath, dirname, join, abspath, isfile
searx_dir = abspath(dirname(__file__))
engine_dir = dirname(realpath(__file__))
static_path = abspath(join(dirname(__file__), 'static'))
settings, settings_load_message = searx.settings.load_settings()
settings, settings_load_message = searx.settings_loader.load_settings()
if settings['ui']['static_path']:
static_path = settings['ui']['static_path']

View File

@ -1,91 +0,0 @@
import collections.abc
import yaml
from searx.exceptions import SearxSettingsException
from os import environ
from os.path import dirname, join, abspath, isfile
searx_dir = abspath(dirname(__file__))
def check_settings_yml(file_name):
if isfile(file_name):
return file_name
else:
return None
def load_yaml(file_name):
try:
with open(file_name, 'r', encoding='utf-8') as settings_yaml:
settings = yaml.safe_load(settings_yaml)
if not isinstance(settings, dict) or len(settings) == 0:
raise SearxSettingsException('Empty file', file_name)
return settings
except IOError as e:
raise SearxSettingsException(e, file_name)
except yaml.YAMLError as e:
raise SearxSettingsException(e, file_name)
def get_default_settings_path():
return check_settings_yml(join(searx_dir, 'settings.yml'))
def get_user_settings_path():
# find location of settings.yml
if 'SEARX_SETTINGS_PATH' in environ:
# if possible set path to settings using the
# enviroment variable SEARX_SETTINGS_PATH
return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
else:
# if not, get it from searx code base or last solution from /etc/searx
return check_settings_yml('/etc/searx/settings.yml')
def update_dict(d, u):
for k, v in u.items():
if isinstance(v, collections.abc.Mapping):
d[k] = update_dict(d.get(k, {}), v)
else:
d[k] = v
return d
def update_settings(default_settings, user_settings):
for k, v in user_settings.items():
if k == 'use_default_settings':
continue
elif k == 'engines':
default_engines = default_settings[k]
default_engines_dict = dict((definition['name'], definition) for definition in default_engines)
default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition)
for definition in v]
else:
update_dict(default_settings[k], v)
return default_settings
def load_settings(load_user_setttings=True):
default_settings_path = get_default_settings_path()
user_settings_path = get_user_settings_path()
if user_settings_path is None or not load_user_setttings:
# no user settings
return (load_yaml(default_settings_path),
'load the default settings from {}'.format(default_settings_path))
# user settings
user_settings = load_yaml(user_settings_path)
if user_settings.get('use_default_settings'):
# the user settings are merged with the default configuration
default_settings = load_yaml(default_settings_path)
update_settings(default_settings, user_settings)
return (default_settings,
'merge the default settings ( {} ) and the user setttings ( {} )'
.format(default_settings_path, user_settings_path))
# the user settings, fully replace the default configuration
return (user_settings,
'load the user settings from {}'.format(user_settings_path))

129
searx/settings_loader.py Normal file
View File

@ -0,0 +1,129 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
from os import environ
from os.path import dirname, join, abspath, isfile
from collections.abc import Mapping
from itertools import filterfalse
import yaml
from searx.exceptions import SearxSettingsException
searx_dir = abspath(dirname(__file__))
def check_settings_yml(file_name):
if isfile(file_name):
return file_name
return None
def load_yaml(file_name):
try:
with open(file_name, 'r', encoding='utf-8') as settings_yaml:
return yaml.safe_load(settings_yaml)
except IOError as e:
raise SearxSettingsException(e, file_name)
except yaml.YAMLError as e:
raise SearxSettingsException(e, file_name)
def get_default_settings_path():
return check_settings_yml(join(searx_dir, 'settings.yml'))
def get_user_settings_path():
# find location of settings.yml
if 'SEARX_SETTINGS_PATH' in environ:
# if possible set path to settings using the
# enviroment variable SEARX_SETTINGS_PATH
return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
# if not, get it from searx code base or last solution from /etc/searx
return check_settings_yml('/etc/searx/settings.yml')
def update_dict(default_dict, user_dict):
for k, v in user_dict.items():
if isinstance(v, Mapping):
default_dict[k] = update_dict(default_dict.get(k, {}), v)
else:
default_dict[k] = v
return default_dict
def update_settings(default_settings, user_settings):
# merge everything except the engines
for k, v in user_settings.items():
if k not in ('use_default_settings', 'engines'):
update_dict(default_settings[k], v)
# parse the engines
remove_engines = None
keep_only_engines = None
use_default_settings = user_settings.get('use_default_settings')
if isinstance(use_default_settings, dict):
remove_engines = use_default_settings.get('engines', {}).get('remove')
keep_only_engines = use_default_settings.get('engines', {}).get('keep_only')
if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None:
engines = default_settings['engines']
# parse "use_default_settings.engines.remove"
if remove_engines is not None:
engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines))
# parse "use_default_settings.engines.keep_only"
if keep_only_engines is not None:
engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines))
# parse "engines"
user_engines = user_settings.get('engines')
if user_engines:
engines_dict = dict((definition['name'], definition) for definition in engines)
for user_engine in user_engines:
default_engine = engines_dict.get(user_engine['name'])
if default_engine:
update_dict(default_engine, user_engine)
else:
engines.append(user_engine)
# store the result
default_settings['engines'] = engines
return default_settings
def is_use_default_settings(user_settings):
use_default_settings = user_settings.get('use_default_settings')
if use_default_settings is True:
return True
if isinstance(use_default_settings, dict):
return True
if use_default_settings is False or use_default_settings is None:
return False
raise ValueError('Invalid value for use_default_settings')
def load_settings(load_user_setttings=True):
default_settings_path = get_default_settings_path()
user_settings_path = get_user_settings_path()
if user_settings_path is None or not load_user_setttings:
# no user settings
return (load_yaml(default_settings_path),
'load the default settings from {}'.format(default_settings_path))
# user settings
user_settings = load_yaml(user_settings_path)
if is_use_default_settings(user_settings):
# the user settings are merged with the default configuration
default_settings = load_yaml(default_settings_path)
update_settings(default_settings, user_settings)
return (default_settings,
'merge the default settings ( {} ) and the user setttings ( {} )'
.format(default_settings_path, user_settings_path))
# the user settings, fully replace the default configuration
return (user_settings,
'load the user settings from {}'.format(user_settings_path))

View File

View File

@ -0,0 +1,2 @@
Test:
**********

View File

@ -0,0 +1,111 @@
general:
debug : False
instance_name : "searx"
search:
safe_search : 0
autocomplete : ""
default_lang : ""
ban_time_on_fail : 5
max_ban_time_on_fail : 120
server:
port : 9000
bind_address : "0.0.0.0"
secret_key : "user_settings_secret"
base_url : False
image_proxy : False
http_protocol_version : "1.0"
method: "POST"
default_http_headers:
X-Content-Type-Options : nosniff
X-XSS-Protection : 1; mode=block
X-Download-Options : noopen
X-Robots-Tag : noindex, nofollow
Referrer-Policy : no-referrer
ui:
static_path : ""
templates_path : ""
default_theme : oscar
default_locale : ""
theme_args :
oscar_style : logicodev
engines:
- name : wikidata
engine : wikidata
shortcut : wd
timeout : 3.0
weight : 2
- name : wikibooks
engine : mediawiki
shortcut : wb
categories : general
base_url : "https://{language}.wikibooks.org/"
number_of_results : 5
search_type : text
- name : wikinews
engine : mediawiki
shortcut : wn
categories : news
base_url : "https://{language}.wikinews.org/"
number_of_results : 5
search_type : text
- name : wikiquote
engine : mediawiki
shortcut : wq
categories : general
base_url : "https://{language}.wikiquote.org/"
number_of_results : 5
search_type : text
locales:
en : English
ar : العَرَبِيَّة (Arabic)
bg : Български (Bulgarian)
bo : བོད་སྐད་ (Tibetian)
ca : Català (Catalan)
cs : Čeština (Czech)
cy : Cymraeg (Welsh)
da : Dansk (Danish)
de : Deutsch (German)
el_GR : Ελληνικά (Greek_Greece)
eo : Esperanto (Esperanto)
es : Español (Spanish)
et : Eesti (Estonian)
eu : Euskara (Basque)
fa_IR : (fārsī) فارسى (Persian)
fi : Suomi (Finnish)
fil : Wikang Filipino (Filipino)
fr : Français (French)
gl : Galego (Galician)
he : עברית (Hebrew)
hr : Hrvatski (Croatian)
hu : Magyar (Hungarian)
ia : Interlingua (Interlingua)
it : Italiano (Italian)
ja : 日本語 (Japanese)
lt : Lietuvių (Lithuanian)
nl : Nederlands (Dutch)
nl_BE : Vlaams (Dutch_Belgium)
oc : Lenga D'òc (Occitan)
pl : Polski (Polish)
pt : Português (Portuguese)
pt_BR : Português (Portuguese_Brazil)
ro : Română (Romanian)
ru : Русский (Russian)
sk : Slovenčina (Slovak)
sl : Slovenski (Slovene)
sr : српски (Serbian)
sv : Svenska (Swedish)
te : తెలుగు (telugu)
ta : தமிழ் (Tamil)
tr : Türkçe (Turkish)
uk : українська мова (Ukrainian)
vi : tiếng việt (Vietnamese)
zh : 中文 (Chinese)
zh_TW : 國語 (Taiwanese Mandarin)

View File

@ -0,0 +1,14 @@
use_default_settings:
engines:
keep_only:
- wikibooks
- wikinews
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value
engines:
- name: wikipedia
- name: newengine
engine: dummy

View File

@ -0,0 +1,10 @@
use_default_settings:
engines:
remove:
- wikibooks
- wikinews
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value

View File

@ -0,0 +1,15 @@
use_default_settings:
engines:
remove:
- wikibooks
- wikinews
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value
engines:
- name: wikipedia
tokens: ['secret_token']
- name: newengine
engine: dummy

View File

@ -0,0 +1,6 @@
use_default_settings: True
server:
secret_key: "user_secret_key"
bind_address: "0.0.0.0"
default_http_headers:
Custom-Header: Custom-Value

View File

@ -0,0 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
from os.path import dirname, join, abspath
from unittest.mock import patch
from searx.testing import SearxTestCase
from searx.exceptions import SearxSettingsException
from searx import settings_loader
test_dir = abspath(dirname(__file__))
class TestLoad(SearxTestCase):
def test_load_zero(self):
with self.assertRaises(SearxSettingsException):
settings_loader.load_yaml('/dev/zero')
with self.assertRaises(SearxSettingsException):
settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml'))
with self.assertRaises(SearxSettingsException):
settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml'))
def test_check_settings_yml(self):
self.assertIsNone(settings_loader.check_settings_yml('/dev/zero'))
bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml')
self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path)
class TestDefaultSettings(SearxTestCase):
def test_load(self):
settings, msg = settings_loader.load_settings(load_user_setttings=False)
self.assertTrue(msg.startswith('load the default settings from'))
self.assertFalse(settings['general']['debug'])
self.assertTrue(isinstance(settings['general']['instance_name'], str))
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
self.assertTrue(isinstance(settings['server']['port'], int))
self.assertTrue(isinstance(settings['server']['bind_address'], str))
self.assertTrue(isinstance(settings['engines'], list))
self.assertTrue(isinstance(settings['locales'], dict))
self.assertTrue(isinstance(settings['doi_resolvers'], dict))
self.assertTrue(isinstance(settings['default_doi_resolver'], str))
class TestUserSettings(SearxTestCase):
def test_is_use_default_settings(self):
self.assertFalse(settings_loader.is_use_default_settings({}))
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True}))
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}}))
with self.assertRaises(ValueError):
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1}))
with self.assertRaises(ValueError):
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0}))
def test_user_settings_not_found(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': '/dev/null'}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('load the default settings from'))
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
def test_user_settings(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
def test_user_settings_remove(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
engine_names = [engine['name'] for engine in settings['engines']]
self.assertNotIn('wikinews', engine_names)
self.assertNotIn('wikibooks', engine_names)
self.assertIn('wikipedia', engine_names)
def test_user_settings_remove2(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
engine_names = [engine['name'] for engine in settings['engines']]
self.assertNotIn('wikinews', engine_names)
self.assertNotIn('wikibooks', engine_names)
self.assertIn('wikipedia', engine_names)
wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines']))
self.assertEqual(wikipedia[0]['engine'], 'wikipedia')
self.assertEqual(wikipedia[0]['tokens'], ['secret_token'])
newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines']))
self.assertEqual(newengine[0]['engine'], 'dummy')
def test_user_settings_keep_only(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('merge the default settings'))
engine_names = [engine['name'] for engine in settings['engines']]
self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine'])
# wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml
self.assertEqual(len(settings['engines'][2]), 1)
def test_custom_settings(self):
with patch.dict(settings_loader.environ,
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}):
settings, msg = settings_loader.load_settings()
self.assertTrue(msg.startswith('load the user settings from'))
self.assertEqual(settings['server']['port'], 9000)
self.assertEqual(settings['server']['secret_key'], "user_settings_secret")
engine_names = [engine['name'] for engine in settings['engines']]
self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote'])

View File

@ -1,98 +0,0 @@
#!/usr/bin/env python
# set path
from sys import path
from os.path import realpath, dirname, join
path.append(realpath(dirname(realpath(__file__)) + '/../'))
import argparse
import sys
import string
import ruamel.yaml
import secrets
import collections
from ruamel.yaml.scalarstring import SingleQuotedScalarString, DoubleQuotedScalarString
from searx.settings import load_settings, check_settings_yml, get_default_settings_path
from searx.exceptions import SearxSettingsException
RANDOM_STRING_LETTERS = string.ascii_lowercase + string.digits + string.ascii_uppercase
def get_random_string():
r = [secrets.choice(RANDOM_STRING_LETTERS) for _ in range(64)]
return ''.join(r)
def main(prog_arg):
yaml = ruamel.yaml.YAML()
yaml.preserve_quotes = True
yaml.indent(mapping=4, sequence=1, offset=2)
user_settings_path = prog_args.get('user-settings-yaml')
try:
default_settings, _ = load_settings(False)
if check_settings_yml(user_settings_path):
with open(user_settings_path, 'r', encoding='utf-8') as f:
user_settings = yaml.load(f.read())
new_user_settings = False
else:
user_settings = yaml.load('use_default_settings: True')
new_user_settings = True
except SearxSettingsException as e:
sys.stderr.write(str(e))
return
if not new_user_settings and not user_settings.get('use_default_settings'):
sys.stderr.write('settings.yml already exists and use_default_settings is not True')
return
user_settings['use_default_settings'] = True
use_default_settings_comment = "settings based on " + get_default_settings_path()
user_settings.yaml_add_eol_comment(use_default_settings_comment, 'use_default_settings')
if user_settings.get('server', {}).get('secret_key') in [None, 'ultrasecretkey']:
user_settings.setdefault('server', {})['secret_key'] = DoubleQuotedScalarString(get_random_string())
user_engines = user_settings.get('engines')
if user_engines:
has_user_engines = True
user_engines_dict = dict((definition['name'], definition) for definition in user_engines)
else:
has_user_engines = False
user_engines_dict = {}
user_engines = []
# remove old engines
if prog_arg.get('add-engines') or has_user_engines:
default_engines_dict = dict((definition['name'], definition) for definition in default_settings['engines'])
for i, engine in enumerate(user_engines):
if engine['name'] not in default_engines_dict:
del user_engines[i]
# add new engines
if prog_arg.get('add-engines'):
for engine in default_settings.get('engines', {}):
if engine['name'] not in user_engines_dict:
user_engines.append({'name': engine['name']})
user_settings['engines'] = user_engines
# output
if prog_arg.get('dry-run'):
yaml.dump(user_settings, sys.stdout)
else:
with open(user_settings_path, 'w', encoding='utf-8') as f:
yaml.dump(user_settings, f)
def parse_args():
parser = argparse.ArgumentParser(description='Update user settings.yml')
parser.add_argument('--add-engines', dest='add-engines', default=False, action='store_true', help='Add new engines')
parser.add_argument('--dry-run', dest='dry-run', default=False, action='store_true', help='Dry run')
parser.add_argument('user-settings-yaml', type=str)
return vars(parser.parse_args())
if __name__ == '__main__':
prog_args = parse_args()
main(prog_args)