mirror of https://github.com/searxng/searxng.git
[enh] settings.yml: add use_default_settings option (2nd version)
This commit is contained in:
Alexandre Flament
parent
1cfe7f2a75
commit
b4b81a5e1a
15
Makefile
15
Makefile
|
|
@ -266,19 +266,4 @@ test.clean:
|
|||
travis.codecov:
|
||||
$(Q)$(PY_ENV_BIN)/python -m pip install codecov
|
||||
|
||||
|
||||
# user-settings
|
||||
# -------------
|
||||
|
||||
PHONY += user-settings.create user-settings.update
|
||||
|
||||
user-settings.update: pyenvinstall
|
||||
$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
|
||||
$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH}
|
||||
|
||||
user-settings.update.engines: pyenvinstall
|
||||
$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
|
||||
$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH}
|
||||
|
||||
|
||||
.PHONY: $(PHONY)
|
||||
|
|
@ -235,68 +235,51 @@ In the following example, the actual settings are the default settings defined i
|
|||
|
||||
.. code-block:: yaml
|
||||
|
||||
use_default_settings: true
|
||||
use_default_settings: True
|
||||
server:
|
||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||
server:
|
||||
bind_address: "0.0.0.0"
|
||||
|
||||
With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section:
|
||||
With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
|
||||
|
||||
* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example).
|
||||
* If the ``engines`` section is defined then:
|
||||
|
||||
* searx loads only the engines declare in the user setttings.
|
||||
* searx merges the configuration according to the engine name.
|
||||
|
||||
In the following example, only three engines are available. Each engine configuration is merged with the default configuration.
|
||||
In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
use_default_settings: true
|
||||
server:
|
||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||
engines:
|
||||
- name: wikipedia
|
||||
- name: wikidata
|
||||
- name: ddg definitions
|
||||
|
||||
Another example where four engines are available. The arch linux wiki engine has a :ref:`token<private engines>`.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
use_default_settings: true
|
||||
use_default_settings: True
|
||||
server:
|
||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||
engines:
|
||||
- name: arch linux wiki
|
||||
tokens: ['$ecretValue']
|
||||
- name: wikipedia
|
||||
- name: wikidata
|
||||
- name: ddg definitions
|
||||
|
||||
automatic update
|
||||
----------------
|
||||
It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
|
||||
|
||||
The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case):
|
||||
.. code-block:: yaml
|
||||
|
||||
.. code-block:: sh
|
||||
use_default_settings:
|
||||
engines:
|
||||
remove:
|
||||
- google
|
||||
server:
|
||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||
engines:
|
||||
- name: arch linux wiki
|
||||
tokens: ['$ecretValue']
|
||||
|
||||
make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update
|
||||
As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
|
||||
|
||||
Set ``SEARX_SETTINGS_PATH`` to your user settings path.
|
||||
.. code-block:: yaml
|
||||
|
||||
As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated.
|
||||
The following command creates or updates the user settings including the ``engines`` section:
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines
|
||||
|
||||
After that ``/etc/searx/settings.yml``
|
||||
|
||||
* has a ``secret key``
|
||||
* has a ``engine`` section if it is not already the case, moreover the command:
|
||||
|
||||
* has deleted engines that do not exist in the default settings.
|
||||
* has added engines that exist in the default settings but are not declare in the user settings.
|
||||
use_default_settings:
|
||||
engines:
|
||||
keep_only:
|
||||
- google
|
||||
- duckduckgo
|
||||
server:
|
||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||
engines:
|
||||
- name: google
|
||||
tokens: ['$ecretValue']
|
||||
- name: duckduckgo
|
||||
tokens: ['$ecretValue']
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
|||
'''
|
||||
|
||||
import logging
|
||||
import searx.settings
|
||||
import searx.settings_loader
|
||||
from os import environ
|
||||
from os.path import realpath, dirname, join, abspath, isfile
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ from os.path import realpath, dirname, join, abspath, isfile
|
|||
searx_dir = abspath(dirname(__file__))
|
||||
engine_dir = dirname(realpath(__file__))
|
||||
static_path = abspath(join(dirname(__file__), 'static'))
|
||||
settings, settings_load_message = searx.settings.load_settings()
|
||||
settings, settings_load_message = searx.settings_loader.load_settings()
|
||||
|
||||
if settings['ui']['static_path']:
|
||||
static_path = settings['ui']['static_path']
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
import collections.abc
|
||||
|
||||
import yaml
|
||||
from searx.exceptions import SearxSettingsException
|
||||
from os import environ
|
||||
from os.path import dirname, join, abspath, isfile
|
||||
|
||||
|
||||
searx_dir = abspath(dirname(__file__))
|
||||
|
||||
|
||||
def check_settings_yml(file_name):
|
||||
if isfile(file_name):
|
||||
return file_name
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def load_yaml(file_name):
|
||||
try:
|
||||
with open(file_name, 'r', encoding='utf-8') as settings_yaml:
|
||||
settings = yaml.safe_load(settings_yaml)
|
||||
if not isinstance(settings, dict) or len(settings) == 0:
|
||||
raise SearxSettingsException('Empty file', file_name)
|
||||
return settings
|
||||
except IOError as e:
|
||||
raise SearxSettingsException(e, file_name)
|
||||
except yaml.YAMLError as e:
|
||||
raise SearxSettingsException(e, file_name)
|
||||
|
||||
|
||||
def get_default_settings_path():
|
||||
return check_settings_yml(join(searx_dir, 'settings.yml'))
|
||||
|
||||
|
||||
def get_user_settings_path():
|
||||
# find location of settings.yml
|
||||
if 'SEARX_SETTINGS_PATH' in environ:
|
||||
# if possible set path to settings using the
|
||||
# enviroment variable SEARX_SETTINGS_PATH
|
||||
return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
|
||||
else:
|
||||
# if not, get it from searx code base or last solution from /etc/searx
|
||||
return check_settings_yml('/etc/searx/settings.yml')
|
||||
|
||||
|
||||
def update_dict(d, u):
|
||||
for k, v in u.items():
|
||||
if isinstance(v, collections.abc.Mapping):
|
||||
d[k] = update_dict(d.get(k, {}), v)
|
||||
else:
|
||||
d[k] = v
|
||||
return d
|
||||
|
||||
|
||||
def update_settings(default_settings, user_settings):
|
||||
for k, v in user_settings.items():
|
||||
if k == 'use_default_settings':
|
||||
continue
|
||||
elif k == 'engines':
|
||||
default_engines = default_settings[k]
|
||||
default_engines_dict = dict((definition['name'], definition) for definition in default_engines)
|
||||
default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition)
|
||||
for definition in v]
|
||||
else:
|
||||
update_dict(default_settings[k], v)
|
||||
|
||||
return default_settings
|
||||
|
||||
|
||||
def load_settings(load_user_setttings=True):
|
||||
default_settings_path = get_default_settings_path()
|
||||
user_settings_path = get_user_settings_path()
|
||||
if user_settings_path is None or not load_user_setttings:
|
||||
# no user settings
|
||||
return (load_yaml(default_settings_path),
|
||||
'load the default settings from {}'.format(default_settings_path))
|
||||
|
||||
# user settings
|
||||
user_settings = load_yaml(user_settings_path)
|
||||
if user_settings.get('use_default_settings'):
|
||||
# the user settings are merged with the default configuration
|
||||
default_settings = load_yaml(default_settings_path)
|
||||
update_settings(default_settings, user_settings)
|
||||
return (default_settings,
|
||||
'merge the default settings ( {} ) and the user setttings ( {} )'
|
||||
.format(default_settings_path, user_settings_path))
|
||||
|
||||
# the user settings, fully replace the default configuration
|
||||
return (user_settings,
|
||||
'load the user settings from {}'.format(user_settings_path))
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
from os import environ
|
||||
from os.path import dirname, join, abspath, isfile
|
||||
from collections.abc import Mapping
|
||||
from itertools import filterfalse
|
||||
|
||||
import yaml
|
||||
|
||||
from searx.exceptions import SearxSettingsException
|
||||
|
||||
|
||||
searx_dir = abspath(dirname(__file__))
|
||||
|
||||
|
||||
def check_settings_yml(file_name):
|
||||
if isfile(file_name):
|
||||
return file_name
|
||||
return None
|
||||
|
||||
|
||||
def load_yaml(file_name):
|
||||
try:
|
||||
with open(file_name, 'r', encoding='utf-8') as settings_yaml:
|
||||
return yaml.safe_load(settings_yaml)
|
||||
except IOError as e:
|
||||
raise SearxSettingsException(e, file_name)
|
||||
except yaml.YAMLError as e:
|
||||
raise SearxSettingsException(e, file_name)
|
||||
|
||||
|
||||
def get_default_settings_path():
|
||||
return check_settings_yml(join(searx_dir, 'settings.yml'))
|
||||
|
||||
|
||||
def get_user_settings_path():
|
||||
# find location of settings.yml
|
||||
if 'SEARX_SETTINGS_PATH' in environ:
|
||||
# if possible set path to settings using the
|
||||
# enviroment variable SEARX_SETTINGS_PATH
|
||||
return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
|
||||
|
||||
# if not, get it from searx code base or last solution from /etc/searx
|
||||
return check_settings_yml('/etc/searx/settings.yml')
|
||||
|
||||
|
||||
def update_dict(default_dict, user_dict):
|
||||
for k, v in user_dict.items():
|
||||
if isinstance(v, Mapping):
|
||||
default_dict[k] = update_dict(default_dict.get(k, {}), v)
|
||||
else:
|
||||
default_dict[k] = v
|
||||
return default_dict
|
||||
|
||||
|
||||
def update_settings(default_settings, user_settings):
|
||||
# merge everything except the engines
|
||||
for k, v in user_settings.items():
|
||||
if k not in ('use_default_settings', 'engines'):
|
||||
update_dict(default_settings[k], v)
|
||||
|
||||
# parse the engines
|
||||
remove_engines = None
|
||||
keep_only_engines = None
|
||||
use_default_settings = user_settings.get('use_default_settings')
|
||||
if isinstance(use_default_settings, dict):
|
||||
remove_engines = use_default_settings.get('engines', {}).get('remove')
|
||||
keep_only_engines = use_default_settings.get('engines', {}).get('keep_only')
|
||||
|
||||
if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None:
|
||||
engines = default_settings['engines']
|
||||
|
||||
# parse "use_default_settings.engines.remove"
|
||||
if remove_engines is not None:
|
||||
engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines))
|
||||
|
||||
# parse "use_default_settings.engines.keep_only"
|
||||
if keep_only_engines is not None:
|
||||
engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines))
|
||||
|
||||
# parse "engines"
|
||||
user_engines = user_settings.get('engines')
|
||||
if user_engines:
|
||||
engines_dict = dict((definition['name'], definition) for definition in engines)
|
||||
for user_engine in user_engines:
|
||||
default_engine = engines_dict.get(user_engine['name'])
|
||||
if default_engine:
|
||||
update_dict(default_engine, user_engine)
|
||||
else:
|
||||
engines.append(user_engine)
|
||||
|
||||
# store the result
|
||||
default_settings['engines'] = engines
|
||||
|
||||
return default_settings
|
||||
|
||||
|
||||
def is_use_default_settings(user_settings):
|
||||
use_default_settings = user_settings.get('use_default_settings')
|
||||
if use_default_settings is True:
|
||||
return True
|
||||
if isinstance(use_default_settings, dict):
|
||||
return True
|
||||
if use_default_settings is False or use_default_settings is None:
|
||||
return False
|
||||
raise ValueError('Invalid value for use_default_settings')
|
||||
|
||||
|
||||
def load_settings(load_user_setttings=True):
|
||||
default_settings_path = get_default_settings_path()
|
||||
user_settings_path = get_user_settings_path()
|
||||
if user_settings_path is None or not load_user_setttings:
|
||||
# no user settings
|
||||
return (load_yaml(default_settings_path),
|
||||
'load the default settings from {}'.format(default_settings_path))
|
||||
|
||||
# user settings
|
||||
user_settings = load_yaml(user_settings_path)
|
||||
if is_use_default_settings(user_settings):
|
||||
# the user settings are merged with the default configuration
|
||||
default_settings = load_yaml(default_settings_path)
|
||||
update_settings(default_settings, user_settings)
|
||||
return (default_settings,
|
||||
'merge the default settings ( {} ) and the user setttings ( {} )'
|
||||
.format(default_settings_path, user_settings_path))
|
||||
|
||||
# the user settings, fully replace the default configuration
|
||||
return (user_settings,
|
||||
'load the user settings from {}'.format(user_settings_path))
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
Test:
|
||||
**********
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
general:
|
||||
debug : False
|
||||
instance_name : "searx"
|
||||
|
||||
search:
|
||||
safe_search : 0
|
||||
autocomplete : ""
|
||||
default_lang : ""
|
||||
ban_time_on_fail : 5
|
||||
max_ban_time_on_fail : 120
|
||||
|
||||
server:
|
||||
port : 9000
|
||||
bind_address : "0.0.0.0"
|
||||
secret_key : "user_settings_secret"
|
||||
base_url : False
|
||||
image_proxy : False
|
||||
http_protocol_version : "1.0"
|
||||
method: "POST"
|
||||
default_http_headers:
|
||||
X-Content-Type-Options : nosniff
|
||||
X-XSS-Protection : 1; mode=block
|
||||
X-Download-Options : noopen
|
||||
X-Robots-Tag : noindex, nofollow
|
||||
Referrer-Policy : no-referrer
|
||||
|
||||
ui:
|
||||
static_path : ""
|
||||
templates_path : ""
|
||||
default_theme : oscar
|
||||
default_locale : ""
|
||||
theme_args :
|
||||
oscar_style : logicodev
|
||||
|
||||
engines:
|
||||
- name : wikidata
|
||||
engine : wikidata
|
||||
shortcut : wd
|
||||
timeout : 3.0
|
||||
weight : 2
|
||||
|
||||
- name : wikibooks
|
||||
engine : mediawiki
|
||||
shortcut : wb
|
||||
categories : general
|
||||
base_url : "https://{language}.wikibooks.org/"
|
||||
number_of_results : 5
|
||||
search_type : text
|
||||
|
||||
- name : wikinews
|
||||
engine : mediawiki
|
||||
shortcut : wn
|
||||
categories : news
|
||||
base_url : "https://{language}.wikinews.org/"
|
||||
number_of_results : 5
|
||||
search_type : text
|
||||
|
||||
- name : wikiquote
|
||||
engine : mediawiki
|
||||
shortcut : wq
|
||||
categories : general
|
||||
base_url : "https://{language}.wikiquote.org/"
|
||||
number_of_results : 5
|
||||
search_type : text
|
||||
|
||||
locales:
|
||||
en : English
|
||||
ar : العَرَبِيَّة (Arabic)
|
||||
bg : Български (Bulgarian)
|
||||
bo : བོད་སྐད་ (Tibetian)
|
||||
ca : Català (Catalan)
|
||||
cs : Čeština (Czech)
|
||||
cy : Cymraeg (Welsh)
|
||||
da : Dansk (Danish)
|
||||
de : Deutsch (German)
|
||||
el_GR : Ελληνικά (Greek_Greece)
|
||||
eo : Esperanto (Esperanto)
|
||||
es : Español (Spanish)
|
||||
et : Eesti (Estonian)
|
||||
eu : Euskara (Basque)
|
||||
fa_IR : (fārsī) فارسى (Persian)
|
||||
fi : Suomi (Finnish)
|
||||
fil : Wikang Filipino (Filipino)
|
||||
fr : Français (French)
|
||||
gl : Galego (Galician)
|
||||
he : עברית (Hebrew)
|
||||
hr : Hrvatski (Croatian)
|
||||
hu : Magyar (Hungarian)
|
||||
ia : Interlingua (Interlingua)
|
||||
it : Italiano (Italian)
|
||||
ja : 日本語 (Japanese)
|
||||
lt : Lietuvių (Lithuanian)
|
||||
nl : Nederlands (Dutch)
|
||||
nl_BE : Vlaams (Dutch_Belgium)
|
||||
oc : Lenga D'òc (Occitan)
|
||||
pl : Polski (Polish)
|
||||
pt : Português (Portuguese)
|
||||
pt_BR : Português (Portuguese_Brazil)
|
||||
ro : Română (Romanian)
|
||||
ru : Русский (Russian)
|
||||
sk : Slovenčina (Slovak)
|
||||
sl : Slovenski (Slovene)
|
||||
sr : српски (Serbian)
|
||||
sv : Svenska (Swedish)
|
||||
te : తెలుగు (telugu)
|
||||
ta : தமிழ் (Tamil)
|
||||
tr : Türkçe (Turkish)
|
||||
uk : українська мова (Ukrainian)
|
||||
vi : tiếng việt (Vietnamese)
|
||||
zh : 中文 (Chinese)
|
||||
zh_TW : 國語 (Taiwanese Mandarin)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
use_default_settings:
|
||||
engines:
|
||||
keep_only:
|
||||
- wikibooks
|
||||
- wikinews
|
||||
server:
|
||||
secret_key: "user_secret_key"
|
||||
bind_address: "0.0.0.0"
|
||||
default_http_headers:
|
||||
Custom-Header: Custom-Value
|
||||
engines:
|
||||
- name: wikipedia
|
||||
- name: newengine
|
||||
engine: dummy
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
use_default_settings:
|
||||
engines:
|
||||
remove:
|
||||
- wikibooks
|
||||
- wikinews
|
||||
server:
|
||||
secret_key: "user_secret_key"
|
||||
bind_address: "0.0.0.0"
|
||||
default_http_headers:
|
||||
Custom-Header: Custom-Value
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
use_default_settings:
|
||||
engines:
|
||||
remove:
|
||||
- wikibooks
|
||||
- wikinews
|
||||
server:
|
||||
secret_key: "user_secret_key"
|
||||
bind_address: "0.0.0.0"
|
||||
default_http_headers:
|
||||
Custom-Header: Custom-Value
|
||||
engines:
|
||||
- name: wikipedia
|
||||
tokens: ['secret_token']
|
||||
- name: newengine
|
||||
engine: dummy
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
use_default_settings: True
|
||||
server:
|
||||
secret_key: "user_secret_key"
|
||||
bind_address: "0.0.0.0"
|
||||
default_http_headers:
|
||||
Custom-Header: Custom-Value
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
from os.path import dirname, join, abspath
|
||||
from unittest.mock import patch
|
||||
|
||||
from searx.testing import SearxTestCase
|
||||
from searx.exceptions import SearxSettingsException
|
||||
from searx import settings_loader
|
||||
|
||||
|
||||
test_dir = abspath(dirname(__file__))
|
||||
|
||||
|
||||
class TestLoad(SearxTestCase):
|
||||
|
||||
def test_load_zero(self):
|
||||
with self.assertRaises(SearxSettingsException):
|
||||
settings_loader.load_yaml('/dev/zero')
|
||||
|
||||
with self.assertRaises(SearxSettingsException):
|
||||
settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml'))
|
||||
|
||||
with self.assertRaises(SearxSettingsException):
|
||||
settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml'))
|
||||
|
||||
def test_check_settings_yml(self):
|
||||
self.assertIsNone(settings_loader.check_settings_yml('/dev/zero'))
|
||||
|
||||
bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml')
|
||||
self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path)
|
||||
|
||||
|
||||
class TestDefaultSettings(SearxTestCase):
|
||||
|
||||
def test_load(self):
|
||||
settings, msg = settings_loader.load_settings(load_user_setttings=False)
|
||||
self.assertTrue(msg.startswith('load the default settings from'))
|
||||
self.assertFalse(settings['general']['debug'])
|
||||
self.assertTrue(isinstance(settings['general']['instance_name'], str))
|
||||
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
|
||||
self.assertTrue(isinstance(settings['server']['port'], int))
|
||||
self.assertTrue(isinstance(settings['server']['bind_address'], str))
|
||||
self.assertTrue(isinstance(settings['engines'], list))
|
||||
self.assertTrue(isinstance(settings['locales'], dict))
|
||||
self.assertTrue(isinstance(settings['doi_resolvers'], dict))
|
||||
self.assertTrue(isinstance(settings['default_doi_resolver'], str))
|
||||
|
||||
|
||||
class TestUserSettings(SearxTestCase):
|
||||
|
||||
def test_is_use_default_settings(self):
|
||||
self.assertFalse(settings_loader.is_use_default_settings({}))
|
||||
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True}))
|
||||
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}}))
|
||||
with self.assertRaises(ValueError):
|
||||
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1}))
|
||||
with self.assertRaises(ValueError):
|
||||
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0}))
|
||||
|
||||
def test_user_settings_not_found(self):
|
||||
with patch.dict(settings_loader.environ,
|
||||
{'SEARX_SETTINGS_PATH': '/dev/null'}):
|
||||
settings, msg = settings_loader.load_settings()
|
||||
self.assertTrue(msg.startswith('load the default settings from'))
|
||||
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
|
||||
|
||||
def test_user_settings(self):
|
||||
with patch.dict(settings_loader.environ,
|
||||
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}):
|
||||
settings, msg = settings_loader.load_settings()
|
||||
self.assertTrue(msg.startswith('merge the default settings'))
|
||||
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
|
||||
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
|
||||
|
||||
def test_user_settings_remove(self):
|
||||
with patch.dict(settings_loader.environ,
|
||||
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}):
|
||||
settings, msg = settings_loader.load_settings()
|
||||
self.assertTrue(msg.startswith('merge the default settings'))
|
||||
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
|
||||
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
|
||||
engine_names = [engine['name'] for engine in settings['engines']]
|
||||
self.assertNotIn('wikinews', engine_names)
|
||||
self.assertNotIn('wikibooks', engine_names)
|
||||
self.assertIn('wikipedia', engine_names)
|
||||
|
||||
def test_user_settings_remove2(self):
|
||||
with patch.dict(settings_loader.environ,
|
||||
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}):
|
||||
settings, msg = settings_loader.load_settings()
|
||||
self.assertTrue(msg.startswith('merge the default settings'))
|
||||
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
|
||||
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
|
||||
engine_names = [engine['name'] for engine in settings['engines']]
|
||||
self.assertNotIn('wikinews', engine_names)
|
||||
self.assertNotIn('wikibooks', engine_names)
|
||||
self.assertIn('wikipedia', engine_names)
|
||||
wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines']))
|
||||
self.assertEqual(wikipedia[0]['engine'], 'wikipedia')
|
||||
self.assertEqual(wikipedia[0]['tokens'], ['secret_token'])
|
||||
newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines']))
|
||||
self.assertEqual(newengine[0]['engine'], 'dummy')
|
||||
|
||||
def test_user_settings_keep_only(self):
|
||||
with patch.dict(settings_loader.environ,
|
||||
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}):
|
||||
settings, msg = settings_loader.load_settings()
|
||||
self.assertTrue(msg.startswith('merge the default settings'))
|
||||
engine_names = [engine['name'] for engine in settings['engines']]
|
||||
self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine'])
|
||||
# wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml
|
||||
self.assertEqual(len(settings['engines'][2]), 1)
|
||||
|
||||
def test_custom_settings(self):
|
||||
with patch.dict(settings_loader.environ,
|
||||
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}):
|
||||
settings, msg = settings_loader.load_settings()
|
||||
self.assertTrue(msg.startswith('load the user settings from'))
|
||||
self.assertEqual(settings['server']['port'], 9000)
|
||||
self.assertEqual(settings['server']['secret_key'], "user_settings_secret")
|
||||
engine_names = [engine['name'] for engine in settings['engines']]
|
||||
self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote'])
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# set path
|
||||
from sys import path
|
||||
from os.path import realpath, dirname, join
|
||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import string
|
||||
import ruamel.yaml
|
||||
import secrets
|
||||
import collections
|
||||
from ruamel.yaml.scalarstring import SingleQuotedScalarString, DoubleQuotedScalarString
|
||||
from searx.settings import load_settings, check_settings_yml, get_default_settings_path
|
||||
from searx.exceptions import SearxSettingsException
|
||||
|
||||
|
||||
RANDOM_STRING_LETTERS = string.ascii_lowercase + string.digits + string.ascii_uppercase
|
||||
|
||||
|
||||
def get_random_string():
|
||||
r = [secrets.choice(RANDOM_STRING_LETTERS) for _ in range(64)]
|
||||
return ''.join(r)
|
||||
|
||||
|
||||
def main(prog_arg):
|
||||
yaml = ruamel.yaml.YAML()
|
||||
yaml.preserve_quotes = True
|
||||
yaml.indent(mapping=4, sequence=1, offset=2)
|
||||
user_settings_path = prog_args.get('user-settings-yaml')
|
||||
|
||||
try:
|
||||
default_settings, _ = load_settings(False)
|
||||
if check_settings_yml(user_settings_path):
|
||||
with open(user_settings_path, 'r', encoding='utf-8') as f:
|
||||
user_settings = yaml.load(f.read())
|
||||
new_user_settings = False
|
||||
else:
|
||||
user_settings = yaml.load('use_default_settings: True')
|
||||
new_user_settings = True
|
||||
except SearxSettingsException as e:
|
||||
sys.stderr.write(str(e))
|
||||
return
|
||||
|
||||
if not new_user_settings and not user_settings.get('use_default_settings'):
|
||||
sys.stderr.write('settings.yml already exists and use_default_settings is not True')
|
||||
return
|
||||
|
||||
user_settings['use_default_settings'] = True
|
||||
use_default_settings_comment = "settings based on " + get_default_settings_path()
|
||||
user_settings.yaml_add_eol_comment(use_default_settings_comment, 'use_default_settings')
|
||||
|
||||
if user_settings.get('server', {}).get('secret_key') in [None, 'ultrasecretkey']:
|
||||
user_settings.setdefault('server', {})['secret_key'] = DoubleQuotedScalarString(get_random_string())
|
||||
|
||||
user_engines = user_settings.get('engines')
|
||||
if user_engines:
|
||||
has_user_engines = True
|
||||
user_engines_dict = dict((definition['name'], definition) for definition in user_engines)
|
||||
else:
|
||||
has_user_engines = False
|
||||
user_engines_dict = {}
|
||||
user_engines = []
|
||||
|
||||
# remove old engines
|
||||
if prog_arg.get('add-engines') or has_user_engines:
|
||||
default_engines_dict = dict((definition['name'], definition) for definition in default_settings['engines'])
|
||||
for i, engine in enumerate(user_engines):
|
||||
if engine['name'] not in default_engines_dict:
|
||||
del user_engines[i]
|
||||
|
||||
# add new engines
|
||||
if prog_arg.get('add-engines'):
|
||||
for engine in default_settings.get('engines', {}):
|
||||
if engine['name'] not in user_engines_dict:
|
||||
user_engines.append({'name': engine['name']})
|
||||
user_settings['engines'] = user_engines
|
||||
|
||||
# output
|
||||
if prog_arg.get('dry-run'):
|
||||
yaml.dump(user_settings, sys.stdout)
|
||||
else:
|
||||
with open(user_settings_path, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(user_settings, f)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Update user settings.yml')
|
||||
parser.add_argument('--add-engines', dest='add-engines', default=False, action='store_true', help='Add new engines')
|
||||
parser.add_argument('--dry-run', dest='dry-run', default=False, action='store_true', help='Dry run')
|
||||
parser.add_argument('user-settings-yaml', type=str)
|
||||
return vars(parser.parse_args())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
prog_args = parse_args()
|
||||
main(prog_args)
|
||||
Loading…
Reference in New Issue