From 4b07df62e5906e98315e9e856db8f39b2f28f36e Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 28 May 2021 18:45:22 +0200 Subject: [PATCH 1/3] [mod] move all default settings into searx.settings_defaults --- searx/__init__.py | 40 +++----- searx/engines/__init__.py | 2 +- searx/network/network.py | 28 +++--- searx/plugins/__init__.py | 10 +- searx/preferences.py | 22 ++--- searx/settings_defaults.py | 194 +++++++++++++++++++++++++++++++++++++ searx/utils.py | 55 +---------- searx/webapp.py | 46 +++------ searx/webutils.py | 8 -- 9 files changed, 249 insertions(+), 156 deletions(-) create mode 100644 searx/settings_defaults.py diff --git a/searx/__init__.py b/searx/__init__.py index 6aac98713..ded0ff2f5 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -17,37 +17,18 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. import logging import searx.settings_loader -from os import environ -from os.path import realpath, dirname, join, abspath, isfile +from searx.settings_defaults import settings_set_defaults +from os.path import dirname, abspath searx_dir = abspath(dirname(__file__)) searx_parent_dir = abspath(dirname(dirname(__file__))) -engine_dir = dirname(realpath(__file__)) -static_path = abspath(join(dirname(__file__), 'static')) settings, settings_load_message = searx.settings_loader.load_settings() -if settings['ui']['static_path']: - static_path = settings['ui']['static_path'] - -''' -enable debug if -the environnement variable SEARX_DEBUG is 1 or true -(whatever the value in settings.yml) -or general.debug=True in settings.yml -disable debug if -the environnement variable SEARX_DEBUG is 0 or false -(whatever the value in settings.yml) -or general.debug=False in settings.yml -''' -searx_debug_env = environ.get('SEARX_DEBUG', '').lower() -if searx_debug_env == 'true' or searx_debug_env == '1': - searx_debug = True -elif searx_debug_env == 'false' or searx_debug_env == '0': - searx_debug = False -else: - searx_debug = settings.get('general', {}).get('debug') +if settings is not None: + settings = settings_set_defaults(settings) +searx_debug = settings['general']['debug'] if searx_debug: logging.basicConfig(level=logging.DEBUG) else: @@ -55,12 +36,13 @@ else: logger = logging.getLogger('searx') logger.info(settings_load_message) -logger.info('Initialisation done') -if 'SEARX_SECRET' in environ: - settings['server']['secret_key'] = environ['SEARX_SECRET'] -if 'SEARX_BIND_ADDRESS' in environ: - settings['server']['bind_address'] = environ['SEARX_BIND_ADDRESS'] +# log max_request_timeout +max_request_timeout = settings['outgoing']['max_request_timeout'] +if max_request_timeout is None: + logger.info('max_request_timeout=%s', repr(max_request_timeout)) +else: + logger.info('max_request_timeout=%i second(s)', max_request_timeout) class _brand_namespace: diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 15212afd9..49990c325 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -144,7 +144,7 @@ def load_engine(engine_data): # exclude onion engines if not using tor. return None - engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0) + engine.timeout += settings['outgoing']['extra_proxy_timeout'] for category_name in engine.categories: categories.setdefault(category_name, []).append(engine) diff --git a/searx/network/network.py b/searx/network/network.py index 37777b0d5..e7dc5b56e 100644 --- a/searx/network/network.py +++ b/searx/network/network.py @@ -224,28 +224,22 @@ def initialize(settings_engines=None, settings_outgoing=None): global NETWORKS - settings_engines = settings_engines or settings.get('engines') - settings_outgoing = settings_outgoing or settings.get('outgoing') + settings_engines = settings_engines or settings['engines'] + settings_outgoing = settings_outgoing or settings['outgoing'] # default parameters for AsyncHTTPTransport # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long default_params = { 'enable_http': False, 'verify': True, - 'enable_http2': settings_outgoing.get('enable_http2', True), - # Magic number kept from previous code - 'max_connections': settings_outgoing.get('pool_connections', 100), - # Picked from constructor - 'max_keepalive_connections': settings_outgoing.get('pool_maxsize', 10), - # - 'keepalive_expiry': settings_outgoing.get('keepalive_expiry', 5.0), - 'local_addresses': settings_outgoing.get('source_ips'), - 'proxies': settings_outgoing.get('proxies'), - # default maximum redirect - # from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55 - 'max_redirects': settings_outgoing.get('max_redirects', 30), - # - 'retries': settings_outgoing.get('retries', 0), + 'enable_http2': settings_outgoing['enable_http2'], + 'max_connections': settings_outgoing['pool_connections'], + 'max_keepalive_connections': settings_outgoing['pool_maxsize'], + 'keepalive_expiry': settings_outgoing['keepalive_expiry'], + 'local_addresses': settings_outgoing['source_ips'], + 'proxies': settings_outgoing['proxies'], + 'max_redirects': settings_outgoing['max_redirects'], + 'retries': settings_outgoing['retries'], 'retry_on_http_error': None, } @@ -274,7 +268,7 @@ def initialize(settings_engines=None, settings_outgoing=None): NETWORKS['ipv6'] = new_network({'local_addresses': '::'}) # define networks from outgoing.networks - for network_name, network in settings_outgoing.get('networks', {}).items(): + for network_name, network in settings_outgoing['networks'].items(): NETWORKS[network_name] = new_network(network) # define networks from engines.[i].network (except references) diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 5e2829201..22f475875 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -21,7 +21,7 @@ from os import listdir, makedirs, remove, stat, utime from os.path import abspath, basename, dirname, exists, join from shutil import copyfile -from searx import logger, settings, static_path +from searx import logger, settings logger = logger.getChild('plugins') @@ -123,7 +123,7 @@ def sync_resource(base_path, resource_path, name, target_dir, plugin_dir): def prepare_package_resources(pkg, name): plugin_dir = 'plugin_' + name - target_dir = join(static_path, 'plugins/external_plugins', plugin_dir) + target_dir = join(settings['ui']['static_path'], 'plugins/external_plugins', plugin_dir) try: makedirs(target_dir, exist_ok=True) except: @@ -170,10 +170,10 @@ plugins.register(search_on_category_select) plugins.register(tracker_url_remover) plugins.register(vim_hotkeys) # load external plugins -if 'plugins' in settings: +if settings['plugins']: plugins.register(*settings['plugins'], external=True) -if 'enabled_plugins' in settings: +if settings['enabled_plugins']: for plugin in plugins: if plugin.name in settings['enabled_plugins']: plugin.default_on = True @@ -181,5 +181,5 @@ if 'enabled_plugins' in settings: plugin.default_on = False # load tor specific plugins -if settings['outgoing'].get('using_tor_proxy'): +if settings['outgoing']['using_tor_proxy']: plugins.register(ahmia_filter) diff --git a/searx/preferences.py b/searx/preferences.py index b0106b195..46ce53ab3 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -333,25 +333,25 @@ class Preferences: choices=categories + ['none'] ), 'language': SearchLanguageSetting( - settings['search'].get('default_lang', ''), + settings['search']['default_lang'], is_locked('language'), choices=list(LANGUAGE_CODES) + [''] ), 'locale': EnumStringSetting( - settings['ui'].get('default_locale', ''), + settings['ui']['default_locale'], is_locked('locale'), choices=list(settings['locales'].keys()) + [''] ), 'autocomplete': EnumStringSetting( - settings['search'].get('autocomplete', ''), + settings['search']['autocomplete'], is_locked('autocomplete'), choices=list(autocomplete.backends.keys()) + [''] ), 'image_proxy': MapSetting( - settings['server'].get('image_proxy', False), + settings['server']['image_proxy'], is_locked('image_proxy'), map={ - '': settings['server'].get('image_proxy', 0), + '': settings['server']['image_proxy'], '0': False, '1': True, 'True': True, @@ -359,12 +359,12 @@ class Preferences: } ), 'method': EnumStringSetting( - settings['server'].get('method', 'POST'), + settings['server']['method'], is_locked('method'), choices=('GET', 'POST') ), 'safesearch': MapSetting( - settings['search'].get('safe_search', 0), + settings['search']['safe_search'], is_locked('safesearch'), map={ '0': 0, @@ -373,12 +373,12 @@ class Preferences: } ), 'theme': EnumStringSetting( - settings['ui'].get('default_theme', 'oscar'), + settings['ui']['default_theme'], is_locked('theme'), choices=themes ), 'results_on_new_tab': MapSetting( - settings['ui'].get('results_on_new_tab', False), + settings['ui']['results_on_new_tab'], is_locked('results_on_new_tab'), map={ '0': False, @@ -393,11 +393,11 @@ class Preferences: choices=DOI_RESOLVERS ), 'oscar-style': EnumStringSetting( - settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'), + settings['ui']['theme_args']['oscar_style'], is_locked('oscar-style'), choices=['', 'logicodev', 'logicodev-dark', 'pointhi']), 'advanced_search': MapSetting( - settings['ui'].get('advanced_search', False), + settings['ui']['advanced_search'], is_locked('advanced_search'), map={ '0': False, diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py new file mode 100644 index 000000000..d1a57369d --- /dev/null +++ b/searx/settings_defaults.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +# pylint: disable=missing-function-docstring, missing-module-docstring + +import typing +import numbers +import errno +import os +import logging +from os.path import dirname, abspath + +from searx.languages import language_codes as languages + +searx_dir = abspath(dirname(__file__)) + +logger = logging.getLogger('searx') +OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss'] +LANGUAGE_CODES = ('', 'all') + tuple(l[0] for l in languages) +OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi') +CATEGORY_ORDER = [ + 'general', + 'images', + 'videos', + 'news', + 'map', + 'music', + 'it', + 'science', + 'files', + 'social medias', +] +STR_TO_BOOL = { + '0': False, + 'false': False, + 'off': False, + '1': True, + 'true': True, + 'on': True, +} +_UNDEFINED = object() + + +class SettingsValue: + """Check and update a setting value + """ + + def __init__(self, + type_definition: typing.Union[None, typing.Any, typing.Tuple[typing.Any]]=None, + default: typing.Any=None, + environ_name: str=None): + self.type_definition = type_definition \ + if type_definition is None or isinstance(type_definition, tuple) \ + else (type_definition,) + self.default = default + self.environ_name = environ_name + + @property + def type_definition_repr(self): + types_str = [t.__name__ if isinstance(t, type) else repr(t) + for t in self.type_definition] + return ', '.join(types_str) + + def check_type_definition(self, value: typing.Any) -> None: + if value in self.type_definition: + return + type_list = tuple(t for t in self.type_definition if isinstance(t, type)) + if not isinstance(value, type_list): + raise ValueError('The value has to be one of these types/values: {}'\ + .format(self.type_definition_repr)) + + def __call__(self, value: typing.Any) -> typing.Any: + if value == _UNDEFINED: + value = self.default + # override existing value with environ + if self.environ_name and self.environ_name in os.environ: + value = os.environ[self.environ_name] + if self.type_definition == (bool,): + value = STR_TO_BOOL[value.lower()] + # + self.check_type_definition(value) + return value + + +class SettingsDirectoryValue(SettingsValue): + """Check and update a setting value that is a directory path + """ + + def check_type_definition(self, value: typing.Any) -> typing.Any: + super().check_type_definition(value) + if not os.path.isdir(value): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), value) + + def __call__(self, value: typing.Any) -> typing.Any: + if value == '': + value = self.default + return super().__call__(value) + + +def apply_schema(settings, schema, path_list): + error = False + for key, value in schema.items(): + if isinstance(value, SettingsValue): + try: + settings[key] = value(settings.get(key, _UNDEFINED)) + except Exception as e: # pylint: disable=broad-except + # don't stop now: check other values + logger.error('%s: %s', '.'.join([*path_list, key]), e) + error = True + elif isinstance(value, dict): + error = error or apply_schema(settings.setdefault(key, {}), schema[key], [*path_list, key]) + else: + settings.setdefault(key, value) + if len(path_list) == 0 and error: + raise ValueError('Invalid settings.yml') + return error + + +SCHEMA = { + 'general': { + 'debug': SettingsValue(bool, False, 'SEARX_DEBUG'), + 'instance_name': SettingsValue(str, 'searxng'), + 'contact_url': SettingsValue((None, False, str), None), + }, + 'brand': { + }, + 'search': { + 'safe_search': SettingsValue((0,1,2), 0), + 'autocomplete': SettingsValue(str, ''), + 'default_lang': SettingsValue(LANGUAGE_CODES, ''), + 'ban_time_on_fail': SettingsValue(numbers.Real, 5), + 'max_ban_time_on_fail': SettingsValue(numbers.Real, 120), + 'formats': SettingsValue(list, OUTPUT_FORMATS), + }, + 'server': { + 'port': SettingsValue(int, 8888), + 'bind_address': SettingsValue(str, '127.0.0.1', 'SEARX_BIND_ADDRESS'), + 'secret_key': SettingsValue(str, environ_name='SEARX_SECRET'), + 'base_url': SettingsValue((False, str), False), + 'image_proxy': SettingsValue(bool, False), + 'http_protocol_version': SettingsValue(('1.0', '1.1'), '1.0'), + 'method': SettingsValue(('POST', 'GET'), 'POST'), + 'default_http_headers': SettingsValue(dict, {}), + }, + 'ui': { + 'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')), + 'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')), + 'default_theme': SettingsValue(str, 'oscar'), + 'default_locale': SettingsValue(str, ''), + 'theme_args': { + 'oscar_style': SettingsValue(OSCAR_STYLE, 'logicodev'), + }, + 'results_on_new_tab': SettingsValue(bool, False), + 'advanced_search': SettingsValue(bool, False), + 'categories_order': SettingsValue(list, CATEGORY_ORDER), + }, + 'preferences': { + 'lock': SettingsValue(list, []), + }, + 'outgoing': { + 'useragent_suffix': SettingsValue(str, ''), + 'request_timeout': SettingsValue(numbers.Real, 3.0), + 'enable_http2': SettingsValue(bool, True), + 'max_request_timeout': SettingsValue((None, numbers.Real), None), + # Magic number kept from previous code + 'pool_connections': SettingsValue(int, 100), + # Picked from constructor + 'pool_maxsize': SettingsValue(int, 10), + 'keepalive_expiry': SettingsValue(numbers.Real, 5.0), + # default maximum redirect + # from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55 + 'max_redirects': SettingsValue(int, 30), + 'retries': SettingsValue(int, 0), + 'proxies': SettingsValue((None, str, dict), None), + 'source_ips': SettingsValue((None, str, list), None), + # Tor configuration + 'using_tor_proxy': SettingsValue(bool, False), + 'extra_proxy_timeout': SettingsValue(int, 0), + 'networks': { + }, + }, + 'plugins': SettingsValue((None, list), None), + 'enabled_plugins': SettingsValue(list, []), + 'checker': { + 'off_when_debug': SettingsValue(bool, True), + }, + 'engines': SettingsValue(list, []), + 'locales': SettingsValue(dict, {'en': 'English'}), + 'doi_resolvers': { + }, +} + +def settings_set_defaults(settings): + apply_schema(settings, SCHEMA, []) + return settings diff --git a/searx/utils.py b/searx/utils.py index ad69816ce..bf9957df4 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -8,7 +8,6 @@ from os.path import splitext, join from random import choice from html.parser import HTMLParser from urllib.parse import urljoin, urlparse -from collections.abc import Mapping from lxml import html from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult @@ -46,7 +45,7 @@ def searx_useragent(): """Return the searx User Agent""" return 'searx/{searx_version} {suffix}'.format( searx_version=VERSION_STRING, - suffix=settings['outgoing'].get('useragent_suffix', '')).strip() + suffix=settings['outgoing']['useragent_suffix'].strip()) def gen_useragent(os=None): @@ -501,58 +500,6 @@ def get_engine_from_settings(name): return {} -NOT_EXISTS = object() -"""Singleton used by :py:obj:`get_value` if a key does not exists.""" - - -def get_value(dictionary, *keys, default=NOT_EXISTS): - """Return the value from a *deep* mapping type (e.g. the ``settings`` object - from yaml). If the path to the *key* does not exists a :py:obj:`NOT_EXISTS` - is returned (non ``KeyError`` exception is raised). - - .. code: python - - >>> from searx import settings - >>> from searx.utils import get_value, NOT_EXISTS - >>> get_value(settings, 'checker', 'additional_tests', 'rosebud', 'result_container') - ['not_empty', ['one_title_contains', 'citizen kane']] - - >>> get_value(settings, 'search', 'xxx') is NOT_EXISTS - True - >>> get_value(settings, 'search', 'formats') - ['html', 'csv', 'json', 'rss'] - - The list returned from the ``search.format`` key is not a mapping type, you - can't traverse along non-mapping types. If you try it, you will get a - :py:ref:`NOT_EXISTS`: - - .. code: python - - >>> get_value(settings, 'search', 'format', 'csv') is NOT_EXISTS - True - >>> get_value(settings, 'search', 'formats')[1] - 'csv' - - For convenience you can replace :py:ref:`NOT_EXISTS` by a default value of - your choice: - - .. code: python - - if 'csv' in get_value(settings, 'search', 'formats', default=[]): - print("csv format is denied") - - """ - - obj = dictionary - for k in keys: - if not isinstance(obj, Mapping): - raise TypeError("expected mapping type, got %s" % type(obj)) - obj = obj.get(k, default) - if obj is default: - return obj - return obj - - def get_xpath(xpath_spec): """Return cached compiled XPath diff --git a/searx/webapp.py b/searx/webapp.py index 47f77acc7..2496fede0 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -56,12 +56,12 @@ from flask_babel import ( ) from searx import logger -from searx import brand, static_path +from searx import brand from searx import ( settings, - searx_dir, searx_debug, ) +from searx.settings_defaults import OUTPUT_FORMATS from searx.exceptions import SearxParameterException from searx.engines import ( categories, @@ -71,7 +71,6 @@ from searx.engines import ( from searx.webutils import ( UnicodeWriter, highlight_content, - get_resources_directory, get_static_files, get_result_templates, get_themes, @@ -88,7 +87,6 @@ from searx.utils import ( gen_useragent, dict_subset, match_language, - get_value, ) from searx.version import VERSION_STRING from searx.query import RawTextQuery @@ -139,7 +137,7 @@ if sys.version_info[0] < 3: logger = logger.getChild('webapp') # serve pages with HTTP/1.1 -WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0')) +WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version']) # check secret_key if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey': @@ -147,25 +145,22 @@ if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey': sys.exit(1) # about static -static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path']) -logger.debug('static directory is %s', static_path) -static_files = get_static_files(static_path) +logger.debug('static directory is %s', settings['ui']['static_path']) +static_files = get_static_files(settings['ui']['static_path']) # about templates +logger.debug('templates directory is %s', settings['ui']['templates_path']) default_theme = settings['ui']['default_theme'] -templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path']) -logger.debug('templates directory is %s', templates_path) +templates_path = settings['ui']['templates_path'] themes = get_themes(templates_path) result_templates = get_result_templates(templates_path) global_favicons = [] for indice, theme in enumerate(themes): global_favicons.append([]) - theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons') + theme_img_path = os.path.join(settings['ui']['static_path'], 'themes', theme, 'img', 'icons') for (dirpath, dirnames, filenames) in os.walk(theme_img_path): global_favicons[indice].extend(filenames) -OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss'] - STATS_SORT_PARAMETERS = { 'name': (False, 'name', ''), 'score': (True, 'score', 0), @@ -177,7 +172,7 @@ STATS_SORT_PARAMETERS = { # Flask app app = Flask( __name__, - static_folder=static_path, + static_folder=settings['ui']['static_path'], template_folder=templates_path ) @@ -517,8 +512,7 @@ def render(template_name, override_theme=None, **kwargs): kwargs['preferences'] = request.preferences kwargs['search_formats'] = [ - x for x in get_value( - settings, 'search', 'formats', default=OUTPUT_FORMATS) + x for x in settings['search']['formats'] if x != 'html'] kwargs['brand'] = brand @@ -545,12 +539,7 @@ def render(template_name, override_theme=None, **kwargs): def _get_ordered_categories(): - ordered_categories = [] - if 'categories_order' not in settings['ui']: - ordered_categories = ['general'] - ordered_categories.extend(x for x in sorted(categories.keys()) if x != 'general') - return ordered_categories - ordered_categories = settings['ui']['categories_order'] + ordered_categories = list(settings['ui']['categories_order']) ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories) return ordered_categories @@ -610,7 +599,7 @@ def pre_request(): @app.after_request def add_default_headers(response): # set default http headers - for header, value in settings['server'].get('default_http_headers', {}).items(): + for header, value in settings['server']['default_http_headers'].items(): if header in response.headers: continue response.headers[header] = value @@ -696,7 +685,7 @@ def search(): if output_format not in OUTPUT_FORMATS: output_format = 'html' - if output_format not in get_value(settings, 'search', 'formats', default=OUTPUT_FORMATS): + if output_format not in settings['search']['formats']: flask.abort(403) # check if there is query (not None and not an empty string) @@ -1069,11 +1058,6 @@ def preferences(): 'time_range_support': time_range_support, } - # - locked_preferences = list() - if 'preferences' in settings and 'lock' in settings['preferences']: - locked_preferences = settings['preferences']['lock'] - # return render('preferences.html', selected_categories=get_selected_categories(request.preferences, request.form), @@ -1098,7 +1082,7 @@ def preferences(): theme=get_current_theme_name(), preferences_url_params=request.preferences.get_as_url_params(), base_url=get_base_url(), - locked_preferences=locked_preferences, + locked_preferences=settings['preferences']['lock'], preferences=True) @@ -1271,7 +1255,7 @@ def favicon(): return send_from_directory( os.path.join( app.root_path, - static_path, + settings['ui']['static_path'], 'themes', get_current_theme_name(), 'img'), diff --git a/searx/webutils.py b/searx/webutils.py index 2464a097f..c27324908 100644 --- a/searx/webutils.py +++ b/searx/webutils.py @@ -47,14 +47,6 @@ class UnicodeWriter: self.writerow(row) -def get_resources_directory(searx_directory, subdirectory, resources_directory): - if not resources_directory: - resources_directory = os.path.join(searx_directory, subdirectory) - if not os.path.isdir(resources_directory): - raise Exception(resources_directory + " is not a directory") - return resources_directory - - def get_themes(templates_path): """Returns available themes list.""" themes = os.listdir(templates_path) From 6fa114c9ba0f9e20168754dd4cbd03e5defef631 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 1 Jun 2021 08:07:26 +0200 Subject: [PATCH 2/3] [mod] settings_default: remove searx.search.max_request_timeout global variable --- searx/search/__init__.py | 12 +----------- tests/unit/test_search.py | 11 ++++++----- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/searx/search/__init__.py b/searx/search/__init__.py index 49a98825c..041a54c4b 100644 --- a/searx/search/__init__.py +++ b/searx/search/__init__.py @@ -23,17 +23,6 @@ from searx.search.checker import initialize as initialize_checker logger = logger.getChild('search') -max_request_timeout = settings.get('outgoing', {}).get('max_request_timeout' or None) -if max_request_timeout is None: - logger.info('max_request_timeout={0}'.format(max_request_timeout)) -else: - if isinstance(max_request_timeout, float): - logger.info('max_request_timeout={0} second(s)'.format(max_request_timeout)) - else: - logger.critical('outgoing.max_request_timeout if defined has to be float') - import sys - sys.exit(1) - def initialize(settings_engines=None, enable_checker=False): settings_engines = settings_engines or settings['engines'] @@ -115,6 +104,7 @@ class Search: default_timeout = max(default_timeout, processor.engine.timeout) # adjust timeout + max_request_timeout = settings['outgoing']['max_request_timeout'] actual_timeout = default_timeout query_timeout = self.search_query.timeout_limit diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index 72732032e..b0988a0c8 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -2,6 +2,7 @@ from searx.testing import SearxTestCase from searx.search import SearchQuery, EngineRef +from searx import settings import searx.search @@ -41,7 +42,7 @@ class SearchTestCase(SearxTestCase): searx.search.initialize(TEST_ENGINES) def test_timeout_simple(self): - searx.search.max_request_timeout = None + settings['outgoing']['max_request_timeout'] = None search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, None) search = searx.search.Search(search_query) @@ -49,7 +50,7 @@ class SearchTestCase(SearxTestCase): self.assertEqual(search.actual_timeout, 3.0) def test_timeout_query_above_default_nomax(self): - searx.search.max_request_timeout = None + settings['outgoing']['max_request_timeout'] = None search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 5.0) search = searx.search.Search(search_query) @@ -57,7 +58,7 @@ class SearchTestCase(SearxTestCase): self.assertEqual(search.actual_timeout, 3.0) def test_timeout_query_below_default_nomax(self): - searx.search.max_request_timeout = None + settings['outgoing']['max_request_timeout'] = None search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 1.0) search = searx.search.Search(search_query) @@ -65,7 +66,7 @@ class SearchTestCase(SearxTestCase): self.assertEqual(search.actual_timeout, 1.0) def test_timeout_query_below_max(self): - searx.search.max_request_timeout = 10.0 + settings['outgoing']['max_request_timeout'] = 10.0 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 5.0) search = searx.search.Search(search_query) @@ -73,7 +74,7 @@ class SearchTestCase(SearxTestCase): self.assertEqual(search.actual_timeout, 5.0) def test_timeout_query_above_max(self): - searx.search.max_request_timeout = 10.0 + settings['outgoing']['max_request_timeout'] = 10.0 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')], 'en-US', SAFESEARCH, PAGENO, None, 15.0) search = searx.search.Search(search_query) From 495d86b520c71e36c6f924a3a2421e499d159c2b Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 1 Jun 2021 16:03:19 +0200 Subject: [PATCH 3/3] [pylint] searx/__init__.py & searx/settings_defaults.py BTW: normalize some indentations in searx/settings_defaults.py Signed-off-by: Markus Heiser --- searx/__init__.py | 25 ++++++------------------- searx/settings_defaults.py | 26 +++++++++++++++++--------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/searx/__init__.py b/searx/__init__.py index ded0ff2f5..a65266de8 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -1,25 +1,12 @@ -''' -searx is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -searx is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with searx. If not, see < http://www.gnu.org/licenses/ >. - -(C) 2013- by Adam Tauber, -''' +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +# pylint: disable=missing-function-docstring, missing-module-docstring +from os.path import dirname, abspath import logging + import searx.settings_loader from searx.settings_defaults import settings_set_defaults -from os.path import dirname, abspath - searx_dir = abspath(dirname(__file__)) searx_parent_dir = abspath(dirname(dirname(__file__))) @@ -45,7 +32,7 @@ else: logger.info('max_request_timeout=%i second(s)', max_request_timeout) -class _brand_namespace: +class _brand_namespace: # pylint: disable=invalid-name @classmethod def get_val(cls, group, name, default=''): diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index d1a57369d..8350d92ba 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint -# pylint: disable=missing-function-docstring, missing-module-docstring +# pylint: disable=missing-function-docstring +"""Implementation of the default settings. + +""" import typing import numbers @@ -48,16 +51,20 @@ class SettingsValue: type_definition: typing.Union[None, typing.Any, typing.Tuple[typing.Any]]=None, default: typing.Any=None, environ_name: str=None): - self.type_definition = type_definition \ - if type_definition is None or isinstance(type_definition, tuple) \ - else (type_definition,) + self.type_definition = ( + type_definition + if type_definition is None or isinstance(type_definition, tuple) + else (type_definition,) + ) self.default = default self.environ_name = environ_name @property def type_definition_repr(self): - types_str = [t.__name__ if isinstance(t, type) else repr(t) - for t in self.type_definition] + types_str = [ + t.__name__ if isinstance(t, type) else repr(t) + for t in self.type_definition + ] return ', '.join(types_str) def check_type_definition(self, value: typing.Any) -> None: @@ -65,8 +72,9 @@ class SettingsValue: return type_list = tuple(t for t in self.type_definition if isinstance(t, type)) if not isinstance(value, type_list): - raise ValueError('The value has to be one of these types/values: {}'\ - .format(self.type_definition_repr)) + raise ValueError( + 'The value has to be one of these types/values: {}'.format( + self.type_definition_repr)) def __call__(self, value: typing.Any) -> typing.Any: if value == _UNDEFINED: @@ -76,7 +84,7 @@ class SettingsValue: value = os.environ[self.environ_name] if self.type_definition == (bool,): value = STR_TO_BOOL[value.lower()] - # + self.check_type_definition(value) return value