zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

WIP

Browse source 
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2022-08-24 18:57:36 +02:00
parent 744d96a16c
commit a5580c785d
13 changed files with 4555 additions and 4428 deletions
Download patch file Download diff file Expand all files Collapse all files

128
searx/engines/__init__.py
View file

@ -12,19 +12,22 @@ usage::
"""
import sys
import json
import copy
import pathlib
import dataclasses
from typing import Dict, List, Optional
from os.path import realpath, dirname
from babel.localedata import locale_identifiers
from searx import logger, settings
from searx.data import ENGINES_LANGUAGES
from searx.utils import load_module, match_language
from searx.data import data_dir, ENGINES_DATAS
logger = logger.getChild('engines')
ENGINE_DIR = dirname(realpath(__file__))
ENGINE_DIR = pathlib.Path(__file__).parent
BABEL_LANGS = [
lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]
for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())
@ -52,32 +55,58 @@ OTHER_CATEGORY = 'other'
@dataclasses.dataclass
class EngineProperties(dict):
"""
The class is intended to be instanciated for each engine.
"""
class EngineData:
"""The class is intended to be instanciated for each engine."""
regions: Dict[str, str] = dataclasses.field(default_factory=dict)
"""
{
'fr-BE' : <engine's region name>
},
.. code:: python
{
'fr-BE' : <engine's region name>,
}
"""
languages: Dict[str, str] = dataclasses.field(default_factory=dict)
"""
{
'ca' : <engine's language name>
},
.. code:: python
{
'ca' : <engine's language name>,
}
"""
def asdict(self):
return {
'type': 'engine_properties',
'regions': self.regions,
'languages': self.languages,
}
data_type: str = 'engine_data'
"""Data type, default is 'engine_data' for vintage use 'supported_languages'"""
class EngineDataEncoder(json.JSONEncoder):
"""Encodes :class:`EngineData` to a serializable object, see
:class:`json.JSONEncoder`."""
def default(self, o):
"""Return dictionary of a :class:`EngineData` object."""
if isinstance(o, EngineData):
return o.__dict__
return super().default(o)
class EngineDataDict(dict):
"""A python dictionary to map :class:`EngineData` by engine name."""
ENGINE_DATA_FILE = (data_dir / 'engines_datas.json').resolve()
def save_data(self):
with open(self.ENGINE_DATA_FILE, 'w', encoding='utf-8') as f:
json.dump(self, f, indent=2, sort_keys=True, cls=EngineDataEncoder)
@classmethod
def from_data(cls):
"""Instantiate :class:`EngineDataDict` object from :py:obj:`ENGINES_DATAS`"""
obj = EngineDataDict()
for k, v in ENGINES_DATAS.items():
obj[k] = EngineData(**v)
return obj
class Engine: # pylint: disable=too-few-public-methods
@ -96,7 +125,7 @@ class Engine: # pylint: disable=too-few-public-methods
safesearch: bool
time_range_support: bool
timeout: float
properties: EngineProperties
engine_data: EngineData
# Defaults for the namespace of an engine module, see :py:func:`load_engine`
@ -110,18 +139,19 @@ engine_shortcuts = {}
engine_shortcuts[engine.shortcut] = engine.name
:meta hide-value:
"""
def load_engine(engine_data: dict) -> Optional[Engine]:
"""Load engine from ``engine_data``.
def load_engine(engine_setting: dict) -> Optional[Engine]:
"""Load engine from ``engine_setting``.
:param dict engine_data: Attributes from YAML ``settings:engines/<engine>``
:param dict engine_setting: Attributes from YAML ``settings:engines/<engine>``
:return: initialized namespace of the ``<engine>``.
1. create a namespace and load module of the ``<engine>``
2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS`
3. update namespace with values from ``engine_data``
3. update namespace with values from ``engine_setting``
If engine *is active*, return namespace of the engine, otherwise return
``None``.
@ -135,7 +165,7 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
"""
engine_name = engine_data['name']
engine_name = engine_setting['name']
if '_' in engine_name:
logger.error('Engine name contains underscore: "{}"'.format(engine_name))
return None
@ -143,10 +173,10 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
if engine_name.lower() != engine_name:
logger.warn('Engine name is not lowercase: "{}", converting to lowercase'.format(engine_name))
engine_name = engine_name.lower()
engine_data['name'] = engine_name
engine_setting['name'] = engine_name
# load_module
engine_module = engine_data['engine']
engine_module = engine_setting['engine']
try:
engine = load_module(engine_module + '.py', ENGINE_DIR)
except (SyntaxError, KeyboardInterrupt, SystemExit, SystemError, ImportError, RuntimeError):
@ -156,7 +186,7 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
logger.exception('Cannot load engine "{}"'.format(engine_module))
return None
update_engine_attributes(engine, engine_data)
update_engine_attributes(engine, engine_setting)
set_language_attributes(engine)
update_attributes_for_tor(engine)
@ -193,15 +223,15 @@ def set_loggers(engine, engine_name):
module.logger = logger.getChild(module_engine_name)
def update_engine_attributes(engine: Engine, engine_data):
# set engine attributes from engine_data
for param_name, param_value in engine_data.items():
def update_engine_attributes(engine: Engine, engine_setting):
# set engine attributes from engine_setting
for param_name, param_value in engine_setting.items():
if param_name == 'categories':
if isinstance(param_value, str):
param_value = list(map(str.strip, param_value.split(',')))
engine.categories = param_value
elif hasattr(engine, 'about') and param_name == 'about':
engine.about = {**engine.about, **engine_data['about']}
engine.about = {**engine.about, **engine_setting['about']}
else:
setattr(engine, param_name, param_value)
@ -211,33 +241,36 @@ def update_engine_attributes(engine: Engine, engine_data):
setattr(engine, arg_name, copy.deepcopy(arg_value))
def set_language_attributes(engine: Engine):
def set_language_attributes(engine: Engine): # pylint: disable=too-many-branches
# assign supported languages from json file
supported_properties = None
engine_data_dict = EngineDataDict.from_data()
engine_data = None
if engine.name in ENGINES_LANGUAGES:
supported_properties = ENGINES_LANGUAGES[engine.name]
if engine.name in engine_data_dict:
engine_data = engine_data_dict[engine.name]
elif engine.engine in ENGINES_LANGUAGES:
# The key of the dictionary ENGINES_LANGUAGES is the *engine name*
elif engine.engine in engine_data_dict:
# The key of the dictionary engine_data_dict is the *engine name*
# configured in settings.xml. When multiple engines are configured in
# settings.yml to use the same origin engine (python module) these
# additional engines can use the languages from the origin engine.
# For this use the configured ``engine: ...`` from settings.yml
supported_properties = ENGINES_LANGUAGES[engine.engine]
engine_data = engine_data_dict[engine.engine]
if not supported_properties:
if not engine_data:
return
if isinstance(supported_properties, dict) and supported_properties.get('type') == 'engine_properties':
engine.supported_properties = supported_properties
engine.language_support = len(supported_properties['languages']) or len(supported_properties['regions'])
if engine_data.data_type == 'engine_data':
engine.engine_data = engine_data
engine.language_support = len(engine_data.languages) or len(engine_data.regions)
elif engine_data.data_type == 'supported_languages':
# vintage
else:
# depricated: does not work for engines that do support languages
# based on a region.
engine.supported_languages = supported_properties
engine.supported_languages = engine_data.languages
engine.language_support = len(engine.supported_languages) > 0
if hasattr(engine, 'language'):
@ -268,6 +301,9 @@ def set_language_attributes(engine: Engine):
):
engine.language_aliases[iso_lang] = engine_lang
else:
raise TypeError('unknown type of engine data: %s' % engine_data.data_type)
def update_attributes_for_tor(engine: Engine) -> bool:
if using_tor_proxy(engine) and hasattr(engine, 'onion_url'):

2
searx/engines/google.py
View file

@ -136,7 +136,7 @@ def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
:param dict param: request parameters of the engine
:param list lang_list: list of supported languages of the engine
:py:obj:`ENGINES_LANGUAGES[engine-name] <searx.data.ENGINES_LANGUAGES>`
:py:obj:`ENGINES_DATAS[engine-name].languages <searx.data.ENGINES_DATAS>`
:param dict lang_list: custom aliases for non standard language codes
(used when calling :py:func:`searx.utils.match_language`)

12
searx/engines/qwant.py
View file

@ -49,7 +49,7 @@ about = {
# engine dependent config
categories = []
paging = True
supported_properties_url = about['website']
engine_data_url = about['website']
qwant_categ = None # web|news|inages|videos
safesearch = True
@ -95,7 +95,7 @@ def request(query, params):
)
# add quant's locale
q_locale = get_engine_locale(params['language'], supported_properties['regions'], default='en_US')
q_locale = get_engine_locale(params['language'], engine_data.regions, default='en_US')
params['url'] += '&locale=' + q_locale
# add safesearch option
@ -243,7 +243,7 @@ def response(resp):
return results
def _fetch_engine_properties(resp, engine_properties):
def _fetch_engine_data(resp, engine_data):
text = resp.text
text = text[text.find('INITIAL_PROPS') :]
@ -270,7 +270,7 @@ def _fetch_engine_properties(resp, engine_properties):
print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
continue
# note: engine_properties.regions (dict)
# note: engine_data.regions (dict)
#
# dict's key is a string build up from a babel.Locale object / the
# notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
@ -278,6 +278,6 @@ def _fetch_engine_properties(resp, engine_properties):
# the engine.
searxng_locale = locale.language + '-' + locale.territory # --> params['language']
engine_properties.regions[searxng_locale] = q_locale
engine_data.regions[searxng_locale] = q_locale
return engine_properties
return engine_data

10
searx/engines/startpage.py
View file

@ -48,7 +48,7 @@ filter_mapping = {0: '0', 1: '1', 2: '1'}
time_range_support = True
time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
supported_properties_url = 'https://www.startpage.com/do/settings'
engine_data_url = 'https://www.startpage.com/do/settings'
# search-url
base_url = 'https://www.startpage.com/'
@ -249,7 +249,7 @@ def response(resp):
return results
def _fetch_engine_properties(resp, engine_properties):
def _fetch_engine_data(resp, engine_data):
# startpage's language & region selectors are a mess.
#
@ -341,7 +341,7 @@ def _fetch_engine_properties(resp, engine_properties):
region_tag = locale.language + '-' + locale.territory
# print("SearXNG locale tag: %s --> Engine tag: %s" % (region_tag, engine_region_tag))
engine_properties.regions[region_tag] = engine_region_tag
engine_data.regions[region_tag] = engine_region_tag
# languages
@ -385,6 +385,6 @@ def _fetch_engine_properties(resp, engine_properties):
lang_code = catalog_engine2code[name]
# print("SearXNG language tag: %s --> Engine tag: %s" % (lang_code, engine_lang))
engine_properties.languages[lang_code] = engine_lang
engine_data.languages[lang_code] = engine_lang
return engine_properties
return engine_data