Merge pull request #2 from mkmtech7/branding-032123

Branding 032123

.devcontainer/Dockerfile

@@ -0,0 +1,4 @@
+FROM mcr.microsoft.com/devcontainers/base:debian
+
+RUN apt-get update && \
+    apt-get -y install python3 python3-venv redis firefox-esr graphviz imagemagick librsvg2-bin fonts-dejavu shellcheck

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+{
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "features": {
+    "ghcr.io/devcontainers/features/github-cli": {}
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-azuretools.vscode-docker"
+      ],
+      "remote.otherPortsAttributes": {
+        "protocol": "https"
+      },
+      "settings": {
+        "files.autoSave": "off",
+        "python.defaultInterpreterPath": "/workspaces/searxng/local/py3/bin/python3",
+        "python.formatting.blackPath": "/workspaces/searxng/local/py3/bin/black",
+        "python.linting.pylintPath": "/workspaces/searxng/local/py3/bin/pylint"
+      }
+    }
+  },
+  "forwardPorts": [8000, 8888],
+  "portsAttributes": {
+    "8000": {"label": "Sphinx documentation"},
+    "8888": {"label": "SearXNG"}
+  },
+  "postCreateCommand": "git pull && make install"
+}

.github/workflows/data-update.yml

@@ -17,7 +17,7 @@ jobs:
           - update_currencies.py
           - update_external_bangs.py
           - update_firefox_version.py
-          - update_languages.py
+          - update_engine_traits.py
           - update_wikidata_units.py
           - update_engine_descriptions.py
     steps:

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    // See https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "SearXNG",
+            "type": "python",
+            "request": "launch",
+            "module": "searx.webapp",
+            "env": {
+                "FLASK_APP": "webapp",
+                "FLASK_DEBUG": "1",
+                "SEARXNG_DEBUG": "1",
+            },
+            "args": [
+                "run"
+            ],
+            "jinja": true,
+            "justMyCode": true,
+            "python": "${workspaceFolder}/local/py3/bin/python",
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./tests",
+        "-p",
+        "test_*.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true,
+}

.vscode/tasks.json

@@ -0,0 +1,36 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "make run",
+            "type": "shell",
+            "command": "make run",
+            "problemMatcher": [],
+            "isBackground": true,
+            "presentation": {
+                "reveal": "always",
+                "panel": "dedicated"
+            },
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        },
+        {
+            "label": "make docs.live",
+            "type": "shell",
+            "command": "make docs.live",
+            "problemMatcher": [],
+            "isBackground": true,
+            "presentation": {
+                "reveal": "always",
+                "panel": "dedicated"
+            },
+            "group": {
+                "kind": "build"
+            }
+        }
+    ]
+}

Makefile

@@ -76,7 +76,7 @@ test.shell:
 
 MANAGE += buildenv
 MANAGE += weblate.translations.commit weblate.push.translations
-MANAGE += data.all data.languages data.useragents data.osm_keys_tags
+MANAGE += data.all data.traits data.useragents
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += docker.build docker.push docker.buildx
 MANAGE += gecko.driver

README.rst

@@ -145,6 +145,25 @@ Help translate SearXNG at `Weblate`_
    :target: https://translate.codeberg.org/projects/searxng/
 
 
+Codespaces
+==========
+
+You can contribute from your browser using `GitHub Codespaces`_:
+
+- Fork the repository
+- Click on the ``<> Code`` green button
+- Click on the ``Codespaces`` tab instead of ``Local``
+- Click on ``Create codespace on master``
+- VSCode is going to start in the browser
+- Wait for ``git pull && make install`` to appears and then to disapear
+- You have `120 hours per month`_ (see also your `list of existing Codespaces`_)
+- You can start SearXNG using ``make run`` in the terminal or by pressing ``Ctrl+Shift+B``.
+
+.. _GitHub Codespaces: https://docs.github.com/en/codespaces/overview
+.. _120 hours per month: https://github.com/settings/billing
+.. _list of existing Codespaces: https://github.com/codespaces
+
+
 Donations
 =========
 

docs/admin/engines/configured_engines.rst

@@ -42,7 +42,7 @@ Explanation of the :ref:`general engine configuration` shown in the table
         - Timeout
         - Weight
         - Paging
-        - Language
+        - Language, Region
         - Safe search
         - Time range
 

docs/admin/engines/settings.rst

@@ -569,10 +569,13 @@ engine is shown.  Most of the options have a default value or even are optional.
   To disable by default the engine, but not deleting it.  It will allow the user
   to manually activate it in the settings.
 
+``inactive``: optional
+  Remove the engine from the settings (*disabled & removed*).
+
 ``language`` : optional
   If you want to use another language for a specific engine, you can define it
-  by using the full ISO code of language and country, like ``fr_FR``, ``en_US``,
-  ``de_DE``.
+  by using the ISO code of language (and region), like ``fr``, ``en-US``,
+  ``de-DE``.
 
 ``tokens`` : optional
   A list of secret tokens to make this engine *private*, more details see

docs/conf.py

@@ -127,6 +127,10 @@ extensions = [
     'notfound.extension',  # https://github.com/readthedocs/sphinx-notfound-page
 ]
 
+autodoc_default_options = {
+    'member-order': 'groupwise',
+}
+
 myst_enable_extensions = [
   "replacements", "smartquotes"
 ]
@@ -135,6 +139,7 @@ suppress_warnings = ['myst.domains']
 
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3/", None),
+    "babel" : ("https://babel.readthedocs.io/en/latest/", None),
     "flask": ("https://flask.palletsprojects.com/", None),
     "flask_babel": ("https://python-babel.github.io/flask-babel/", None),
     # "werkzeug": ("https://werkzeug.palletsprojects.com/", None),

docs/dev/engine_overview.rst

@@ -54,6 +54,7 @@ Engine File
                                        - ``offline`` :ref:`[ref] <offline engines>`
                                        - ``online_dictionary``
                                        - ``online_currency``
+                                       - ``online_url_search``
    ======================= =========== ========================================================
 
 .. _engine settings:
@@ -131,8 +132,10 @@ Passed Arguments (request)
 These arguments can be used to construct the search query.  Furthermore,
 parameters with default value can be redefined for special purposes.
 
+.. _engine request online:
 
-.. table:: If the ``engine_type`` is ``online``
+.. table:: If the ``engine_type`` is :py:obj:`online
+           <searx.search.processors.online.OnlineProcessor.get_params>`
    :width: 100%
 
    ====================== ============== ========================================================================
@@ -149,12 +152,16 @@ parameters with default value can be redefined for special purposes.
    safesearch             int            ``0``, between ``0`` and ``2`` (normal, moderate, strict)
    time_range             Optional[str]  ``None``, can be ``day``, ``week``, ``month``, ``year``
    pageno                 int            current pagenumber
-   language               str            specific language code like ``'en_US'``, or ``'all'`` if unspecified
+   searxng_locale         str            SearXNG's locale selected by user.  Specific language code like
+                                         ``'en'``, ``'en-US'``, or ``'all'`` if unspecified.
    ====================== ============== ========================================================================
 
 
-.. table:: If the ``engine_type`` is ``online_dictionary``, in addition to the
-           ``online`` arguments:
+.. _engine request online_dictionary:
+
+.. table:: If the ``engine_type`` is :py:obj:`online_dictionary
+           <searx.search.processors.online_dictionary.OnlineDictionaryProcessor.get_params>`,
+           in addition to the :ref:`online <engine request online>` arguments:
    :width: 100%
 
    ====================== ============== ========================================================================
@@ -165,8 +172,11 @@ parameters with default value can be redefined for special purposes.
    query                  str            the text query without the languages
    ====================== ============== ========================================================================
 
-.. table:: If the ``engine_type`` is ``online_currency```, in addition to the
-           ``online`` arguments:
+.. _engine request online_currency:
+
+.. table:: If the ``engine_type`` is :py:obj:`online_currency
+           <searx.search.processors.online_currency.OnlineCurrencyProcessor.get_params>`,
+           in addition to the :ref:`online <engine request online>` arguments:
    :width: 100%
 
    ====================== ============== ========================================================================
@@ -179,6 +189,26 @@ parameters with default value can be redefined for special purposes.
    to_name                str            currency name
    ====================== ============== ========================================================================
 
+.. _engine request online_url_search:
+
+.. table:: If the ``engine_type`` is :py:obj:`online_url_search
+           <searx.search.processors.online_url_search.OnlineUrlSearchProcessor.get_params>`,
+           in addition to the :ref:`online <engine request online>` arguments:
+   :width: 100%
+
+   ====================== ============== ========================================================================
+   argument               type           default-value, information
+   ====================== ============== ========================================================================
+   search_url             dict           URLs from the search query:
+
+                                         .. code:: python
+
+                                            {
+                                              'http': str,
+                                              'ftp': str,
+                                              'data:image': str
+                                            }
+   ====================== ============== ========================================================================
 
 Specify Request
 ---------------

docs/dev/searxng_extra/update.rst

@@ -52,12 +52,12 @@ Scripts to update static data in :origin:`searx/data/`
   :members:
 
 
-``update_languages.py``
-=======================
+``update_engine_traits.py``
+===========================
 
-:origin:`[source] <searxng_extra/update/update_languages.py>`
+:origin:`[source] <searxng_extra/update/update_engine_traits.py>`
 
-.. automodule:: searxng_extra.update.update_languages
+.. automodule:: searxng_extra.update.update_engine_traits
   :members:
 
 

docs/src/searx.engine.archlinux.rst

@@ -0,0 +1,9 @@
+.. _archlinux engine:
+
+==========
+Arch Linux
+==========
+
+.. automodule:: searx.engines.archlinux
+  :members:
+

docs/src/searx.engine.dailymotion.rst

@@ -0,0 +1,8 @@
+.. _dailymotion engine:
+
+===========
+Dailymotion
+===========
+
+.. automodule:: searx.engines.dailymotion
+  :members:

docs/src/searx.engine.duckduckgo.rst

@@ -0,0 +1,22 @@
+.. _duckduckgo engines:
+
+=================
+DukcDukGo engines
+=================
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.duckduckgo
+   :members:
+
+.. automodule:: searx.engines.duckduckgo_images
+   :members:
+
+.. automodule:: searx.engines.duckduckgo_definitions
+   :members:
+
+.. automodule:: searx.engines.duckduckgo_weather
+   :members:

docs/src/searx.enginelib.rst

@@ -0,0 +1,17 @@
+.. _searx.enginelib:
+
+============
+Engine model
+============
+
+.. automodule:: searx.enginelib
+  :members:
+
+.. _searx.enginelib.traits:
+
+=============
+Engine traits
+=============
+
+.. automodule:: searx.enginelib.traits
+  :members:

docs/src/searx.engines.bing.rst

@@ -0,0 +1,43 @@
+.. _bing engines:
+
+============
+Bing Engines
+============
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+
+.. _bing web engine:
+
+Bing WEB
+========
+
+.. automodule:: searx.engines.bing
+  :members:
+
+.. _bing images engine:
+
+Bing Images
+===========
+
+.. automodule:: searx.engines.bing_images
+  :members:
+
+.. _bing videos engine:
+
+Bing Videos
+===========
+
+.. automodule:: searx.engines.bing_videos
+  :members:
+
+.. _bing news engine:
+
+Bing News
+=========
+
+.. automodule:: searx.engines.bing_news
+  :members:

docs/src/searx.engines.google.rst

@@ -12,15 +12,21 @@ Google Engines
 
 .. _google API:
 
-google API
+Google API
 ==========
 
 .. _Query Parameter Definitions:
    https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
 
+SearXNG's implementation of the Google API is mainly done in
+:py:obj:`get_google_info <searx.engines.google.get_google_info>`.
+
 For detailed description of the *REST-full* API see: `Query Parameter
-Definitions`_.  Not all parameters can be appied and some engines are *special*
-(e.g. :ref:`google news engine`).
+Definitions`_.  The linked API documentation can sometimes be helpful during
+reverse engineering.  However, we cannot use it in the freely accessible WEB
+services; not all parameters can be applied and some engines are more *special*
+than other (e.g. :ref:`google news engine`).
+
 
 .. _google web engine:
 
@@ -30,6 +36,13 @@ Google WEB
 .. automodule:: searx.engines.google
   :members:
 
+.. _google autocomplete:
+
+Google Autocomplete
+====================
+
+.. autofunction:: searx.autocomplete.google_complete
+
 .. _google images engine:
 
 Google Images
@@ -53,3 +66,11 @@ Google News
 
 .. automodule:: searx.engines.google_news
   :members:
+
+.. _google scholar engine:
+
+Google Scholar
+==============
+
+.. automodule:: searx.engines.google_scholar
+  :members:

docs/src/searx.engines.peertube.rst

@@ -0,0 +1,27 @@
+.. _peertube engines:
+
+================
+Peertube Engines
+================
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+
+.. _peertube video engine:
+
+Peertube Video
+==============
+
+.. automodule:: searx.engines.peertube
+  :members:
+
+.. _sepiasearch engine:
+
+SepiaSearch
+===========
+
+.. automodule:: searx.engines.sepiasearch
+  :members:

docs/src/searx.engines.rst

@@ -1,8 +1,8 @@
-.. _load_engines:
+.. _searx.engines:
 
-============
-Load Engines
-============
+=================
+SearXNG's engines
+=================
 
 .. automodule:: searx.engines
   :members:

docs/src/searx.engines.startpage.rst

@@ -0,0 +1,13 @@
+.. _startpage engines:
+
+=================
+Startpage engines
+=================
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.startpage
+   :members:

docs/src/searx.engines.wikipedia.rst

@@ -0,0 +1,27 @@
+.. _wikimedia engines:
+
+=========
+Wikimedia
+=========
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+
+.. _wikipedia engine:
+
+Wikipedia
+=========
+
+.. automodule:: searx.engines.wikipedia
+  :members:
+
+.. _wikidata engine:
+
+Wikidata
+=========
+
+.. automodule:: searx.engines.wikidata
+  :members:

docs/src/searx.locales.rst

@@ -4,5 +4,17 @@
 Locales
 =======
 
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
 .. automodule:: searx.locales
   :members:
+
+
+SearXNG's locale codes
+======================
+
+.. automodule:: searx.sxng_locales
+  :members:

docs/src/searx.search.processors.rst

@@ -0,0 +1,47 @@
+.. _searx.search.processors:
+
+=================
+Search processors
+=================
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+
+Abstract processor class
+========================
+
+.. automodule:: searx.search.processors.abstract
+  :members:
+
+Offline processor
+=================
+
+.. automodule:: searx.search.processors.offline
+  :members:
+
+Online processor
+================
+
+.. automodule:: searx.search.processors.online
+  :members:
+
+Online currency processor
+=========================
+
+.. automodule:: searx.search.processors.online_currency
+  :members:
+
+Online Dictionary processor
+===========================
+
+.. automodule:: searx.search.processors.online_dictionary
+  :members:
+
+Online URL search processor
+===========================
+
+.. automodule:: searx.search.processors.online_url_search
+  :members:

manage

@@ -63,7 +63,7 @@ PYLINT_SEARXNG_DISABLE_OPTION="\
 I,C,R,\
 W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
 E1136"
-PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases,logger,categories"
+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="traits,supported_languages,language_aliases,logger,categories"
 PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
 
 help() {
@@ -77,9 +77,9 @@ weblate.:
   push.translations: push translation changes from SearXNG to Weblate's counterpart
   to.translations: Update 'translations' branch with last additions from Weblate.
 data.:
-  all       : update searx/languages.py and ./data/*
-  languages : update searx/data/engines_languages.json & searx/languages.py
-  useragents: update searx/data/useragents.json with the most recent versions of Firefox.
+  all       : update searx/sxng_locales.py and searx/data/*
+  traits    : update searx/data/engine_traits.json & searx/sxng_locales.py
+  useragents: update searx/data/useragents.json with the most recent versions of Firefox
 docs.:
   html      : build HTML documentation
   live      : autobuild HTML documentation while editing
@@ -386,27 +386,33 @@ weblate.push.translations() {
 
 data.all() {
     (   set -e
+
         pyenv.activate
-        data.languages
+        data.traits
         data.useragents
-        data.osm_keys_tags
+
+        build_msg DATA "update searx/data/osm_keys_tags.json"
+        pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
         build_msg DATA "update searx/data/ahmia_blacklist.txt"
         python searxng_extra/update/update_ahmia_blacklist.py
         build_msg DATA "update searx/data/wikidata_units.json"
         python searxng_extra/update/update_wikidata_units.py
         build_msg DATA "update searx/data/currencies.json"
         python searxng_extra/update/update_currencies.py
+        build_msg DATA "update searx/data/external_bangs.json"
+        python searxng_extra/update/update_external_bangs.py
+        build_msg DATA "update searx/data/engine_descriptions.json"
+        python searxng_extra/update/update_engine_descriptions.py
     )
 }
 
 
-data.languages() {
+data.traits() {
     (   set -e
         pyenv.activate
-        build_msg ENGINES "fetch languages .."
-        python searxng_extra/update/update_languages.py
-        build_msg ENGINES "update update searx/languages.py"
-        build_msg DATA "update searx/data/engines_languages.json"
+        build_msg DATA "update searx/data/engine_traits.json"
+        python searxng_extra/update/update_engine_traits.py
+        build_msg ENGINES "update searx/sxng_locales.py"
     )
     dump_return $?
 }
@@ -417,12 +423,6 @@ data.useragents() {
     dump_return $?
 }
 
-data.osm_keys_tags() {
-    build_msg DATA "update searx/data/osm_keys_tags.json"
-    pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
-    dump_return $?
-}
-
 docs.prebuild() {
     build_msg DOCS "build ${DOCS_BUILD}/includes"
     (

requirements-dev.txt

@@ -2,9 +2,9 @@ mock==5.0.1
 nose2[coverage_plugin]==0.12.0
 cov-core==1.15.0
 black==22.12.0
-pylint==2.17.0
+pylint==2.17.1
 splinter==0.19.0
-selenium==4.8.2
+selenium==4.8.3
 twine==4.0.2
 Pallets-Sphinx-Themes==2.0.3
 Sphinx==5.3.0
@@ -15,8 +15,8 @@ sphinxcontrib-programoutput==0.17
 sphinx-autobuild==2021.3.14
 sphinx-notfound-page==0.8.3
 myst-parser==1.0.0
-linuxdoc==20221127
+linuxdoc==20230321
 aiounittest==1.4.2
-yamllint==1.29.0
+yamllint==1.30.0
 wlc==1.13
 coloredlogs==15.0.1

requirements.txt

@@ -1,5 +1,5 @@
 certifi==2022.12.7
-babel==2.11.0
+babel==2.12.1
 flask-babel==3.0.1
 flask==2.2.3
 jinja2==3.1.2
@@ -12,7 +12,7 @@ Brotli==1.0.9
 uvloop==0.17.0
 httpx-socks[asyncio]==0.7.2
 setproctitle==1.3.2
-redis==4.5.1
+redis==4.5.4
 markdown-it-py==2.2.0
 typing_extensions==4.5.0
 fasttext-predict==0.9.2.1

searx/autocomplete.py

@@ -5,20 +5,20 @@
 """
 # pylint: disable=use-dict-literal
 
-from json import loads
+import json
 from urllib.parse import urlencode
 
-from lxml import etree
+import lxml
 from httpx import HTTPError
 
 from searx import settings
-from searx.data import ENGINES_LANGUAGES
+from searx.engines import (
+    engines,
+    google,
+)
 from searx.network import get as http_get
 from searx.exceptions import SearxEngineResponseException
 
-# a fetch_supported_languages() for XPath engines isn't available right now
-# _brave = ENGINES_LANGUAGES['brave'].keys()
-
 
 def get(*args, **kwargs):
     if 'timeout' not in kwargs:
@@ -55,34 +55,58 @@ def dbpedia(query, _lang):
     results = []
 
     if response.ok:
-        dom = etree.fromstring(response.content)
+        dom = lxml.etree.fromstring(response.content)
         results = dom.xpath('//Result/Label//text()')
 
     return results
 
 
-def duckduckgo(query, _lang):
-    # duckduckgo autocompleter
-    url = 'https://ac.duckduckgo.com/ac/?{0}&type=list'
+def duckduckgo(query, sxng_locale):
+    """Autocomplete from DuckDuckGo. Supports DuckDuckGo's languages"""
 
-    resp = loads(get(url.format(urlencode(dict(q=query)))).text)
-    if len(resp) > 1:
-        return resp[1]
-    return []
+    traits = engines['duckduckgo'].traits
+    args = {
+        'q': query,
+        'kl': traits.get_region(sxng_locale, traits.all_locale),
+    }
+
+    url = 'https://duckduckgo.com/ac/?type=list&' + urlencode(args)
+    resp = get(url)
+
+    ret_val = []
+    if resp.ok:
+        j = resp.json()
+        if len(j) > 1:
+            ret_val = j[1]
+    return ret_val
 
 
-def google(query, lang):
-    # google autocompleter
-    autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&'
+def google_complete(query, sxng_locale):
+    """Autocomplete from Google.  Supports Google's languages and subdomains
+    (:py:obj:`searx.engines.google.get_google_info`) by using the async REST
+    API::
 
-    response = get(autocomplete_url + urlencode(dict(hl=lang, q=query)))
+        https://{subdomain}/complete/search?{args}
 
+    """
+
+    google_info = google.get_google_info({'searxng_locale': sxng_locale}, engines['google'].traits)
+
+    url = 'https://{subdomain}/complete/search?{args}'
+    args = urlencode(
+        {
+            'q': query,
+            'client': 'gws-wiz',
+            'hl': google_info['params']['hl'],
+        }
+    )
     results = []
-
-    if response.ok:
-        dom = etree.fromstring(response.text)
-        results = dom.xpath('//suggestion/@data')
-
+    resp = get(url.format(subdomain=google_info['subdomain'], args=args))
+    if resp.ok:
+        json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
+        data = json.loads(json_txt)
+        for item in data[0]:
+            results.append(lxml.html.fromstring(item[0]).text_content())
     return results
 
 
@@ -109,9 +133,9 @@ def seznam(query, _lang):
     ]
 
 
-def startpage(query, lang):
-    # startpage autocompleter
-    lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')
+def startpage(query, sxng_locale):
+    """Autocomplete from Startpage. Supports Startpage's languages"""
+    lui = engines['startpage'].traits.get_language(sxng_locale, 'english')
     url = 'https://startpage.com/suggestions?{query}'
     resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))
     data = resp.json()
@@ -122,20 +146,20 @@ def swisscows(query, _lang):
     # swisscows autocompleter
     url = 'https://swisscows.ch/api/suggest?{query}&itemsCount=5'
 
-    resp = loads(get(url.format(query=urlencode({'query': query}))).text)
+    resp = json.loads(get(url.format(query=urlencode({'query': query}))).text)
     return resp
 
 
-def qwant(query, lang):
-    # qwant autocompleter (additional parameter : lang=en_en&count=xxx )
-    url = 'https://api.qwant.com/api/suggest?{query}'
-
-    resp = get(url.format(query=urlencode({'q': query, 'lang': lang})))
-
+def qwant(query, sxng_locale):
+    """Autocomplete from Qwant. Supports Qwant's regions."""
     results = []
 
+    locale = engines['qwant'].traits.get_region(sxng_locale, 'en_US')
+    url = 'https://api.qwant.com/v3/suggest?{query}'
+    resp = get(url.format(query=urlencode({'q': query, 'locale': locale, 'version': '2'})))
+
     if resp.ok:
-        data = loads(resp.text)
+        data = resp.json()
         if data['status'] == 'success':
             for item in data['data']['items']:
                 results.append(item['value'])
@@ -143,21 +167,38 @@ def qwant(query, lang):
     return results
 
 
-def wikipedia(query, lang):
-    # wikipedia autocompleter
-    url = 'https://' + lang + '.wikipedia.org/w/api.php?action=opensearch&{0}&limit=10&namespace=0&format=json'
+def wikipedia(query, sxng_locale):
+    """Autocomplete from Wikipedia. Supports Wikipedia's languages (aka netloc)."""
+    results = []
+    eng_traits = engines['wikipedia'].traits
+    wiki_lang = eng_traits.get_language(sxng_locale, 'en')
+    wiki_netloc = eng_traits.custom['wiki_netloc'].get(wiki_lang, 'en.wikipedia.org')
 
-    resp = loads(get(url.format(urlencode(dict(search=query)))).text)
-    if len(resp) > 1:
-        return resp[1]
-    return []
+    url = 'https://{wiki_netloc}/w/api.php?{args}'
+    args = urlencode(
+        {
+            'action': 'opensearch',
+            'format': 'json',
+            'formatversion': '2',
+            'search': query,
+            'namespace': '0',
+            'limit': '10',
+        }
+    )
+    resp = get(url.format(args=args, wiki_netloc=wiki_netloc))
+    if resp.ok:
+        data = resp.json()
+        if len(data) > 1:
+            results = data[1]
+
+    return results
 
 
 def yandex(query, _lang):
     # yandex autocompleter
     url = "https://suggest.yandex.com/suggest-ff.cgi?{0}"
 
-    resp = loads(get(url.format(urlencode(dict(part=query)))).text)
+    resp = json.loads(get(url.format(urlencode(dict(part=query)))).text)
     if len(resp) > 1:
         return resp[1]
     return []
@@ -166,7 +207,7 @@ def yandex(query, _lang):
 backends = {
     'dbpedia': dbpedia,
     'duckduckgo': duckduckgo,
-    'google': google,
+    'google': google_complete,
     'seznam': seznam,
     'startpage': startpage,
     'swisscows': swisscows,
@@ -177,12 +218,11 @@ backends = {
 }
 
 
-def search_autocomplete(backend_name, query, lang):
+def search_autocomplete(backend_name, query, sxng_locale):
     backend = backends.get(backend_name)
     if backend is None:
         return []
-
     try:
-        return backend(query, lang)
+        return backend(query, sxng_locale)
     except (HTTPError, SearxEngineResponseException):
         return []

searx/data/__init__.py

@@ -7,7 +7,7 @@
 """
 
 __all__ = [
-    'ENGINES_LANGUAGES',
+    'ENGINE_TRAITS',
     'CURRENCIES',
     'USER_AGENTS',
     'EXTERNAL_URLS',
@@ -42,7 +42,6 @@ def ahmia_blacklist_loader():
         return f.read().split()
 
 
-ENGINES_LANGUAGES = _load('engines_languages.json')
 CURRENCIES = _load('currencies.json')
 USER_AGENTS = _load('useragents.json')
 EXTERNAL_URLS = _load('external_urls.json')
@@ -50,3 +49,4 @@ WIKIDATA_UNITS = _load('wikidata_units.json')
 EXTERNAL_BANGS = _load('external_bangs.json')
 OSM_KEYS_TAGS = _load('osm_keys_tags.json')
 ENGINE_DESCRIPTIONS = _load('engine_descriptions.json')
+ENGINE_TRAITS = _load('engine_traits.json')

searx/data/currencies.json

@@ -21,6 +21,7 @@
         "jungtinių arabų emyratų dirhamas": "AED",
         "യുണൈറ്റഡ് അറബ് എമിരേറ്റ്സ് ദിർഹം": "AED",
         "dirham emiriah arab bersatu": "AED",
+        "diram emirati": "AED",
         "ਸੰਯੁਕਤ ਅਰਬ ਇਮਰਾਤੀ ਦਿਰਹਾਮ": "AED",
         "dirham zjednoczonych emiratów arabskich": "AED",
         "dirrã dos emirados árabes unidos": "AED",
@@ -654,6 +655,7 @@
         "ਬ੍ਰਾਜ਼ੀਲੀ ਰਿਆਲ": "BRL",
         "real brazylijski": "BRL",
         "бразильский реал": "BRL",
+        "brazílsky real": "BRL",
         "பிரேசிலிய ரெயால்": "BRL",
         "เรอัลบราซิล": "BRL",
         "brezilya reali": "BRL",
@@ -894,7 +896,7 @@
         "šveices franks": "CHF",
         "zwitserse frank": "CHF",
         "franc soís": "CHF",
-        "ਸਵਿੱਸ ਫ਼ਰਾਂਕ": "CHF",
+        "ਸਵਿੱਸ ਫ਼ਰੈਂਕ": "CHF",
         "frank szwajcarski": "CHF",
         "franco suíço": "CHF",
         "franc elvețian": "CHF",
@@ -2232,7 +2234,7 @@
         "kaaimaneilandse dollar": "KYD",
         "ਕੇਮਨ ਟਾਪੂ ਡਾਲਰ": "KYD",
         "dolar kajmański": "KYD",
-        "dólar das ilhas cayman": "KYD",
+        "dólar das ilhas caimã": "KYD",
         "доллар каймановых островов": "KYD",
         "долар кајманских острва": "KYD",
         "caymansk dollar": "KYD",
@@ -2315,6 +2317,7 @@
         "ліванський фунт": "LBP",
         "روبية سريلانكية": "LKR",
         "шриланкийска рупия": "LKR",
+        "শ্রীলঙ্কান রুপি": "LKR",
         "rupia de sri lanka": "LKR",
         "srílanská rupie": "LKR",
         "srilankanske rupee": "LKR",
@@ -4450,6 +4453,7 @@
         "zelts kā investīcija": "XAU",
         "സ്വർണവും സാമ്പത്തിക ശാസ്ത്രവും": "XAU",
         "emas sebagai pelaburan": "XAU",
+        "investiciono zlato": "XAU",
         "investeringsguld": "XAU",
         "kênh đầu tư vàng": "XAU",
         "دولار شرق الكاريبي": "XCD",
@@ -5040,6 +5044,7 @@
         "அரூபா ஃபுளோரின்": "AWG",
         "арубський флорін": "AWG",
         "₼": "AZN",
+        "A.M.": "AZN",
         "مانات": "AZN",
         "manat d'azerbaidjan": "AZN",
         "manat de l'azerbaidjan": "AZN",
@@ -5078,7 +5083,6 @@
         "அசர்பைச்சானிய மனாத்து": "AZN",
         "azerbaycan yeni manatı": "AZN",
         "yeni manat": "AZN",
-        "A.M.": "AZN",
         "KM": "BAM",
         "bosenská konvertibilní marka": "BAM",
         "mark cyfnewidiol": "BAM",
@@ -5579,6 +5583,7 @@
         "リヒテンシュタインの通貨": "CHF",
         "스위스프랑": "CHF",
         "zwitserse franc": "CHF",
+        "ਸਵਿੱਸ ਫ਼ਰਾਂਕ": "CHF",
         "franco da suíça": "CHF",
         "franco suiço": "CHF",
         "francos suíços": "CHF",
@@ -6083,7 +6088,11 @@
         "engelske pund": "GBP",
         "britisches pfund": "GBP",
         "british pound": "GBP",
+        "great british pound": "GBP",
+        "pence": "GBP",
+        "penny": "GBP",
         "pound": "GBP",
+        "pounds": "GBP",
         "quid": "GBP",
         "britaj pundoj": "GBP",
         "sterlingo": "GBP",
@@ -6142,7 +6151,6 @@
         "libra estrelina": "GBP",
         "libra inglesa": "GBP",
         "lire sterline": "GBP",
-        "penny": "GBP",
         "ukl": "GBP",
         "английский фунт": "GBP",
         "английский фунт стерлингов": "GBP",
@@ -6845,6 +6853,7 @@
         "kaimanų doleris": "KYD",
         "caymaneilandse dollar": "KYD",
         "dolar de las illas caiman": "KYD",
+        "dólar das ilhas cayman": "KYD",
         "доллар островов кайман": "KYD",
         "кајмански долар": "KYD",
         "₸": "KZT",
@@ -6880,7 +6889,10 @@
         "kip laosià": "LAK",
         "lak": "LAK",
         "₭n": "LAK",
+        "kīp": "LAK",
         "kip laotien": "LAK",
+        "quip": "LAK",
+        "quipe": "LAK",
         "laoški kip": "LAK",
         "kip laos": "LAK",
         "ラオスの通貨": "LAK",
@@ -7954,6 +7966,7 @@
         "qar": "QAR",
         "ر.ق": "QAR",
         "لو روماني": "RON",
+        "lej": "RON",
         "لي روماني": "RON",
         "румънска леа": "RON",
         "румънски леи": "RON",
@@ -8344,6 +8357,7 @@
         "saotomska dobra": "STN",
         "dobra di sao tomé e principe": "STN",
         "サントメ・プリンシペ・ドブラ": "STN",
+        "dobra saotomejska": "STN",
         "stn": "STN",
         "валюта сан томе и принсипи": "STN",
         "добра": "STN",
@@ -8786,7 +8800,6 @@
         "우간다실링": "UGX",
         "ugandese shilling": "UGX",
         "валюта уганды": "UGX",
-        "щ.д.": "USD",
         "usd": "USD",
         "us $": "USD",
         "american dollar": "USD",
@@ -8829,7 +8842,6 @@
         "đôla mỹ": "USD",
         "đồng bạc mĩ": "USD",
         "đồng bạc mỹ": "USD",
-        "US$": "USD",
         "us dollar [next day]": "USN",
         "uruguay peso en unidades indexadas": "UYI",
         "ui": "UYI",
@@ -9283,6 +9295,7 @@
             "ml": "യുണൈറ്റഡ് അറബ് എമിരേറ്റ്സ് ദിർഹം",
             "ms": "Dirham Emiriah Arab Bersatu",
             "nl": "VAE-Dirham",
+            "oc": "Diram emirati",
             "pa": "ਸੰਯੁਕਤ ਅਰਬ ਇਮਰਾਤੀ ਦਿਰਹਾਮ",
             "pl": "Dirham",
             "pt": "Dirham dos Emirados Árabes Unidos",
@@ -9441,6 +9454,7 @@
             "lt": "Dramas",
             "lv": "Armēnijas drams",
             "nl": "Armeense dram",
+            "oc": "dram",
             "pa": "ਅਰਮੀਨੀਆਈ ਦਰਾਮ",
             "pl": "Dram",
             "pt": "dram arménio",
@@ -9453,8 +9467,7 @@
             "ta": "ஆர்மேனிய டிராம்",
             "tr": "Ermeni dramı",
             "uk": "вірменський драм",
-            "cy": "Dram Armenia",
-            "oc": "dram"
+            "cy": "Dram Armenia"
         },
         "ANG": {
             "ar": "غيلدر الأنتيل الهولندية",
@@ -10035,6 +10048,7 @@
             "pt": "real",
             "ro": "Real",
             "ru": "бразильский реал",
+            "sk": "Brazílsky real",
             "sr": "бразилски реал",
             "sv": "Real",
             "ta": "பிரசிலியன் ரியால்",
@@ -10999,7 +11013,7 @@
             "uk": "Фолклендський фунт"
         },
         "GBP": {
-            "af": "Pond sterling",
+            "af": "pond sterling",
             "ar": "جنيه إسترليني",
             "bg": "британска лира",
             "bn": "পাউন্ড স্টার্লিং",
@@ -11012,10 +11026,10 @@
             "eo": "brita pundo",
             "es": "libra esterlina",
             "et": "Suurbritannia naelsterling",
-            "eu": "Libera esterlina",
+            "eu": "libera esterlina",
             "fi": "Englannin punta",
             "fr": "livre sterling",
-            "gl": "Libra esterlina",
+            "gl": "libra esterlina",
             "he": "לירה שטרלינג",
             "hr": "Britanska funta",
             "hu": "font sterling",
@@ -11027,14 +11041,14 @@
             "lv": "sterliņu mārciņa",
             "ms": "paun sterling",
             "nl": "pond sterling",
-            "oc": "Liure esterlina",
+            "oc": "liure esterlina",
             "pa": "ਪਾਊਂਡ ਸਟਰਲਿੰਗ",
             "pl": "funt szterling",
             "pt": "libra esterlina",
             "ro": "liră sterlină",
             "ru": "фунт стерлингов",
-            "sk": "Libra šterlingov",
-            "sl": "Funt šterling",
+            "sk": "libra šterlingov",
+            "sl": "funt šterling",
             "sr": "британска фунта",
             "sv": "Brittiskt pund",
             "ta": "பிரித்தானிய பவுண்டு",
@@ -12168,6 +12182,7 @@
         "LKR": {
             "ar": "روبية سريلانكية",
             "bg": "Шриланкийска рупия",
+            "bn": "শ্রীলঙ্কান রুপি",
             "ca": "rupia de Sri Lanka",
             "cs": "Srílanská rupie",
             "da": "Sri Lanka rupee",
@@ -13691,7 +13706,7 @@
             "ar": "كرونة سويدية",
             "bg": "Шведска крона",
             "ca": "corona sueca",
-            "cs": "Švédská koruna",
+            "cs": "švédská koruna",
             "da": "svensk krone",
             "de": "schwedische Krone",
             "en": "Swedish krona",
@@ -13797,7 +13812,7 @@
         },
         "SLE": {
             "ar": "ليون سيراليوني",
-            "bg": "Леоне на Сиера Леоне",
+            "bg": "леоне на Сиера Леоне",
             "ca": "leone",
             "cs": "sierraleonský leone",
             "de": "Sierra-leonischer Leone",
@@ -13806,30 +13821,30 @@
             "es": "leone",
             "fi": "Sierra Leonen leone",
             "fr": "leone",
-            "gl": "Leone",
+            "gl": "leone",
             "he": "ליאון",
             "hr": "Sijeraleonski leone",
             "hu": "Sierra Leone-i leone",
-            "id": "Leone",
+            "id": "leone",
             "it": "leone sierraleonese",
-            "ja": "レオン (通貨)",
+            "ja": "レオン",
             "ko": "시에라리온 레온",
-            "lt": "Leonė",
-            "ms": "Leone",
+            "lt": "leonė",
+            "ms": "leone",
             "nl": "Sierra Leoonse leone",
-            "pl": "Leone",
+            "pl": "leone",
             "pt": "leone",
-            "ro": "Leone",
+            "ro": "leone",
             "ru": "леоне",
             "sr": "сијералеонски леоне",
             "sv": "Sierraleonsk Leone",
             "tr": "Sierra Leone leonesi",
-            "uk": "Леоне",
-            "oc": "Leone"
+            "uk": "леоне",
+            "oc": "leone"
         },
         "SLL": {
             "ar": "ليون سيراليوني",
-            "bg": "Леоне на Сиера Леоне",
+            "bg": "леоне на Сиера Леоне",
             "ca": "leone",
             "cs": "sierraleonský leone",
             "de": "Sierra-leonischer Leone",
@@ -13838,26 +13853,26 @@
             "es": "leone",
             "fi": "Sierra Leonen leone",
             "fr": "leone",
-            "gl": "Leone",
+            "gl": "leone",
             "he": "ליאון",
             "hr": "Sijeraleonski leone",
             "hu": "Sierra Leone-i leone",
-            "id": "Leone",
+            "id": "leone",
             "it": "leone sierraleonese",
-            "ja": "レオン (通貨)",
+            "ja": "レオン",
             "ko": "시에라리온 레온",
-            "lt": "Leonė",
-            "ms": "Leone",
+            "lt": "leonė",
+            "ms": "leone",
             "nl": "Sierra Leoonse leone",
-            "pl": "Leone",
+            "pl": "leone",
             "pt": "leone",
-            "ro": "Leone",
+            "ro": "leone",
             "ru": "леоне",
             "sr": "сијералеонски леоне",
             "sv": "Sierraleonsk Leone",
             "tr": "Sierra Leone leonesi",
-            "uk": "Леоне",
-            "oc": "Leone"
+            "uk": "леоне",
+            "oc": "leone"
         },
         "SOS": {
             "ar": "شلن صومالي",
@@ -14738,6 +14753,7 @@
             "lv": "Zelts kā investīcija",
             "ml": "സ്വർണവും സാമ്പത്തിക ശാസ്ത്രവും",
             "ms": "Emas sebagai pelaburan",
+            "sr": "Investiciono zlato",
             "sv": "Investeringsguld",
             "vi": "Kênh đầu tư vàng"
         },

searx/data/engine_descriptions.json

@@ -71,7 +71,7 @@
   "arxiv":"arXiv is een verzameling van elektronische vooruitgaven van wetenschappelijke artikelen in de wiskunde, natuurkunde, sterrenkunde, informatica, mathematische biologie, statistiek en mathematische economie, die online geraadpleegd kunnen worden. In bepaalde deelgebieden van de wis- en natuurkunde kunnen bijna alle wetenschappelijke artikelen op arXiv gevonden worden. arXiv is opgericht op 14 augustus 1991 en passeerde de grens van een half miljoen artikelen op 3 oktober 2008. In 2012 werden er meer dan 7000 artikelen per maand op arXiv geplaatst.",
   "bandcamp":"Bandcamp is een Amerikaanse onderneming gestart in 2007 door Ethan Diamond en Shawn Grunberger, samen met programmeurs Joe Holt en Neal Tucker. In 2008 heeft het bedrijf een online muziekwinkel geopend en een platform voor artiestpromotie, gericht op onafhankelijke artiesten. In 2013 kwam daar een app bij.",
   "wikipedia":"Wikipedia is een meertalige internetencyclopedie, die door vrijwillige auteurs wordt geschreven. Wikipedia wordt gepubliceerd onder een vrije licentie, waardoor de inhoud elders gratis mits met bronvermelding opnieuw te gebruiken is. De website is eigendom van de Amerikaanse Wikimedia Foundation. Het is het oudste en bekendste project van deze organisatie.",
-  "bing":"Bing, codenaam Kumo, is een zoekmachine van Microsoft, die op 3 juni 2009 werd gelanceerd. Het is de op een na grootste zoekmachine op het internet na Google. Bing heeft elk jaar een groeiend aantal gebruikers. Bing heeft in 2019 een wereldwijd marktaandeel van ongeveer 2,5%. Het programma is vooral populair in de Verenigde Staten en in India. Op 15 november 2011 werd de bètatag ervan verwijderd.",
+  "bing":"Bing, codenaam Kumo, is een zoekmachine van Microsoft, die op 3 juni 2009 werd gelanceerd. Het is de op een na grootste zoekmachine op het internet na Google. Bing heeft in 2019 een wereldwijd marktaandeel van ongeveer 2,8%, en haalde in februari 2023 gemiddeld 100 miljoen actieve gebruikers per dag. Het programma is vooral populair in de Verenigde Staten en in India. Op 15 november 2011 werd de bètatag ervan verwijderd.",
   "bing images":[
    "bing:nl-BE",
    "ref"
@@ -103,7 +103,11 @@
   ],
   "apple maps":"Apple Kaarten is een online kaartendienst van het Amerikaanse bedrijf Apple. Het is de standaardkaartendienst van de besturingssystemen iOS, macOS en watchOS en beschikt onder andere over stapsgewijze navigatie, navigatie met het openbaar vervoer en verkeersinformatie. Ook zijn op een aantal locaties — vooral grote steden — zogenaamde \"Flyovers\" beschikbaar, waarbij die locatie vanuit verschillende perspectieven fotorealistisch wordt getoond.",
   "emojipedia":"Emojipedia is een online naslagwerk voor emoji dat de betekenis en het gemeenschappelijk gebruik van emoji-tekens in de Unicode-Standaard documenteert. De site werd in 2013 opgestart door de Australiër Jeremy Burge.",
-  "tineye":"TinEye is een zoekmachine voor afbeeldingen. TinEye is eigendom van het in Canada gesitueerde bedrijf Idée, Inc. Met de webapplicatie TinEye kunnen gebruikers naar afbeeldingen zoeken. Afbeeldingen kunnen geüpload worden op de website van TinEye of door een URL in te voeren naar een bestaande afbeelding.",
+  "tineye":"TinEye is een zoekmachine voor afbeeldingen. TinEye is eigendom van het Canadese bedrijf Idée, Inc.. Met de webapplicatie TinEye kunnen gebruikers naar afbeeldingen zoeken. Afbeeldingen kunnen geüpload worden op de website van TinEye of door een URL in te voeren naar een bestaande afbeelding.",
+  "etymonline":[
+   "Online Engels etymologisch woordenboek",
+   "wikidata"
+  ],
   "flickr":"Flickr is een website voor het delen van foto's en videofragmenten met een internetgemeenschap. Net als Delicious wordt het gezien als een Web 2.0-applicatie die tagging (trefwoorden) gebruikt om een niet-hiërarchische classificering mogelijk te maken (folksonomie).",
   "free software directory":"De Free Software Directory is een website opgericht door de Free Software Foundation (FSF) en de UNESCO. Het biedt een overzicht van vrije software, met name software voor vrije besturingssystemen, zoals Linux en BSD. Voordat besloten wordt tot de opname van een programma in de Free Software Directory, wordt nagegaan of de aangeduide licentie wel degelijk de juiste licentie is.",
   "genius":"Genius is een online kennisbank. De website biedt gebruikers de mogelijkheid om liedteksten, nieuwsberichten, belangrijke documenten, poëzie, en andere vormen van tekst van toelichtingen te voorzien.",
@@ -166,12 +170,12 @@
   ]
  },
  "zh-HK":{
-  "archive is":"archive.today又称archive.is，是一個私人資助的网页存档網站，資料中心位於歐洲法國的北部-加来海峡。這個網站典藏檔案館使用Apache Hadoop與Apache Accumulo軟體。它可以一次取回一個類似於WebCite的小於50MB的頁面，並能收錄Google地圖與Twitter。",
+  "archive is":"archive.today，又称archive.is或archive.ph，是一個私人資助的网页存档網站，資料中心位於歐洲法國的北部-加来海峡。這個網站典藏檔案館使用Apache Hadoop與Apache Accumulo軟體。它可以一次取回一個類似於WebCite的小於50MB的頁面，並能收錄Google地圖與Twitter。",
   "artic":"芝加哥藝術博物館（英語：），是一座位於美國伊利諾州芝加哥的美術館，於1879年成立，是世界上最古老、規模最大的藝術博物館之一。該博物館因其策展與展示大量藝術家的作品而受到歡迎，每年共有約150萬人參觀。該博物館的收藏由11個策展部門管理，並保存了喬治·秀拉的《大碗岛的星期天下午》、巴勃羅·畢卡索的《老吉他手》 、愛德華·霍普的《夜遊者》和格兰特·伍德的《美国哥特式》等名作，博物館永久收藏近300,000件藝術品，每年舉辦30多個特展。",
   "arxiv":"arXiv 是一個收集物理學、數學、計算機科學、生物學與數理經濟學的論文預印本的網站，成立于1991年8月14日。截至2008年10月，arXiv.org已收集超過50萬篇預印本；至2014年底，藏量達到1百萬篇。截至2016年10月，每月提交量超過10,000篇。",
   "bandcamp":"Bandcamp是一家美国線上音乐公司， 由前Oddpost联合创始人Ethan Diamond与程序员Shawn Grunberger、Joe Holt和Neal Tucker于2008年创立，总部位于加利福尼亚。",
   "wikipedia":"維基百科 是维基媒体基金会运营的一个多语言的線上百科全書，并以创建和维护作为开放式协同合作项目，特点是自由內容、自由编辑、自由版权。目前是全球網絡上最大且最受大眾歡迎的参考工具书，名列全球二十大最受歡迎的網站，其在搜尋引擎中排名亦較為靠前。維基百科目前由非營利組織維基媒體基金會負責營運。Wikipedia是混成詞，分别取自於網站核心技術「Wiki」以及英文中百科全書之意的「encyclopedia」。截至2021年初，所有語種的維基百科條目數量達5,500萬。",
-  "bing":"是一款由微软公司推出的網路搜尋引擎。Bing的历史可追溯至于1998年的第三个季度发布的MSN Search，它的由Looksmart和Inktomi等提供；2006年3月8日，微软发布了Windows Live Search的公测版，并于同年9月11日让其取代了MSN Search，该引擎开始使用搜尋選項卡；次年3月，微软将其与Windows Live分开并更名Live Search，在此期间，其子服务曾经多次重组、关闭。到了2009年6月3日，微软将Live Search改造成了今天的Bing并正式发布。微软认为一词简单明了、易于拼写，容易被人记住；而中文品牌名稱「必應」源自成语「有求必应」。微软声称，此款搜尋引擎将以全新的姿态面世並带来革命。Bing的内测代号为Kumo（「蜘蛛」的日文），其后才被命名为。2020年10月5日，Bing更名為Microsoft Bing。",
+  "bing":"是一款由微软公司推出的網路搜尋引擎。该服务起源于微软以前的搜索引擎：MSN Search，Windows Live Search和后来的Live Search。Bing提供各种搜索服务，包括Web、视频、图像、学术、词典、新闻、地图、旅游等搜索产品，以及翻译和人工智能产品Bing Chat。",
   "bing images":[
    "bing:zh-HK",
    "ref"
@@ -233,6 +237,10 @@
    "bing:af",
    "ref"
   ],
+  "bing videos":[
+   "Intelligente soektog van Bing maak dit makliker om vinnig te kry waarna jy soek en beloon jou.",
+   "https://www.bing.com/videos"
+  ],
   "currency":"DuckDuckGo is ’n soekenjin op die Internet. Een van hulle grootste trekpleisters is dat hulle meer privaatheidsbewus is en nie op gebruikers spioeneer nie. Die gebruiker se gedrag en profiel beïnvloed dus nie die resultate nie. As gevolg hiervan sal elke gebruiker presies dieselfde resultate vir dieselfde soektog kry.",
   "ddg definitions":[
    "currency:af",
@@ -283,7 +291,7 @@
   "artic":"معهد الفن في شيكاغو متحف عام للفن، ومركز ثقافي وتعليمي في الولايات المتحدة. تأسس معهد الفن عام 1866 باسم أكاديمية شيكاغو للتصميم، تضم المجموعات المعروضة في المعهد اللوحات والمنحوتات، والمطبوعات والرسومات والفنون الزخرفية الأوروبية والأمريكية والفن الشرقي والكلاسيكي وفن التصوير الضوئي والمنسوجات وفنون وحرف أفريقيا وأمريكا اللاتينية وجزر المحيط الهادئ وأمريكا ما قبل كولمبوس.",
   "arxiv":"أرخايف ‏ بحيث تُنطق في النهاية «أركايف» أو «أرخايف». أرخايف هو أرشيف لمسودات أوراق علمية إلكترونية مكتوبة في مجالات الفيزياء، الرياضيات، الفلك، علم الحاسوب، والإحصاء التي يمكن الوصول إليها عبر الإنترنت. هذه الأرشيفات موجودة على موقع arXiv.org.",
   "wikipedia":"ويكيبيديا والكلمة مشتقة من مقطعين: ويكي wiki وتعني بلغة هاواي \"بالغ السرعة\"، والثاني بيديا pedia ومشتق من كلمة موسوعة encyclopedia، ويكيبيديا هي موسوعة متعددة اللغات، مبنية على الويب، ذات محتوى حر، تشغلها مؤسسة ويكيميديا، التي هي منظمة غير ربحية. ويكيبيديا هي موسوعة يمكن لأي مستخدم تعديل وتحرير وإنشاء مقالات جديدة فيها.",
-  "bing":"مايكروسوفت بينغ ‏ واختصارًا وهو الاسم السابق: بينغ ‏ ؛ هو محرك بحث (أعلن عنه تحت اسم ، طور تحت اسم بالعربية: كومو وإنجليزية: kumo, هو محرك بحث في ويب لشركة مايكروسوفت، صمم لمنافسة رواد هذا المجال جوجل وياهو!، تم الإفصاح عنه من قبل ستيف بالمر الرئيس التنفيذي السابق في مايكروسوفت في 28 مايو 2009 في مؤتمر All Things D في سان دييغو، بينغ هو بديل لايف سيرش Live Search، بدأ عمله بالكامل في 3 يونيو 2009.",
+  "bing":"مايكروسوفت بينغ ‏ واختصارًا وهو الاسم السابق: بينغ ‏ ؛ هو محرك بحث أعلن عنه تحت اسم ، طور تحت اسم بالعربية: كومو وإنجليزية: kumo, هو محرك بحث في ويب لشركة مايكروسوفت، صمم لمنافسة رواد هذا المجال جوجل وياهو!، تم الإفصاح عنه من قبل ستيف بالمر الرئيس التنفيذي السابق في مايكروسوفت في 28 مايو 2009 في مؤتمر All Things D في سان دييغو، بينغ هو بديل لايف سيرش Live Search، بدأ عمله بالكامل في 3 يونيو 2009.",
   "bing images":[
    "bing:ar",
    "ref"
@@ -340,7 +348,7 @@
   "google news":"أخبار جوجل ‏ هو برنامج مبني على الويب، تقدمه شركة جوجل. البرنامج عبارة عن مجمع أخبار. يستخدم البرنامج خوارزمية Storyrank التي يمكن القول أنها الخوارزمية الشقيقة لخوارزمية Pagerank التي جعلت باحوث جوجل من أفضل محركات البحث على الإنترنت. تم تطوير الخوارزمية بواسطة كاريشنا باهارات في العام 2001. تم تقديم أخبار جوجل كنسخة بيتا في أبريل 2002. خرج البرنامج من مرحلة البيتا في 23 يناير 2006. هناك عدة نسخ من البرنامج مخصصة ل 20 إقليما في العالم تستخدم 12 لغة. اللغات المتاحة حالياً هي: العربية، العبرية، الصينية ، اليبانية، الكورية، الفرنسية، الألمانية، الإسبانية، الإنجليزية، الإيطالية، البرتغالية، الهولندية، النرويجية، والسويدية.",
   "google videos":"جوجل فيديو هو موقع فيديو مجاني، وكذلك محرك بحث للفيديو من غوغل. غوغل فيديو يسمح للفيديو أن يكون جزءًا لا يتجزأ من مواقع أخرى بعيدة ويوفر ما يلزم من كود HTML ويسمح كذلك للمواقع باستضافة كميات كبيرة من الفيديو عن بعد على جوجل فيديو دون الوقوع في عرض النطاق الترددي. بعض أشرطة الفيديو معروضة أيضاً للبيع من خلال متجر غوغل فيديو.",
   "google scholar":"جوجل سكولار أو الباحث العلمي الخاص بجوجل ‏ هو محرك بحث خاص بالمؤلفات العلمية والأكاديمية التي يحتاج إليها الباحثون والدارسون. من مكان واحد، يمكنك البحث عبر العديد من المجالات العلمية ومصادر المعلومات: أبحاث معتمدة ورسائل علمية وكتب وملخصات ومقالات من ناشرين أكاديميين وجمعيات متخصصة ومراكز جمع المعلومات قبل طباعتها والجامعات وغير ذلك من مؤسسات البحث العلمي. يساعدك الباحث العلمي من جوجل على التعرف على أكثر الأبحاث العلمية صلة بمجال بحثك في عالم البحث العلمي.",
-  "google play apps":"جوجل بلاي ‏ وسابقاً سوق أندرويد ‏ هي خدمة توزيع رقمية يتم تشغيلها وتطويرها بواسطة جوجل. وهو بمثابة متجر التطبيقات الرسميّ للأجهزة المُعتمدة التي تعمل على نظام التشغيل أندرويد، ممَّا يسمح للمستخدمين بتصفح وتنزيل التطبيقات التي تمَّ تطويرها بِاستخدام مجموعة تطوير برامج أندرويد (SDK) ونشرها عبر جوجل. يُعد جوجل بلاي أيضًا بمثابة متجر وسائط رقمية، حيثُ يعرضُ المُوسيقى والكتب والأفلام والبرامج التلفزيونية. عرض المتجر سابقًا أجهزة جوجل للشراء حتّى طُرح متجر تجزئة منفصل للأجهزة عبر الإنترنت والمسمى متجر جوجل وذلك في 11 مارس 2015. عرض المتجر أيضًا منشورات إخباريَّة ومجلات قبل تجديد أخبار جوجل في 15 مايو 2018. عرض المتجر أيضاً المُوسيقى والبودكاست كجزء من موسيقى جوجل بلاي حتّى ديسمبر 2020 عندما تمَّ استبدال الخدمة بيوتيوب ميوزيك وجوجل بودكاست.",
+  "google play apps":"جوجل بلاي ‏ وسابقًا سوق أندرويد ‏ هي خدمة توزيع رقمية يتم تشغيلها وتطويرها بواسطة جوجل، وهو بمثابة متجر التطبيقات الرسميّ للأجهزة المُعتمدة التي تعمل على نظام التشغيل أندرويد، ممَّا يسمح للمستخدمين بتصفح وتنزيل التطبيقات التي تمَّ تطويرها بِاستخدام مجموعة تطوير برامج أندرويد (SDK) ونشرها عبر جوجل. يُعد جوجل بلاي أيضًا بمثابة متجر وسائط رقمية، حيثُ يعرضُ المُوسيقى، والكتب، والأفلام، والبرامج التلفزيونية. عرض المتجر سابقًا أجهزة جوجل للشراء حتّى طُرح متجر تجزئة منفصل للأجهزة عبر الإنترنت والمسمى متجر جوجل وذلك في 11 مارس 2015. عرض المتجر أيضًا منشورات إخباريَّة ومجلات قبل تجديد أخبار جوجل في 15 مايو 2018. عرض المتجر أيضاً المُوسيقى والبودكاست كجزء من موسيقى جوجل بلاي حتّى ديسمبر 2020 عندما تمَّ استبدال الخدمة بيوتيوب ميوزيك وجوجل بودكاست.",
   "google play movies":[
    "google play apps:ar",
    "ref"
@@ -386,7 +394,7 @@
   "naver":"نافير هي منصة كورية جنوبية على الإنترنت تديرها شركة نافير. ظهرت لأول مرة في عام 1999 كأول بوابة ويب في كوريا الجنوبية لتطوير واستخدام محرك البحث الخاص بها. كما كانت أول مشغل في العالم يقدم ميزة البحث الشامل، والتي تجمع نتائج البحث من مختلف الفئات وتقدمها في صفحة واحدة. أضافت نافير منذ ذلك الحين العديد من الخدمات الجديدة التي تتراوح من الميزات الأساسية مثل البريد الإلكتروني والأخبار إلى أول منصة للأسئلة والأجوبة عبر الإنترنت في العالم يعرف ب«نولج ان».",
   "rumble":"رامبل ‏ هو موقع صناعة محتويات فيديو كندي. تم تأسيسه في سنة 2013 من قبل كريس بافلوفسكي. بدأ كموقع لنشر فيديوهات عن الحيوانات الأليفة والأطفال الرضع وثم زادت شهرته ليكون منصة بديلة للمحافظين واليمينيين عن يوتيوب. في يوم 11 يناير 2021 رفع الموقع دعوى ضد غوغل بتهمة التلاعب بنتائج البحث عن الموقع واخفائه وطالب بتعويض ملياري دولار على ذلك. يدير الموقع حالياً دان بونجينو.",
   "wttr.in":[
-   "Des Moines, Iowa, United States تقرير حالة ألطقس",
+   "not found تقرير حالة ألطقس",
    "https://wttr.in"
   ]
  },
@@ -456,6 +464,19 @@
   "library of congress":"Библиотеката на Конгреса е фактически национална библиотека на Съединените щати. Тя е най-старото федерално културно учреждение в САЩ.",
   "openstreetmap":"„Оупън Стрийт Мап“, съкратено ОСМ, е сътруднически проект за създаване на свободна и редактируема карта на света.",
   "piratebay":"The Pirate Bay е известен шведски уеб сайт, най-големият BitTorrent тракер в света, позволяващ надежден трансфер на големи файлове, с повече от 22 млн. потребители. Той също служи като индекс за торент файлове.",
+  "qwant":"Qwant е интернет търсачка, която акцентира върху поверителността.",
+  "qwant news":[
+   "qwant:bg",
+   "ref"
+  ],
+  "qwant images":[
+   "qwant:bg",
+   "ref"
+  ],
+  "qwant videos":[
+   "qwant:bg",
+   "ref"
+  ],
   "reddit":"Reddit е социален агрегатор на новини (SNA) и социално-медийна онлайн платформа, изградена от отделни общности, наречени „subreddits“, чиито потребители могат да споделят снимки, изображения, видео, линкове и текстове по определена тема, да участват в дискусии и да гласуват за публикуваното съдържание. Насочеността и правилата на поведение във всяка общност в Reddit се определят от нейните потребители, а някои от тях участват и в прилагането им в качеството на модератори на съдържанието. Reddit поддържа прогресивно уеб приложение (PWA) и има мобилни версии за iOS и Android.",
   "soundcloud":"„Саундклауд“ е онлайн платформа за разпространение на аудио и музика, базирана в Берлин, Германия, която позволява на потребителите да качват и споделят звукови файлове.",
   "stackoverflow":"Stack Exchange е мрежа от сайтове за въпроси и отговори по различни тематики в различни сфери, като всеки сайт покрива определена тематика. В тези сайтове потребителите се ползват с ограничение/възнаграждаване на техните онлайн действия в зависимост от активността им онлайн и „репутацията“, която получават в тези сайтове като резултат от тяхната онлайн активност. Тези сайтове са моделирани по примера на популярния сайт Stack Overflow, интернет форум за компютърно програмиране и въпроси по тази тематика, като това е оригиналният начален сайт в тази мрежа. Идеята на системата с онлайн репутацията е сайтовете да са само-модериращи се.",
@@ -478,14 +499,13 @@
   "wolframalpha":"WolframAlpha е отговаряща машина, разработена от компанията Wolfram Research, чийто основател и главен изпълнителен директор е Стивън Волфрам.",
   "rumble":"Rumble е канадска алт-тех видео платформа. Сайтът е основан от Крис Павловски, технологичен предприемач.",
   "wttr.in":[
-   "Прогноза за времето в: Des Moines, Iowa, United States",
+   "Прогноза за времето в: not found",
    "https://wttr.in"
   ]
  },
  "bn":{
-  "apple app store":"অ্যাপ স্টোর একটি অ্যাপ স্টোর প্ল্যাটফর্ম যা অ্যাপল ইনক দ্বারা মোবাইল অ্যাপ্লিকেশন এর আইওএস এবং [[আইপ্যাডএস)] এর জন্য তৈরি এবং রক্ষণাবেক্ষণ করা হয় ]] অপারেটিং সিস্টেম। স্টোর ব্যবহারকারীদের অ্যাপলের আইওএস সফটওয়্যার ডেভলপমেন্ট কিট এর সাহায্যে উন্নত অ্যাপ্লিকেশনগুলি ব্রাউজ এবং ডাউনলোড করতে দেয়। অ্যাপ্লিকেশনগুলি আইফোন স্মার্টফোন, আইপড টাচ হ্যান্ডহেল্ড কম্পিউটার, বা আইপ্যাড ট্যাবলেট কম্পিউটারে ডাউনলোড করা যেতে পারে এবং কিছুগুলি অ্যাপল ওয়াচ স্মার্টওয়াচ বা চতুর্থ- প্রজন্ম বা নতুন অ্যাপল টিভি এর আইফোন অ্যাপ্লিকেশনগুলির এক্সটেনশন হিসাবে।",
   "arxiv":"arXiv হল সংশোধনের পর প্রকাশনার জন্য অনুমোদিত ইলেকট্রনিক প্রাক মুদ্রণের একটি সংগ্রহস্থল, যেটি গণিত, পদার্থবিজ্ঞান, জ্যোতির্বিজ্ঞান, কম্পিউটার বিজ্ঞান, পরিমাণগত জীববিদ্যা, পরিসংখ্যান এবং পরিমাণগত অর্থব্যবস্থা বিভাগের বৈজ্ঞানিক কাগজপত্র নিয়ে গঠিত। এগুলিতে অনলাইনের মাধ্যমে প্রবেশ করা যায়।",
-  "wikipedia":"উইকিপিডিয়া সম্মিলিতভাবে সম্পাদিত, বহুভাষিক, মুক্ত প্রবেশাধিকার, মুক্ত কন্টেন্ট সংযুক্ত স্বেচ্ছাসেবী সম্প্রদায় চালিত একটি ইন্টারনেট বিশ্বকোষ। স্বেচ্ছাসেবীরা বিশ্বব্যাপী সম্মিলিতভাবে ৩২৯টি ভাষার উইকিপিডিয়ায় প্রায় ৪০০ লক্ষ নিবন্ধ রচনা করেছেন, যার মধ্যে শুধুমাত্র ইংরেজি উইকিপিডিয়াতেই রয়েছে ৬৬ লক্ষের অধিক নিবন্ধ। যে কেউ উইকিপিডিয়া ওয়েবসাইটে প্রবেশের করে যে কোনো নিবন্ধের সম্পাদনা করতে পারেন, যা সম্মিলিতভাবে ইন্টারনেটের সর্ববৃহৎ এবং সর্বাধিক জনপ্রিয় সাধারণ তথ্যসূত্রের ঘাটতি পূরণ করে থাকে। ফেব্রুয়ারি ২০১৪ সালে, দ্য নিউ ইয়র্ক টাইমস জানায় উইকিপিডিয়া সমস্ত ওয়েবসাইটের মধ্যে বিশ্বব্যাপী পঞ্চম স্থানে অবস্থান করছে, \"মাসিক প্রায় ১৮ বিলিয়ন পৃষ্ঠা প্রদর্শন এবং প্রায় ৫০০ মিলিয়ন স্বতন্ত্র পরিদর্শক রয়েছে। উইকিপিডিয়ায় ইয়াহু, ফেসবুক, মাইক্রোসফট এবং গুগলের পথানুসরণ করে, সর্বাধিক ১.২ বিলিয়ন স্বতন্ত্র পরিদর্শক রয়েছে।\"",
+  "wikipedia":"উইকিপিডিয়া হলো সম্মিলিতভাবে সম্পাদিত, বহুভাষিক, মুক্ত প্রবেশাধিকার, মুক্ত কন্টেন্ট সংযুক্ত অনলাইন বিশ্বকোষ যা উইকিপিডিয়ান বলে পরিচিত স্বেচ্ছাসেবকদের একটি সম্প্রদায় কর্তৃক লিখিত এবং রক্ষণাবেক্ষণকৃত। স্বেচ্ছাসেবকরা উন্মুক্ত সহযোগিতার মাধ্যমে এবং মিডিয়াউইকি নামে একটি উইকি -ভিত্তিক সম্পাদনা ব্যবস্থা ব্যবহার করে। এটি ধারাবাহিকভাবে সিমিলারওয়েব এবং পূর্বে আলেক্সা কর্তৃক র‍্যাঙ্ককৃত ১০ টি জনপ্রিয় ওয়েবসাইটগুলির মধ্যে একটি; ২০২৩-এর হিসাব অনুযায়ী উইকিপিডিয়া বিশ্বের ৫ তম জনপ্রিয় সাইট হিসেবে স্থান পেয়েছে। ফেব্রুয়ারি ২০১৪ সালে, দ্য নিউ ইয়র্ক টাইমস জানায় উইকিপিডিয়া সমস্ত ওয়েবসাইটের মধ্যে বিশ্বব্যাপী পঞ্চম স্থানে অবস্থান করছে, \"মাসিক প্রায় ১৮ বিলিয়ন পৃষ্ঠা প্রদর্শন এবং প্রায় ৫০০ মিলিয়ন স্বতন্ত্র পরিদর্শক রয়েছে। উইকিপিডিয়ায় ইয়াহু, ফেসবুক, মাইক্রোসফট এবং গুগলের পথানুসরণ করে, সর্বাধিক ১.২ বিলিয়ন স্বতন্ত্র পরিদর্শক রয়েছে। উইকিপিডিয়ায় ইয়াহু, ফেসবুক, মাইক্রোসফট এবং গুগলের পথানুসরণ করে, সর্বাধিক ১.২ বিলিয়ন স্বতন্ত্র পরিদর্শক রয়েছে।\"",
   "bing":"বিং মাইক্রোসফট কর্তৃক নিয়ন্ত্রিত একটি ওয়েব অনুসন্ধান ইঞ্জিন । বিং বিভিন্ন ধরনের অনুসন্ধান সেবা প্রদান করে যেমন - ওয়েব, ভিডিও, চিত্র এবং মানচিত্র ইত্যাদি অনুসন্ধান সরবরাহ করে। এটি এএসপি ডট নেট ব্যবহার করে তৈরি করা।",
   "bing images":[
    "bing:bn",
@@ -543,7 +563,7 @@
   "soundcloud":"সাউন্ডক্লাউড হল জার্মানির রাজধানী বার্লিন-এ স্থাপিত একটি অনলাইন অডিও বণ্টন ভিত্তিক প্রচারের মাধ্যম, যেটি ব্যবহারকারীকে তার নিজেস্ব তৈরীকৃত শব্দ বা সঙ্গীত আপলোড, রেকর্ড, এর উন্নীতকরণ এবং সবার উদ্দেশ্যে তা প্রচারের অধিকার প্রদান করে।",
   "semantic scholar":"সিম্যান্টিক স্কলার হলো কৃত্রিম বুদ্ধিমত্তা-চালিত গবেষণার সরঞ্জাম। \"অ্যালেন ইনস্টিটিউট ফর এআই\" সাহিত্যের বৈজ্ঞানিক গবেষণার জন্য এটি তৈরি করেছে। ২০১৫ সালের নভেম্বরে এটি সর্বজনীনভাবে প্রকাশিত হয়। এটি পাণ্ডিত্যপূর্ণ কাগজপত্রের সারাংশ প্রদানের জন্য স্বাভাবিক ভাষা প্রক্রিয়াকরণে অগ্রগতি ব্যবহার করে। সিম্যান্টিক স্কলার টিম সক্রিয়ভাবে স্বাভাবিক ভাষা প্রক্রিয়াজাতকরণ, যন্ত্রীয় শিখন, মানব-কম্পিউটার মিথস্ক্রিয়া এবং তথ্য পুনরুদ্ধারে কৃত্রিম-বুদ্ধিমত্তার ব্যবহার নিয়ে গবেষণা করছে।",
   "startpage":"স্টার্টপেজ হল গোপনীয়তা নির্ভর সার্চ ইঞ্জিন। এটি আগে মেটাসার্চ এবং আইএক্সকুইক নামে ভিন্ন দুটি সার্চ ইঞ্জিন ছিল। ২০১৬ সালে দুটি কোম্পানি একীভূত হয়। পূর্বে এটি আইএক্স কুইক অনুসন্ধান ইঞ্জিন নামে পরিচিত ছিলো।",
-  "youtube":"ইউটিউব হলো সান ব্রুনো, ক্যালিফোর্নিয়া ভিত্তিক একটি মার্কিন অনলাইন ভিডিও প্ল্যাটফর্ম সেবার সাইট, যা ২০০৫ সালের ফেব্রুয়ারিতে প্রকাশিত হয়। ইউটিউব বর্তমানে গুরুত্বপূর্ণ একটি প্ল্যাটফর্ম। ২০০৬ সালের অক্টোবরে, গুগল সাইটটিকে ১.৬৫ বিলিয়ন মার্কিন ডলারের বিনিময়ে ক্রয় করে নেয়। ইউটিউব বর্তমানে গুগলের অন্যতম অধীনস্থ প্রতিষ্ঠান হিসেবে পরিচালিত হচ্ছে।",
+  "youtube":"ইউটিউব হলো সান ব্রুনো, ক্যালিফোর্নিয়া ভিত্তিক একটি বৈশ্বিক অনলাইন ভিডিও প্ল্যাটফর্ম সেবার সাইট এবং সামাজিক যোগাযোগ মাধ্যম, যা ২০০৫ সালের ফেব্রুয়ারিতে প্রকাশিত হয়। ইউটিউব বর্তমানে গুরুত্বপূর্ণ একটি প্ল্যাটফর্ম। ২০০৬ সালের অক্টোবরে, গুগল সাইটটিকে ১.৬৫ বিলিয়ন মার্কিন ডলারের বিনিময়ে ক্রয় করে নেয়। ইউটিউব বর্তমানে গুগলের অন্যতম অধীনস্থ প্রতিষ্ঠান হিসেবে পরিচালিত হচ্ছে।",
   "wikibooks":"উইকিবই হল উইকি-ভিত্তিক পরিবারের একটি উইকিমিডিয়া প্রকল্প যা মিডিয়াউইকি সফটওয়্যারের মাধ্যমে চালিত এবং উইকিমিডিয়া ফাউন্ডেশন কর্তৃক হোস্টকৃত। এটি একটি প্রকল্প, যেখানে বিভিন্ন প্রকার পাঠ্যপুস্তক পঠনযোগ্য আকারে সংরক্ষণ করা হয়।",
   "wikinews":"উইকিনিউজ বা উইকিসংবাদ হল উইকি মুক্ত বিষয়বস্তুর আলোকে পরিচালিত সংবাদ বিষয়ক ওয়েবসাইট এবং উইকিমিডিয়া ফাউন্ডেশনের একটি প্রকল্প যা বিশ্বব্যাপী সহযোগিতামুলক সাংবাদিকতার মাধ্যমে কাজ করে থাকে। উইকিপিডিয়ার সহ-প্রতিষ্ঠাতা জিমি ওয়েলস উইকিপিডিয়া থেকে উইকিসংবাদ আলাদা করার প্রসঙ্গে বলেন, \"উইকিসংবাদে, প্রতিটি গল্প বিশ্বকোষীয় নিবন্ধ থেকে ভিন্ন আঙ্গিকে মুলতঃ একটি সংবাদ হিসেবে লেখা হবে।\" উইকিসংবাদ নিরপেক্ষ দৃষ্টিভঙ্গি নীতি অনুসরণের মাধ্যমে সাংবাদিকতায় বস্তুনিষ্ঠতার ধারণা প্রতিষ্ঠা করে যা অন্যান্য নাগরিক সাংবাদিকতা চর্চার ওয়েবসাইট যেমন, ইন্ডেমিডিয়া ও ওহমাইনিউজ থেকে ভিন্নতর।",
   "wikiquote":"উইকিউক্তি উইকি-ভিত্তিক পরিবারের একটি প্রকল্প যা মিডিয়াউইকি সফটওয়্যারের মাধ্যমে চালিত এবং উইকিমিডিয়া ফাউন্ডেশন কর্তৃক পরিচালিত। এটি ড্যানিয়েল অ্যালস্টনের ধারণার উপর ভিত্তি করে এবং ব্রিয়ন ভিবের কর্তৃক বাস্তবায়িত। উইকিউক্তি পাতাসমূহ উইকিপিডিয়ার উল্লেখযোগ্য ব্যক্তিত্ব সম্পর্কে নিবন্ধে ক্রস-সংযুক্ত করা হয়।",
@@ -554,7 +574,7 @@
   "1337x":"১৩৩৭এক্স হল একটি ওয়েবসাইট যা বিটটরেন্ট প্রোটোকলের মাধ্যমে পিয়ার-টু-পিয়ার ফাইল আদান প্রদানের জন্য ব্যবহৃত টরেন্ট ফাইল এবং ম্যাগনেট লিঙ্কগুলির একটি ডিরেক্টরি প্রদান করে। টরেন্টফ্রিক নিউজ ব্লগ অনুসারে, ১৩৩৭এক্স ২০২১ সালের হিসাবে তৃতীয় সর্বাধিক জনপ্রিয় টরেন্ট ওয়েবসাইট।",
   "naver":"নেইভার একটি দক্ষিণ কোরীয় ইন্টারনেট ভিত্তিমঞ্চ। কোরিয়ার নেইভার কর্পোরেশন এটির পরিচালক। ১৯৯৯ সালে দক্ষিণ কোরিয়ার স্ব-উদ্ভাবিত অনুসন্ধান ইঞ্জিন ব্যবহারকারী প্রথম আন্তর্জাল প্রবেশদ্বার হিসেবে এটি যাত্রা শুরু করে। এটি ছিল বিশ্বের প্রথম পূর্ণাঙ্গ অনুসন্ধান সুবিধা প্রদানকারী ওয়েবসাইট, যেখানে বিভিন্ন শ্রেণীর অনুসন্ধান ফলাফল সংকলিত একটিমাত্র ফলাফল পাতায় সেগুলিকে প্রকাশ করা হত। এরপর নেইভার আরও বেশ কিছু নতুন সেবা যোগ করেছে, যাদের মধ্যে বৈদ্যুতিন ডাক (ই-মেইল) ও সংবাদের মতো প্রাথমিক সুবিধাগুলি থেকে শুরু করে বিশ্বের প্রথম ইন্টারনেটভিত্তিক প্রশ্নোত্তর ভিত্তিমঞ্চ \"নলেজ ইন\" অন্তর্ভুক্ত।",
   "wttr.in":[
-   "আবহাওয়া সঙ্ক্রান্ত তথ্য Des Moines, Iowa, United States",
+   "আবহাওয়া সঙ্ক্রান্ত তথ্য not found",
    "https://wttr.in"
   ]
  },
@@ -674,6 +694,10 @@
    "wikidata"
   ],
   "peertube":"PeerTube és una plataforma de vídeo federada i descentralitzada de codi obert, alimentada per ActivityPub i WebTorrent, que utilitza tecnologia peer-to-peer per reduir la càrrega en servidors individuals quan es visualitzen vídeos.",
+  "wttr.in":[
+   "Informe del temps per a: not found",
+   "https://wttr.in"
+  ],
   "brave":"Brave Search és un motor de cerca desenvolupat per Brave Software, Inc., que està establert com a motor de cerca predeterminat per als usuaris del navegador web Brave en determinats països.",
   "petalsearch":[
    "cercador web",
@@ -747,10 +771,6 @@
   ],
   "hoogle":"Haskell je standardizovaný funkcionální programovací jazyk používající líné vyhodnocování, pojmenovaný na počest logika Haskella Curryho. Jazyk se rychle vyvíjí, především díky svým implementacím Hugs a GHC.",
   "imdb":"Internet Movie Database (IMDb) je on-line databáze informací o filmech, televizních pořadech, hercích, herečkách, režisérech a všem ostatním, co s filmovou tvorbou souvisí.",
-  "invidious":[
-   "alternativní frontend pro YouTube",
-   "wikidata"
-  ],
   "library genesis":"Library Genesis je internetová knihovna poskytující knihy, akademické články, komiksy, audioknihy, časopisy a další obsah, který by byl jinak přístupný jen placeně, nebo vůbec ne.",
   "library of congress":"Knihovna Kongresu je národní knihovna Spojených států amerických a vědeckovýzkumné středisko Kongresu spojených států. Sídlí ve Washingtonu, D. C. S počtem 155 miliónů knihovních jednotek jde o největší knihovnu na světě.",
   "metacpan":"CPAN je softwarový repozitář obsahující jednak moduly pro programovací jazyk Perl a jednak aplikace napsané v tomto jazyce. První myšlenky k jeho zřízení se objevily už v roce 1993 inspirované repozitářem CTAN typografického systému TeX, ale do provozu byl oficiálně uveden až v roce 1995. Jméno CPAN nese kromě samotného repozitáře i perlový program, který slouží k stažení a instalaci modulů. Kromě toho je možné do repozitáře přistupovat i přes webové rozhraní, kde je například možné i bez instalace číst dokumentaci patřičného modulu generovanou ze standardního formátu POD.",
@@ -798,7 +818,7 @@
   "naver":"Naver je jihokorejská online platforma provozovaná společností Naver Corporation. Debutoval v roce 1999 jako první webový portál v Jižní Koreji. Byl také prvním operátorem na světě, který zavedl funkci komplexního vyhledávání, která sestavuje výsledky vyhledávání z různých kategorií a prezentuje je na jediné stránce. Naver od té doby přidal množství nových služeb, od základních funkcí, jako je e-mail a zprávy, až po světově první online platformu otázek a odpovědí Knowledge iN.",
   "peertube":"PeerTube je webová platforma pro hostování souborů, která je decentralizovaná a je svobodným softwarem pod licencí AGPL. Je postavena na protokolu Activity Pub a javascriptovém klientu WebTorrent, který umí používat technologii BitTorrent pro P2P stahování datových proudů pomocí webového prohlížeče.",
   "wttr.in":[
-   "Předpověď počasí pro: Des Moines, Iowa, United States",
+   "Předpověď počasí pro: not found",
    "https://wttr.in"
   ],
   "brave":"Brave Search je webový vyhledávač vyvíjený americkou společností Brave Software, Inc. Je přednastaveným vyhledávačem jejího webového prohlížeče Brave. Podobně jako prohlížeč klade velký důraz na soukromí uživatelů, tedy jde proti trendů využívání nástrojů webové analytiky."
@@ -817,6 +837,10 @@
    "Newyddion o ffynonellau newyddion byd, cenedlaethol a lleol, wedi eu trefnu i drafod newyddion chwaraeon, adloniant, busnes, gwleidyddiaeth, tywydd a mwy mewn manylder.",
    "https://www.bing.com/news"
   ],
+  "bing videos":[
+   "Mae chwilio deallus gan Bing yn ei gwneud yn haws i chi canfod yr hyn rydych chi'n chwilio amdano ac yn eich gwobrwyo.",
+   "https://www.bing.com/videos"
+  ],
   "wikidata":"Prosiect cydweithredol, byd-eang ydy Wicidata gan gymuned Wicimedia ; fe'i bwriedir i ganoli data ar gyfer prosiectau megis Wicipedia, fel a wneir gyda Comin Wicimedia. Mae'r cynnwys, fel gyda gweddill y teulu \"Wici\" wedi'i drwyddedu ar ffurf cynnwys rhydd, agored tebyg i'r CC-BY-SA a ddefnyddir ar y wici hwn.",
   "flickr":"Gwefan sy'n cynnal lluniau a fideos gan gymuned ar y we yw Flickr.",
   "gentoo":[
@@ -845,7 +869,7 @@
   "wikisource":"Prosiect Wicifryngau yw Wicidestun, sy'n ceisio adeiladu ystorfa testunau gwreiddiol sy'n eiddo cyhoeddus neu o dan termau'r Drwydded Dogfennaeth Rhydd GNU (\"GFDL\"). Mae'r safle yn rhan o'r Sefydliad Wicifryngau.",
   "wiktionary":"Un o brosiectau Sefydliad Wicifryngau gyda'r nod o greu geiriadur wici rhydd ym mhob iaith yw Wiciadur sy'n eiriadur Cymraeg - Saesneg. Erbyn Medi 2012 roedd gan y Wiciadur dros 17,000 o gofnodion mewn 65 o ieithoedd gwahanol. Gyda'r Wiciadur Cymraeg, darperir diffiniadau o ystyron geiriau ac ymadroddion Cymraeg eu hiaith tra bod cyfieithiadau o eiriau mewn ieithoedd eraill yn cael eu darparu.",
   "wttr.in":[
-   "Adroddiad tywydd ar gyfer: Des Moines, Iowa, United States",
+   "Adroddiad tywydd ar gyfer: not found",
    "https://wttr.in"
   ]
  },
@@ -923,7 +947,11 @@
   "wiktionary":"Wiktionary er en ordbog med åbent indhold fra Wikimedia Foundation. Den engelsksprogede version blev etableret i december 2002 og rummer mere end 3.500.000 artikler.",
   "wikiversity":"Wikiversity er et projekt etableret af Wikimedia Foundation, der ønsker af skabe et frit universitet ved hjælp af wikiteknik. Projektet er flersprogligt og omfatter foreløbig 17 sprog, dog ikke dansk.",
   "wikivoyage":"Wikivoyage er en fri webbaseret rejseguide for både rejsedestinationer og emner relateret til rejser, der er skrevet af frivillige forfattere.",
-  "wolframalpha":"Wolfram Alpha er en beregningsmæssig vidensmotor eller svarmaskine udviklet af Wolfram Research. Det er en online søgemaskine som kan svare på faktuelle forespørgsler direkte ved at behandle disse vha. eksterne kilder, i stedet for at oplyse en liste med links, som en normal søgemaskine måske ville."
+  "wolframalpha":"Wolfram Alpha er en beregningsmæssig vidensmotor eller svarmaskine udviklet af Wolfram Research. Det er en online søgemaskine som kan svare på faktuelle forespørgsler direkte ved at behandle disse vha. eksterne kilder, i stedet for at oplyse en liste med links, som en normal søgemaskine måske ville.",
+  "wttr.in":[
+   "Vejret i: not found",
+   "https://wttr.in"
+  ]
  },
  "de":{
   "9gag":"9GAG ist eine englischsprachige Online-Plattform, auf der Bilder, GIF-Animationen und Videos von Nutzern geteilt, kommentiert und bewertet werden. Bei einem Großteil der Postings handelt es sich um humoristische Inhalte wie Internet-Memes oder Rage Comics. Teilweise wird 9GAG aufgrund fehlender Quellenangaben bei den Beiträgen kritisiert.",
@@ -1006,10 +1034,6 @@
   "hoogle":"Haskell ist eine rein funktionale Programmiersprache, benannt nach dem US-amerikanischen Mathematiker Haskell Brooks Curry, dessen Arbeiten zur mathematischen Logik eine Grundlage funktionaler Programmiersprachen bilden. Haskell basiert auf dem Lambda-Kalkül, weshalb auch der griechische Buchstabe Lambda als Logo verwendet wird. Die wichtigste Implementierung ist der Glasgow Haskell Compiler (GHC).",
   "imdb":"Die Internet Movie Database ist eine US-amerikanische Datenbank zu Filmen, Fernsehserien, Videoproduktionen und Computerspielen sowie über Personen, die daran mitgewirkt haben. Im Juni 2022 gab es zu über elf Millionen Titeln Einträge. Diese verteilen sich unter anderem auf Einträge zu 613.000 verschiedenen Spielfilmproduktionen, über 260.000 Videofilmen, über 136.000 TV-Filmen, 227.000 TV-Serien, fast 6,8 Millionen Serienepisoden und mehr als 1,9 Millionen Einträge zu Podcast-Episoden. Zudem sind über 11,7 Millionen Film- und Fernsehschaffende aufgeführt. Betrieben wird die Datenbank seit 1998 von Amazon.",
   "ina":"Das Institut national de l’audiovisuel (INA) ist ein öffentlich-rechtliches französisches Unternehmen. Das INA war das erste digitalisierte Archiv Europas. Sein Auftrag ist es, alle französischen Rundfunk- und Fernsehproduktionen zu sammeln, zu bewahren und öffentlich zugänglich zu machen, in etwa vergleichbar dem Auftrag der Bibliothèque nationale de France (BnF), geschriebene und gedruckte Dokumente aller Art zu archivieren. Die Sammlungen des INA sind über seine Website für die Öffentlichkeit teilweise kostenlos zugänglich, der Rest unterliegt urheberrechtlichen Beschränkungen.",
-  "invidious":[
-   "alternatives Frontend für YouTube",
-   "wikidata"
-  ],
   "jisho":[
    "Japanisch-Englisch Wörterbuch",
    "wikidata"
@@ -1058,10 +1082,11 @@
   ],
   "startpage":"Startpage ist eine Suchmaschine, die die eingegebenen Suchanfragen an die Google-Suchmaschine weiterleitet und dadurch anonymisiert die Suchergebnisse anzeigt. Startpage will damit den Datenschutz ihrer Nutzer gewährleisten. Startpage wird von der niederländischen Startpage B.V. betrieben, die zur Surfboard Holding B.V. gehört.",
   "unsplash":"Unsplash ist eine internationale Website für Fotos, die von ihren Urhebern der Online-Community zur kostenlosen Verwendung zur Verfügung gestellt werden.",
+  "yahoo news":"Die Altaba Inc. war eine US-amerikanische Beteiligungsgesellschaft, die unter anderem Anteile an Alibaba und Yahoo! Japan hielt. Gegründet wurde das Unternehmen als Internetunternehmen von David Filo und Jerry Yang im Januar 1994 unter dem Namen Yahoo.",
   "youtube":"YouTube ist ein 2005 gegründetes Videoportal des US-amerikanischen Unternehmens YouTube, LLC, seit 2006 eine Tochtergesellschaft von Google LLC, mit Sitz im kalifornischen San Bruno. Die Benutzer können auf dem Portal kostenlos Videoclips ansehen, bewerten, kommentieren und selbst hochladen. 2019 erzielte YouTube einen Jahresumsatz von 15 Milliarden Dollar. Die Einnahmen werden zum Großteil durch das Abspielen von Werbespots generiert.",
   "dailymotion":"Dailymotion ist ein Videoportal des gleichnamigen französischen Unternehmens, bei dem Videos hochgeladen und öffentlich angeschaut werden können. Es wurde 2005 in Paris gegründet und gehört zu den führenden Videoportalen. Dailymotion war die erste bekannte Videoplattform, die eine Auflösung von 720p (HD) unterstützte.",
   "vimeo":"Vimeo ist ein 2004 gegründetes Videoportal des US-amerikanischen Unternehmens Vimeo LLC mit Sitz ursprünglich in White Plains im Bundesstaat New York und inzwischen am Hudson River in Manhattan. Es unterstützt seit 2007 das Streaming von Videos in HD und seit 2015 in 4K Ultra HD. Neben der kostenlosen Nutzung des Portals bietet es auch die Möglichkeit, kostenpflichtige Inhalte zu veröffentlichen.",
-  "wikibooks":"Wikibooks [ˌvɪkiˈbʊks] ist ein so genanntes Wiki zur Erstellung von Lehr-, Sach- und Fachbüchern unter der Creative Commons Attribution/Share-Alike Lizenz 3.0 und GNU-Lizenz für freie Dokumentation.",
+  "wikibooks":"Wikibooks ist ein so genanntes Wiki zur Erstellung von Lehr-, Sach- und Fachbüchern unter der Creative Commons Attribution/Share-Alike Lizenz 3.0 und GNU-Lizenz für freie Dokumentation.",
   "wikinews":"Wikinews ist ein internationales Wikimedia-Projekt zur gemeinschaftlichen Erstellung einer freien und neutralen Nachrichtenquelle. Wikinews ermöglicht es jedem Internet-Nutzer, Nachrichten zu einem breiten Themenkreis zu veröffentlichen. Dazu setzt es wie seine Schwesterprojekte Wikipedia, Wiktionary, Wikibooks, Wikiquote, Wikispecies und Wikisource die Wiki-Software MediaWiki ein.",
   "wikiquote":"Wikiquote [ˌvɪkiˈkwoʊt] ist ein freies Online-Projekt mit dem Ziel, auf Wiki-Basis eine freie Zitatensammlung mit Zitaten in jeder Sprache zu schaffen. Wikiquote basiert wie die Wikipedia auf der Software MediaWiki. Für zusätzliche Informationen sorgen Links in die Wikipedia und zu Wikimedia-Projekten wie den Wikimedia Commons, Wikisource oder dem Wiktionary.",
   "wikisource":"Wikisource [ˌvɪkiˈsɔːɹs] ist ein freies Online-Projekt zur Sammlung und Edition von Texten, die entweder urheberrechtsfrei (gemeinfrei) sind oder unter einer freien Lizenz stehen. Wie das Schwesterprojekt Wikipedia wird Wikisource von der Wikimedia Foundation betrieben und nutzt als Software MediaWiki.",
@@ -1092,7 +1117,7 @@
    "wikidata"
   ],
   "wttr.in":[
-   "Wetterbericht für: Des Moines, Iowa, United States",
+   "Wetterbericht für: not found",
    "https://wttr.in"
   ],
   "brave":"Brave Search ist eine Internet-Suchmaschine des US-amerikanischen Browserherstellers Brave Software Inc. Die Suchmaschine legt dabei ähnlich wie der Webbrowser vom selben Unternehmen Wert auf die Privatsphäre des Nutzers, so dass Tracking und Werbung herausgefiltert werden. Brave Search setzt auf einen eigenen Index, um die Suchergebnisse auszugeben.",
@@ -1190,7 +1215,7 @@
   "wikiversity":"Το Βικιεπιστήμιο είναι ένα έργο του Ιδρύματος Wikimedia που υποστηρίζει τις κοινότητες μάθησης, το μαθησιακό τους υλικό και τις συνακόλουθες δραστηριότητες. Διαφέρει από τα πιο δομημένα έργα όπως η Βικιπαίδεια, επειδή προσφέρει μια σειρά από μαθήματα, τμήματα και σχολές για την προώθηση της μάθησης παρά για το περιεχόμενο.",
   "wikivoyage":"Τα Βικιταξίδια είναι ένας ελεύθερος διαδικτυακός ταξιδιωτικός οδηγός, για ταξιδιωτικούς προορισμούς και θέματα ευρύτερου ταξιδιωτικού ενδιαφέροντος, ο οποίος συντάσσεται από εθελοντές. Το όνομα αποτελεί συνδυασμό της λέξης \"Wiki\" και \"Ταξίδια\".",
   "wttr.in":[
-   "Πρόγνωση καιρού για: Des Moines, Iowa, United States",
+   "Πρόγνωση καιρού για: not found",
    "https://wttr.in"
   ]
  },
@@ -1212,7 +1237,7 @@
   "artic":"The Art Institute of Chicago in Chicago's Grant Park, founded in 1879, is one of the oldest and largest art museums in the world. Recognized for its curatorial efforts and popularity among visitors, the museum hosts approximately 1.5 million people annually. Its collection, stewarded by 11 curatorial departments, is encyclopedic, and includes iconic works such as Georges Seurat's A Sunday on La Grande Jatte, Pablo Picasso's The Old Guitarist, Edward Hopper's Nighthawks, and Grant Wood's American Gothic. Its permanent collection of nearly 300,000 works of art is augmented by more than 30 special exhibitions mounted yearly that illuminate aspects of the collection and present cutting-edge curatorial and scientific research.",
   "arxiv":"arXiv is an open-access repository of electronic preprints and postprints approved for posting after moderation, but not peer review. It consists of scientific papers in the fields of mathematics, physics, astronomy, electrical engineering, computer science, quantitative biology, statistics, mathematical finance and economics, which can be accessed online. In many fields of mathematics and physics, almost all scientific papers are self-archived on the arXiv repository before publication in a peer-reviewed journal. Some publishers also grant permission for authors to archive the peer-reviewed postprint. Begun on August 14, 1991, arXiv.org passed the half-million-article milestone on October 3, 2008, and had hit a million by the end of 2014. As of April 2021, the submission rate is about 16,000 articles per month.",
   "bandcamp":"Bandcamp is an American online audio distribution platform founded in 2007 by Oddpost co-founder Ethan Diamond and programmers Shawn Grunberger, Joe Holt and Neal Tucker, with headquarters in Oakland, California, US. On March 2, 2022, Bandcamp was acquired by Epic Games.",
-  "wikipedia":"Wikipedia is a multilingual free online encyclopedia written and maintained by a community of volunteers, known as Wikipedians, through open collaboration and using a wiki-based editing system called MediaWiki. Wikipedia is the largest and most-read reference work in history. It is consistently one of the 10 most popular websites ranked by Similarweb and formerly Alexa; as of 2022, Wikipedia was ranked the 5th most popular site in the world. It is hosted by the Wikimedia Foundation, an American non-profit organization funded mainly through donations.",
+  "wikipedia":"Wikipedia is a multilingual free online encyclopedia written and maintained by a community of volunteers, known as Wikipedians, through open collaboration and using a wiki-based editing system called MediaWiki. Wikipedia is the largest and most-read reference work in history. It is consistently one of the 10 most popular websites ranked by Similarweb and formerly Alexa; as of 2023, Wikipedia was ranked the 5th most popular site in the world. It is hosted by the Wikimedia Foundation, an American non-profit organization funded mainly through donations.",
   "bing":"Microsoft Bing is a web search engine owned and operated by Microsoft. The service has its origins in Microsoft's previous search engines: MSN Search, Windows Live Search and later Live Search. Bing provides a variety of search services, including web, video, image and map search products. It is developed using ASP.NET.",
   "bing images":[
    "bing:en",
@@ -1233,13 +1258,13 @@
    "Search over 600 million free and openly licensed images, photos, audio, and other media types for reuse and remixing.",
    "https://wordpress.org/openverse/"
   ],
-  "crossref":"Crossref is an official digital object identifier (DOI) Registration Agency of the International DOI Foundation. It is run by the Publishers International Linking Association Inc. (PILA) and was launched in early 2000 as a cooperative effort among publishers to enable persistent cross-publisher citation linking in online academic journals. In August 2022, Crossref lists that index more than 60 million journal studies were made free to view and reuse, and they made a challenge publicly to other publishers, to add their reference data to the index.",
+  "crossref":"Crossref is an official digital object identifier (DOI) Registration Agency of the International DOI Foundation. It is run by the Publishers International Linking Association Inc. (PILA) and was launched in early 2000 as a cooperative effort among publishers to enable persistent cross-publisher citation linking in online academic journals. In August 2022, Crossref lists that index more than 60 million journal studies were made free to view and reuse, publicly challenging other publishers to add their reference data to the index.",
   "yep":[
    "When you search with Yep, you’re putting actual dollars in the pockets of your favorite content creators.",
    "https://yep.com"
   ],
   "curlie":"DMOZ was a multilingual open-content directory of World Wide Web links. The site and community who maintained it were also known as the Open Directory Project (ODP). It was owned by AOL but constructed and maintained by a community of volunteer editors.",
-  "currency":"DuckDuckGo (DDG) is an internet search engine that emphasizes protecting searchers' privacy and avoiding the filter bubble of personalized search results. DuckDuckGo does not show search results from content farms. It uses various APIs of other websites to show quick results to queries and for traditional links it uses the help of its partners and its own crawler. Because of its anonymity, it is impossible to know how many people use DuckDuckGo.",
+  "currency":"DuckDuckGo (DDG) is an internet search engine that emphasizes protecting searchers' privacy and used to avoid the filter bubble of personalized search results. DuckDuckGo does not show search results from content farms. It uses various APIs of other websites to show quick results to queries and for traditional links it uses the help of its partners and its own crawler. Because of its anonymity, it is impossible to know how many people use DuckDuckGo.",
   "deezer":"Deezer is a French online music streaming service. It allows users to listen to music content from record labels, as well as podcasts on various devices online or offline.",
   "deviantart":"DeviantArt is an American online art community that features artwork, videography and photography, launched on August 7, 2000 by Angelo Sotira, Scott Jarkoff, and Matthew Stephens among others.",
   "ddg definitions":[
@@ -1276,10 +1301,10 @@
   "flickr":"Flickr is an American image hosting and video hosting service, as well as an online community, founded in Canada and headquartered in the United States. It was created by Ludicorp in 2004 and was a popular way for amateur and professional photographers to host high-resolution photos. It has changed ownership several times and has been owned by SmugMug since April 20, 2018.",
   "free software directory":"The Free Software Directory (FSD) is a project of the Free Software Foundation (FSF). It catalogs free software that runs under free operating systems—particularly GNU and Linux. The cataloged projects are often able to run in several other operating systems. The project was formerly co-run by UNESCO.",
   "frinkiac":"Frinkiac is a website for users to search for words or phrases from episodes of the American animated sitcom The Simpsons. It returns screenshots related to the search terms, from which it generates memes and animated GIFs. Created by Paul Kehrer, Sean Schulte and Allie Young, the site is named after a computer built by one of the show's recurring characters, Professor Frink. The site was critically acclaimed upon its launch, and Newsweek wrote that it \"may be the greatest feat of Internet engineering we've ever seen\". As of May 2016, screenshots from the first seventeen seasons of The Simpsons are in Frinkiac's database.",
-  "genius":"Genius is an American digital media company founded on August 27, 2009, by Tom Lehman, Ilan Zechory, and Mahbod Moghadam. The site allows users to provide annotations and interpretation to song lyrics, news stories, sources, poetry, and documents.",
+  "genius":"Genius is an American digital media company founded on August 27, 2009, by Tom Lehman, Ilan Zechory, and Mahbod Moghadam. Its site of the same name allows users to provide annotations and interpretation to song lyrics, news stories, sources, poetry, and documents.",
   "gigablast":"Gigablast is an American free and open-source web search engine and directory. Founded in 2000, it is an independent engine and web crawler, developed and maintained by Matt Wells, a former Infoseek employee and New Mexico Tech graduate.",
-  "gentoo":"Gentoo Linux is a Linux distribution built using the Portage package management system. Unlike a binary software distribution, the source code is compiled locally according to the user's preferences and is often optimized for the specific type of computer. Precompiled binaries are available for some larger packages or those with no available source code.",
-  "gitlab":"GitLab Inc. is an open-core company that operates GitLab, a DevOps software package which can develop, secure, and operate software. The open source software project was created by Ukrainian developer Dmitriy Zaporozhets and Dutch developer Sytse Sijbrandij. In 2018, GitLab Inc. was considered the first partly-Ukrainian unicorn.",
+  "gentoo":"Gentoo Linux is a Linux distribution built using the Portage package management system. Unlike a binary software distribution, the source code is compiled locally according to the user's preferences and is often optimized for the specific type of computer. Precompiled binaries are available for some packages.",
+  "gitlab":"GitLab Inc. is an open-core company that operates GitLab, a DevOps software package which can develop, secure, and operate software. The open source software project was created by Ukrainian developer Dmytro Zaporozhets and Dutch developer Sytse Sijbrandij. In 2018, GitLab Inc. was considered the first partly-Ukrainian unicorn.",
   "github":"GitHub, Inc. is an Internet hosting service for software development and version control using Git. It provides the distributed version control of Git plus access control, bug tracking, software feature requests, task management, continuous integration, and wikis for every project. Headquartered in California, it has been a subsidiary of Microsoft since 2018.",
   "codeberg":[
    "Codeberg is founded as a Non-Profit Organization, with the objective to give the Open-Source code that is running our world a safe and friendly home, and to ensure that free code remains free and secure forever.",
@@ -1304,8 +1329,8 @@
   "imdb":"IMDb is an online database of information related to films, television series, podcasts, home videos, video games, and streaming content online – including cast, production crew and personal biographies, plot summaries, trivia, ratings, and fan and critical reviews. IMDb began as a fan-operated movie database on the Usenet group \"rec.arts.movies\" in 1990, and moved to the Web in 1993. Since 1998, it is now owned and operated by IMDb.com, Inc., a subsidiary of Amazon.",
   "ina":"The Institut national de l'audiovisuel, is a repository of all French radio and television audiovisual archives. Additionally it provides free access to archives of countries such as Afghanistan and Cambodia. It has its headquarters in Bry-sur-Marne.",
   "invidious":[
-   "alternative front end for YouTube",
-   "wikidata"
+   "Invidious Instances",
+   "https://api.invidious.io/"
   ],
   "jisho":[
    "online Japanese-English dictionary",
@@ -1313,7 +1338,7 @@
   ],
   "kickass":"KickassTorrents was a website that provided a directory for torrent files and magnet links to facilitate peer-to-peer file sharing using the BitTorrent protocol. It was founded in 2008 and by November 2014, KAT became the most visited BitTorrent directory in the world, overtaking The Pirate Bay, according to the site's Alexa ranking. KAT went offline on 20 July 2016 when the domain was seized by the U.S. government. The site's proxy servers were shut down by its staff at the same time.",
   "library genesis":"Library Genesis (Libgen) is a file-sharing based shadow library website for scholarly journal articles, academic and general-interest books, images, comics, audiobooks, and magazines. The site enables free access to content that is otherwise paywalled or not digitized elsewhere. Libgen describes itself as a \"links aggregator\", providing a searchable database of items \"collected from publicly available public Internet resources\" as well as files uploaded \"from users\".",
-  "library of congress":"The Library of Congress (LOC) is a research library in Washington, D.C. that serves as the library of the U.S. Congress and the de facto national library of the United States. Founded in 1800, the library is the United States's oldest federal cultural institution. The library is housed in three buildings in the Capitol Hill area of Washington. The Library also maintains a conservation center in Culpeper, Virginia. The library's functions are overseen by the Librarian of Congress, and its buildings are maintained by the Architect of the Capitol. The Library of Congress is one of the largest libraries in the world. Its collections contain approximately 173 million items, and it has more than 3000 employees. Its \"collections are universal, not limited by subject, format, or national boundary, and include research materials from all parts of the world and in more than 470 languages.\"",
+  "library of congress":"The Library of Congress (LOC) is a research library in Washington, D.C. that serves as the library of the U.S. Congress and the de facto national library of the United States. Founded in 1800, the library is the United States's oldest federal cultural institution. The library is housed in three buildings in the Capitol Hill area of Washington. The Library also maintains a conservation center in Culpeper, Virginia. The library's functions are overseen by the Librarian of Congress, and its buildings are maintained by the Architect of the Capitol. The Library of Congress is one of the largest libraries in the world. Its collections contain approximately 173 million items, and it has more than 3,000 employees. Its \"collections are universal, not limited by subject, format, or national boundary, and include research materials from all parts of the world and in more than 470 languages.\"",
   "lingva":[
    "Alternative front-end for Google Translate, serving as a Free and Open Source translator with over a hundred languages available",
    "https://lingva.ml"
@@ -1323,7 +1348,7 @@
    "wikidata"
   ],
   "azlyrics":[
-   "AZLyrics - request for access",
+   "AZLyrics - Song Lyrics from A to Z",
    "https://azlyrics.com"
   ],
   "metacpan":"The Comprehensive Perl Archive Network (CPAN) is a repository of over 250,000 software modules and accompanying documentation for 39,000 distributions, written in the Perl programming language by over 12,000 contributors. CPAN can denote either the archive network or the Perl program that acts as an interface to the network and as an automated software installer. Most software on CPAN is free and open source software.",
@@ -1385,7 +1410,7 @@
    "https://sepiasearch.org"
   ],
   "soundcloud":"SoundCloud is a German music streaming service that enables its users to upload, promote, and share audio. Founded in 2007 by Alexander Ljung and Eric Wahlforss, SoundCloud is one of the largest music streaming services in the world and is available in 190 countries and territories. The service has more than 76 million active monthly users and over 200 million audio tracks as of November 2021. SoundCloud offers both free and paid memberships on the platform, available for mobile, desktop and Xbox devices. SoundCloud has evolved from a traditional online streaming platform to an entertainment company.",
-  "stackoverflow":"Stack Exchange is a network of question-and-answer (Q&A) websites on topics in diverse fields, each site covering a specific topic, where questions, answers, and users are subject to a reputation award process. The reputation system allows the sites to be self-moderating. As of August 2019, the three most actively-viewed sites in the network are Stack Overflow, Super User, and Ask Ubuntu.",
+  "stackoverflow":"Stack Exchange is a network of question-and-answer (Q&A) websites on topics in diverse fields, each site covering a specific topic, where questions, answers, and users are subject to a reputation award process. The reputation system allows the sites to be self-moderating. As of March 2023, the three most actively-viewed sites in the network are Stack Overflow, Unix & Linux, and Mathematics.",
   "askubuntu":[
    "stackoverflow:en",
    "ref"
@@ -1414,7 +1439,7 @@
    "https://search.yahoo.com/"
   ],
   "yahoo news":"Yahoo! News is a news website that originated as an internet-based news aggregator by Yahoo!. The site was created by a Yahoo! software engineer named Brad Clawsie in August 1996. Articles originally came from news services such as the Associated Press, Reuters, Fox News, Al Jazeera, ABC News, USA Today, CNN and BBC News.",
-  "youtube":"YouTube is a global online video sharing and social media platform headquartered in San Bruno, California. It was launched on February 14, 2005, by Steve Chen, Chad Hurley, and Jawed Karim. It is owned by Google, and is the second most visited website, after Google Search. YouTube has more than 2.5 billion monthly users who collectively watch more than one billion hours of videos each day. As of May 2019, videos were being uploaded at a rate of more than 500 hours of content per minute.",
+  "youtube":"YouTube is an American global online video sharing and social media platform headquartered in San Bruno, California, United States. It was launched on February 14, 2005, by Steve Chen, Chad Hurley, and Jawed Karim. It is owned by Google and is the second most visited website, after Google Search. YouTube has more than 2.5 billion monthly users, who collectively watch more than one billion hours of videos each day. As of May 2019, videos were being uploaded at a rate of more than 500 hours of content per minute.",
   "dailymotion":"Dailymotion is a French video-sharing technology platform owned by Vivendi. North American launch partners included Vice Media, Bloomberg and Hearst Digital Media. It is among the earliest known platforms to support HD (720p) resolution video. Dailymotion is available worldwide in 183 languages and 43 localised versions featuring local home pages and local content.",
   "vimeo":"Vimeo, Inc. is an American video hosting, sharing, and services platform provider headquartered in New York City. Vimeo focuses on the delivery of high-definition video across a range of devices. Vimeo's business model is through software as a service (SaaS). They derive revenue by providing subscription plans for businesses and video content producers. Vimeo provides its subscribers with tools for video creation, editing, and broadcasting, enterprise software solutions, as well as the means for video professionals to connect with clients and other professionals. As of December 2021, the site has 260 million users, with around 1.6 million subscribers to its services.",
   "wiby":[
@@ -1448,7 +1473,7 @@
   "rubygems":"RubyGems is a package manager for the Ruby programming language that provides a standard format for distributing Ruby programs and libraries, a tool designed to easily manage the installation of gems, and a server for distributing them. It was created by Chad Fowler, Jim Weirich, David Alan Black, Paul Brannan and Richard Kilmer during RubyConf 2004.",
   "peertube":"PeerTube is a free and open-source, decentralized, ActivityPub federated video platform powered by WebTorrent, that uses peer-to-peer technology to reduce load on individual servers when viewing videos.",
   "mediathekviewweb":"MediathekView is a free open-source software designed to manage the online multimedia libraries of several German public broadcasters as well as an Austrian, a Swiss and a Franco-German public broadcaster. The software comes with a German user interface that lists broadcasts available online. In October 2016, the developer announced that maintenance of the project would be discontinued at the end of the year. Three weeks later, the user community had formed a team to continue the project, and the software continues to remain open-source.",
-  "rumble":"Rumble is an online video platform and cloud services business headquartered in Toronto, Ontario with its U.S. headquarters in Longboat Key, Florida. It was founded in October 2013 by Chris Pavlovski, a Canadian technology entrepreneur. The cloud services business hosts Truth Social, and the video platform is popular among American right and far-right users. The platform has been described as part of \"alt-tech\".",
+  "rumble":"Rumble is an online video platform, web hosting and cloud services business headquartered in Toronto, Ontario, with its U.S. headquarters in Longboat Key, Florida. It was founded in October 2013 by Chris Pavlovski, a Canadian technology entrepreneur. The cloud services business hosts Truth Social, and the video platform is popular among American right and far-right users. The platform has been described as part of \"alt-tech\".",
   "wordnik":"Wordnik, a nonprofit organization, is an online English dictionary and language resource that provides dictionary and thesaurus content. Some of the content is based on print dictionaries such as the Century Dictionary, the American Heritage Dictionary, WordNet, and GCIDE. Wordnik has collected a corpus of billions of words which it uses to display example sentences, allowing it to provide information on a much larger set of words than a typical dictionary. Wordnik uses as many real examples as possible when defining a word.",
   "sjp.pwn":[
    "Polish online dictionary",
@@ -1463,10 +1488,7 @@
    "wikidata"
   ],
   "brave":"Brave Search is a search engine developed by Brave Software, Inc., which is set as the default search engine for Brave web browser users in certain countries.",
-  "petalsearch":[
-   "search engine",
-   "wikidata"
-  ],
+  "petalsearch":"Petal Search is a search engine owned and operated by Huawei.",
   "petalsearch images":[
    "petalsearch:en",
    "ref"
@@ -1531,10 +1553,6 @@
    "muzeo en Francio",
    "wikidata"
   ],
-  "invidious":[
-   "alternativa antaŭa finaĵo por YouTube",
-   "wikidata"
-  ],
   "library of congress":"La Kongresa Biblioteko estas la neoficiala nacia biblioteko de Usono. Havante pli ol 128 milionojn da eroj ĝi estas la dua plej granda bibliodeko de la mondo, superata sole de la Brita Biblioteko. Ĝiaj kolektoj enhavas pli ol 28 milionojn da katalogitaj libroj kaj aliaj presaĵoj en 470 lingvoj, pli ol 50 milionojn da manuskriptoj, la kolekton plej grandan de libroj raraj en Nordameriko, inkluzivantan la biblion de Gutenberg, kaj la kolekton plej grandan en la mondo de materialoj leĝaj, filmoj, kartoj kaj surbendigaĵoj sonaj.",
   "npm":[
    "pako-administrilo por Node.js",
@@ -1667,9 +1685,9 @@
    "qwant:es",
    "ref"
   ],
-  "reddit":"Reddit es un sitio web de marcadores sociales y agregador de noticias donde los usuarios pueden añadir textos, imágenes, videos o enlaces. Los usuarios pueden votar a favor o en contra del contenido, haciendo que aparezcan en las publicaciones destacadas. Se trata de un mapa de discusión, como parte de un DDS global distribuido. Su público es mayoritariamente anglosajón y la mayoría de la actividad se realiza en inglés. Reddit fue software libre desde el 19 de junio de 2009 hasta septiembre de 2017, cuando la compañía archivó y cerró el acceso a sus repositorios en Github, que incluían todo el código escrito para Reddit excepto las partes anti-spam.",
+  "reddit":"Reddit es un [sitio web] de marcadores sociales y buscadores web y agregador de noticias donde los usuarios pueden añadir textos, imágenes, videos o enlaces. Los usuarios pueden votar a favor o en contra del contenido, haciendo que aparezcan en las publicaciones destacadas. Se trata de un mapa de discusión, como parte de un DDR global distribuido. Su público es mayoritariamente anglosajón y la mayoría de la actividad se realiza en inglés. Reddit fue software libre desde el 19 de junio de 2009 hasta septiembre de 2017, cuando la compañía archivó y cerró el acceso a sus repositorios en Github, que incluían todo el código escrito para Reddit excepto las partes anti-spam.",
   "soundcloud":"SoundCloud es un servicio de retransmisión de música vía streaming que, a diferencia de Spotify y otras plataformas, tiene la opción de poder subir canciones y álbumes directamente, sin la necesidad de distribuidoras externas.",
-  "stackoverflow":"Stack Exchange es una red de webs de preguntas y respuestas sobre distintos temas, donde las preguntas, respuestas y los usuarios están sujetos a un sistema de reputación y recompensas. Dicho sistema permite que los sitios se automoderen.",
+  "stackoverflow":"Stack Exchange es una red de webs de preguntas y respuestas sobre distintos temas, donde las preguntas, respuestas y los usuarios están sujetos a un sistema de reputación y recompensas, parecido al de Reddit. Dicho sistema permite que los sitios se automoderen.",
   "askubuntu":[
    "stackoverflow:es",
    "ref"
@@ -1681,6 +1699,10 @@
   "semantic scholar":"Semantic Scholar es un motor de búsqueda respaldado por un sistema de inteligencia artificial dedicado a trabajar con publicaciones académicas. Desarrollado en el Allen Institute for Artificial Intelligence, se lanzó al público en noviembre de 2015. Utiliza avances recientes en el procesamiento del lenguaje natural para proporcionar resúmenes de artículos académicos.",
   "startpage":"Startpage es un motor de búsqueda holandés, que destaca la privacidad como su característica distintiva. El sitio web anuncia que permite a los usuarios obtener resultados del Buscador de Google protegiendo la privacidad de los usuarios al no almacenar información personal ni datos de búsqueda y eliminar todos los rastreadores. Startpage también incluye una función de navegación anónima que permite a los usuarios la opción de abrir los resultados de búsqueda a través de un proxy para aumentar el anonimato. Dado que la empresa tiene su sede en los Países Bajos, está protegida por las leyes de privacidad neerlandesa y de la Unión Europea, por lo que no está sujeta a los programas de vigilancia de Estados Unidos, como PRISM.",
   "unsplash":"Unsplash es el sitio web internacional en el que están colocadas las fotografías de stock con licencia Unsplash. Desde 2021 es propiedad de Getty Images. El sitio web cuenta con más de 207,000 fotógrafos colaboradores y genera más de 17 mil millones de impresiones fotográficas por mes en su creciente biblioteca de más de 2 millones de fotos. Unsplash es uno de los sitios web principales de fotografía.",
+  "yahoo news":[
+   "sitio web de noticias de Yahoo!",
+   "wikidata"
+  ],
   "youtube":"YouTube es un sitio web de origen estadounidense dedicado a compartir videos. Presenta una variedad de clips de películas, programas de televisión y vídeos musicales, así como contenidos amateur como videoblogs y YouTube Gaming. Las personas que crean contenido para esta plataforma generalmente son conocidas como youtubers.",
   "dailymotion":"Dailymotion es un sitio web en el cual los usuarios pueden subir, ver y compartir vídeos. Aloja una variedad de clips de películas, programas de televisión y vídeos musicales, así como contenidos amateur como videoblogs.",
   "vimeo":"Vimeo es una red social de Internet basada en videos, lanzada en noviembre de 2004 por la compañía InterActiveCorp (IAC). Es una plataforma de vídeo sin publicidad con sede en la ciudad de Nueva York, que proporciona servicios de visualización de vídeo libres.",
@@ -1703,7 +1725,7 @@
   "rumble":"Rumble es una plataforma de video en línea canadiense con sede en Toronto. Fue fundada en 2013 por Chris Pavlovski, un emprendedor tecnológico de Canadá. El recuento mensual de usuarios de Rumble ha experimentado un rápido crecimiento desde julio de 2020, pasando de 1,6 millones de usuarios mensuales a 31,9 millones al final del primer trimestre de 2021.",
   "wikimini":"Wikimini es una enciclopedia en línea para niños, gratuita, que tiene la particularidad de ser escrita colaborativamente por niños y adolescentes. Su contenido está dirigido a lectores de 8 a 13 años y está publicado bajo licencia libre, lo que permite su difusión y reutilización. Desde que se puso en línea el 1 de octubre de 2008 por el friburgués Laurent Jauquier, el sitio ha experimentando un crecimiento en aumento dentro de la comunidad francófona.",
   "wttr.in":[
-   "El tiempo en: Des Moines, Iowa, United States",
+   "El tiempo en: not found",
    "https://wttr.in"
   ],
   "brave":"Brave Search es un motor de búsqueda desarrollado por Brave Software, Inc. y está configurado como el motor de búsqueda predeterminado para los usuarios del navegador web Brave en ciertos países.",
@@ -1778,7 +1800,7 @@
  },
  "eu":{
   "artic":"Chicagoko Arte Institutua, ingelesez: Art Institute of Chicago, AEBetako Chicago hirian dagoen arte-museo bat da. Munduko arte-museo garrantzitsuenetako bat da, eta ziurrenik AEBetako hiru arte-museo nabarmenetako bat, New Yorkeko Metropoliar Museoa eta Bostongo Arte Ederren Museoarekin batera.",
-  "wikipedia":"Wikipedia eduki askeko entziklopedia bat da, lankidetzaz editatua, eleanitza, Interneten argitaratua, Wikimedia Fundazioa irabazi asmorik gabeko erakundeak sustengatua. Wikipedia mundu osoko boluntarioek idazten dute. Internetera konektatutako edonork parte har dezake Wikipediako artikuluetan, aldatu lotura sakatuz. 2015ko azaroaren bostean, 291 hizkuntzatako edizioak zituen, eta horietatik 275 zeuden aktibo. Proiektuaren xedea da ahalik eta hizkuntza gehienetan idatzitako entziklopedia sortu eta hedatzea. Guztira 37 milioi artikulu ditu, horietatik 406.309 euskaraz eta bost milioitik gora ingelesez.",
+  "wikipedia":"Wikipedia eduki askeko entziklopedia bat da, lankidetzaz editatua, eleanitza, Interneten argitaratua, Wikimedia Fundazioa irabazi asmorik gabeko erakundeak sustengatua. Wikipedia mundu osoko boluntarioek idazten dute. Internetera konektatutako edonork parte har dezake Wikipediako artikuluetan, aldatu lotura sakatuz. 2015ko azaroaren bostean, 291 hizkuntzatako edizioak zituen, eta horietatik 275 zeuden aktibo. Proiektuaren xedea da ahalik eta hizkuntza gehienetan idatzitako entziklopedia sortu eta hedatzea. Guztira 37 milioi artikulu ditu, horietatik 408.697 euskaraz eta bost milioitik gora ingelesez.",
   "bing":[
    "Microsoft enpresak garatutako bilaketa motorra",
    "wikidata"
@@ -1884,6 +1906,10 @@
    "اخبار به دست آمده از منابع جهانی، ملی و محلی، به‌گونه‌ای سازماندهی شده‌اند تا پوشش جامع خبری را در حوزه ورزش، سرگرمی، کسب و کار، سیاست، آب و هوا، و غیره به شما ارائه دهند.",
    "https://www.bing.com/news"
   ],
+  "bing videos":[
+   "جستجوی هوشمند Bing یافتن آنچه را که به دنبالش هستید آسان‌تر می‌کند و به شما پاداش می‌دهد.",
+   "https://www.bing.com/videos"
+  ],
   "crossref":"کراس‌رف یک موسسهٔ ثبت نشانگر دیجیتالی شیء (DOI) و متعلق به موسسه بین‌المللی DOI است. این موسسه در سال ۲۰۰۰ به عنوان تلاشی مشترک میان ناشران شروع به کار کرد تا قابلیت ارجاع دهی دائمی میان ناشران مختلف در نشریات الکترونیکی فراهم شود.",
   "currency":[
    "داکداکگو موتور جستجوی وب",
@@ -1911,6 +1937,10 @@
    "ref"
   ],
   "apple maps":"اپل مپس یا نقشه اپل یک سرویس نقشه‌برداری وب توسعه یافته توسط شرکت اپل است. این سرویس به‌طور پیش فرض بر روی آی‌اواس، مک اواس و واچ اواس در دسترس است. اپل مپس اطلاعاتی از قبیل جهت و زمان تخمینی رسیدن به مقصد برای خودرو، عابر پیاده و ناوبری حمل و نقل عمومی را برای کاربر فراهم می‌کند. همچنین اپل مپس دارای یک نمای منحصر به فرد به نام نمای بالا است که کاربر را قادر می‌سازد تا در یک نمای سه‌بعدی (3D) از بالا در مکان‌های مختلف به کاوش بپردازد که در این نما می‌توان ساختمانها و سازهها را تماشا کرد.",
+  "etymonline":[
+   "دیکشنری ریشه شناسی انگلیسی آنلاین",
+   "wikidata"
+  ],
   "fdroid":"اف-دروید مخزن نرم‌افزاری برای برنامه‌های اندروید است، مشابه گوگل پلی کار می‌کند ولی فقط شامل نرم‌افزارهای آزاد و متن‌باز است. برنامه‌های اف-دروید می‌توانند از طریق وب‌گاه اف-دروید یا از برنامهٔ کلاینت اف-دروید نصب شوند.",
   "flickr":"فلیکر یکی از بزرگ‌ترین سایت‌های اشتراک‌گذاری تصویر و ویدئو، خدمات وب و جوامع آنلاین است که توسط شرکت ludicorp در سال ۲۰۰۴ ایجاد شد و در سال ۲۰۰۵ توسط یاهو خریداری شد.",
   "genius":"جینیس به معنی نابغه، یک پایگاه دانش آنلاین است. این سایت به کاربر اجازه می‌دهد تا ترانه‌ها، داستان‌های جدید، اشعار، اسناد و دیگر متن‌ها را تفسیر کند. این سایت در ۲۰۰۹، با نام رپ جینیس با تمرکز روی موسیقی رپ راه اندازی شد. در ۲۰۱۴ سبک‌های دیگری مثل پاپ، آر اند بی و نوشتجات ادبی را نیز پوشش داد. همان سال یک برنامه آیفون برای سایت منتشر شد. برای انعکاس بهتر اهداف جدید سایت، دوباره در ژوئیه ۲۰۱۴ با نام جینیس راه اندازی شد. همچنین نسخه اندروید برنامه در اوت ۲۰۱۵ منتشر شد.",
@@ -1927,7 +1957,7 @@
   "google news":"گوگل نیوز به معنای اخبار گوگل، یک وبگاه جمع‌آوری‌کنندهٔ خبرخوان توسط گوگل است. این وبگاه در سپتامبر ۲۰۰۲ تأسیس‌شد و ۲۸ زبان زندهٔ جهان را پشتیبانی می‌کند. اما زبان فارسی در این ۲۸ زبان قرارندارد.",
   "google videos":"گوگل ویدئو یک سرویس برای بارگذاری و به اشتراک گذاشتن ویدئو بود. این سرویس امکان فروش ویدئوها را هم فراهم می‌کرد. این سرویس در ۲۵ ژانویه ۲۰۰۵ ایجاد شد و در اواخر سال ۲۰۰۹ به فعالیت خود پایان داد. علت این کار هم‌پوشانی سرویس با یوتیوب و صرفه‌جویی در فضای سرورها بود.",
   "google scholar":"گوگل اسکالر جستجوگری از شرکت گوگل است که امکان جستجوی واژه‌های کلیدی در مقاله‌ها، رساله‌های علمی و گزارش‌های فنی را فراهم می‌کند.",
-  "google play apps":"گوگل پلی یک سرویس پخش دیجیتال محتوای چندرسانه‌ای از شرکت گوگل است که شامل یک فروشگاه آنلاین برای موسیقی، فیلم، کتاب و اپلیکیشن‌ها و بازی‌های اندروید و یک مدیا پلیر ابری می‌باشد. این خدمات از طریق وب، اپلیکیشن موبایل Play Store در آندوید و گوگل تی‌وی در دسترس می‌باشد. خریداری محتوا برای تمام پلتفرم‌ها/دستگاه‌ها قابل دسترس است. گوگل پلی در ماه مارس ۲۰۱۲ معرفی شد، زمانی که گوگل، خدمات دو برند سابق اندروید مارکت و گوگل موزیک را ادغام کرد و برند جدید گوگل پلی را معرفی کرد.",
+  "google play apps":"گوگل پلی که با عنوان گوگل پلی استور یا پلی استور نیز شناخته می‌شود، یک سرویس پخش دیجیتال محتوای چندرسانه‌ای از شرکت گوگل است که شامل یک فروشگاه آنلاین برای موسیقی، فیلم، کتاب و اپلیکیشن‌ها و بازی‌های اندروید و یک مدیا پلیر ابری می‌باشد. این خدمات از طریق وب، اپلیکیشن موبایل Play Store در آندوید و گوگل تی‌وی در دسترس می‌باشد. خریداری محتوا برای تمام پلتفرم‌ها/دستگاه‌ها قابل دسترس است. گوگل پلی در ماه مارس ۲۰۱۲ معرفی شد، زمانی که گوگل، خدمات دو برند سابق اندروید مارکت و گوگل موزیک را ادغام کرد و برند جدید گوگل پلی را معرفی کرد.",
   "google play movies":[
    "google play apps:fa-IR",
    "ref"
@@ -1984,7 +2014,7 @@
   "peertube":"پیرتیوب یک سکوی ویدیوی آزاد، غیر متمرکز و فِدِرِیتِد بر پایهٔ اکتیویتی‌پاب و وب‌تورنت است که از فناوری همتابه‌همتا برای کاهش بار بر روی سرورها هنگام دیدن ویدیو استفاده می‌کند. توسعه این نرم‌افزار در سال ۲۰۱۵ توسط برنامه‌نویسی معروف به Chocobozzz آغاز شد و هم‌اکنون توسط مؤسسه غیرانتفاعی فرانسوی فراماسافت به پیش می‌رود. هدف این پروژه، ارائه جایگزین برای سکوهای متمرکز مانند یوتیوب، ویمیو و دیلی موشن است.",
   "rumble":"رامبل یک پلتفرم سرویس اشتراک ویدئو و ارائه دهنده خدمات رایانش ابری برای کسب‌وکارهاست. دفتر اصلی رامبل در شهر تورنتو، مرکز استان انتاریو کشور کانادا قرار دارد و دفتر مرکزی آن در آمریکا هم در شهرک لانگبوت کی ایالت فلوریدا هست. رامبل در اکتبر سال ۲۰۱۳ توسط کریس پاولوفسکی کانادایی، کارآفرین حوزه تکنولوژی تاسیس شد. بخش سرویس خدمات ابری رامبل هاست (میزبان) شبکه اجتماعی تروث سوشال است و بخش پلتفرم ویدئویی رامبل هم بین کاربران حزب محافظه‌کار و گروه‌های راست افراطی آمریکا محبوب است. پلتفرم رامبل به‌عنوان بخشی از تکنولوژی آلترناتیو (alt-tech) شناخته می‌شود.",
   "wttr.in":[
-   "Des Moines, Iowa, United States اوه و بآ تیعضو شرازگ",
+   "not found اوه و بآ تیعضو شرازگ",
    "https://wttr.in"
   ],
   "brave":"بریو سرچ یک موتور جستجو است که توسط بریو سافتور اینک Brave Software, Inc. ساخت و توسعه یافته‌است و به عنوان موتور جستجوی پیش فرض برای کاربران مرورگر بریو است."
@@ -2012,7 +2042,7 @@
   ],
   "bitbucket":"Bitbucket on lähdekoodin hallinnointiin ja versiohallintaan tarkoitettu sivusto.",
   "currency":"DuckDuckGo on hakukone, joka painottaa yksityisyyttä eikä kerää dataa käyttäjiltä. Se käyttää hakuihinsa oman hakurobottinsa lisäksi useita eri lähteitä, muun muassa joukkoutettuja verkkosivuja, kuten Wikipediaa.",
-  "deezer":"Deezer on ranskalainen musiikkipalvelu, jossa käyttäjät voivat kuunnella musiikkia suoratoistona Internetistä. Deezer julkaistiin 24. elokuuta 2007. Palvelua kustannetaan mainostuloilla. Toukokuussa 2013 Deezerissä oli 20 miljoonaa kappaletta. Palvelun käyttö ei vaadi erillistä ohjelmaa, vaan sivustolla olevia kappaleita pystyy kuuntelemaan suoraan verkkosivuston kautta. Palvelu on saatavissa myös älypuhelimille erillisen sovelluksen kautta. Palvelu avattiin Suomessa toukokuussa 2013. Deezer on Spotifyn kilpailija. Deezerissä on tarjolla yli 53 miljoonaa kappaletta.",
+  "deezer":"Deezer on ranskalainen musiikkipalvelu, jossa käyttäjät voivat kuunnella musiikkia suoratoistona Internetistä. Deezer julkaistiin 24. elokuuta 2007. Palvelua kustannetaan mainostuloilla. Toukokuussa 2013 Deezerissä oli 20 miljoonaa kappaletta. Palvelun käyttö ei vaadi erillistä ohjelmaa, vaan sivustolla olevia kappaleita pystyy kuuntelemaan suoraan verkkosivuston kautta. Palvelu on saatavissa myös älypuhelimille erillisen sovelluksen kautta. Palvelu avattiin Suomessa toukokuussa 2013. Deezerissä on tarjolla yli 53 miljoonaa kappaletta.milloin?",
   "deviantart":"DeviantArt on suosittu Internetissä toimiva taidesivusto. Sen ovat perustaneet elokuussa 2000 Angelo Sotira, Scott Jarkoff ja Matthew Stephens. DeviantArtiin rekisteröityneet käyttäjät voivat julkaista omia ja kommentoida muiden valokuvia, piirroksia, tatuointeja, maalauksia, Flash-animaatioita, runoja ja proosaa. Sivuston kautta voi myös myydä omaa taidettaan.",
   "ddg definitions":[
    "currency:fi",
@@ -2088,7 +2118,7 @@
   "wikivoyage":"Wikimatkat on internetissä oleva matkaopas, jota muokkaavat vapaaehtoiset käyttäjät. Tällä hetkellä se on saatavissa 21 eri kielellä.",
   "wolframalpha":"Wolfram Alpha on Wolfram Researchin kehittelemä haku- ja vastauskone, joka julkaistiin 15.5.2009. Wolfram Alphan toiminta pohjautuu Wolfram Researchin kehittelemään Mathematica-laskentaohjelmaan tiedon etsimisessä ja käsittelyssä.",
   "wttr.in":[
-   "Säätiedotus: Des Moines, Iowa, United States",
+   "Säätiedotus: not found",
    "https://wttr.in"
   ]
  },
@@ -2103,6 +2133,10 @@
    "Balita mula sa buong daigdig, bansa, at lokal, organisado para bigyan ka ng malawakang coverage ng sports, entertainment, negosyo, pulitika, panahon, at marami pang iba.",
    "https://www.bing.com/news"
   ],
+  "bing videos":[
+   "Pinapadali ng matalinong paghahanap mula sa Bing na mabilis na mahanap ang iyong hinahanap at binibigyan ka ng reward.",
+   "https://www.bing.com/videos"
+  ],
   "wikidata":"Ang Wikidata ay isang internet na wiki na pagkalagay ng datos sa mga wikang pag-aari ng Pundasyong Wikimedia.",
   "gentoo":[
    "gentoo:ru",
@@ -2143,7 +2177,7 @@
   "artic":"L'Art Institute of Chicago est un musée situé à Chicago aux États-Unis. Deuxième plus grand musée d'art du pays après le Metropolitan Museum of Art de New York, il abrite l'une des plus importantes collections d'art des États-Unis. Ouvert au public depuis 1879, le musée fut établi initialement à l'angle sud-ouest de State Street et de Monroe Street. Lors de l'Exposition universelle de 1893, la ville construisit un nouveau bâtiment pour abriter les collections de l'Art Institute. Depuis 1893, le musée se trouve dans le Grant Park, au 111 South Michigan Avenue, juste en face du Symphony Center, dans le centre-ville de Chicago.",
   "arxiv":"arXiv est une archive ouverte de prépublications électroniques d'articles scientifiques dans les domaines de la physique, des mathématiques, de l'informatique, de la biologie quantitative, de la finance quantitative, de la statistique, de l'ingénierie électrique et des systèmes, et de l'économie, et qui est accessible gratuitement par Internet.",
   "bandcamp":"Bandcamp est un magasin de musique en ligne qui s'adresse principalement aux artistes indépendants.",
-  "wikipedia":"Wikipédia est une encyclopédie universelle et multilingue créée par Jimmy Wales et Larry Sanger le 15 janvier 2001. Il s'agit d'une œuvre libre, c'est-à-dire que chacun est libre de la rediffuser. Gérée en wiki dans le site web wikipedia.org grâce au logiciel MediaWiki, elle permet à tous les internautes d'écrire et de modifier des articles, ce qui lui vaut d'être qualifiée d'encyclopédie participative. Elle est devenue en quelques années l'encyclopédie la plus fournie et la plus consultée au monde.",
+  "wikipedia":"Wikipédia est une encyclopédie collaborative, généraliste et multilingue créée par Jimmy Wales et Larry Sanger le 15 janvier 2001. Il s'agit d'une œuvre libre, c'est-à-dire que chacun est libre de la rediffuser. Gérée en wiki dans le site web wikipedia.org grâce au logiciel MediaWiki, elle permet à tous les internautes d'écrire et de modifier des articles, ce qui lui vaut d'être qualifiée d'encyclopédie participative. Elle est devenue en quelques années l'encyclopédie la plus fournie et la plus consultée au monde.",
   "bing":"Microsoft Bing, est un moteur de recherche élaboré par la société Microsoft. Il a été rendu public le 3 juin 2009.",
   "bing images":[
    "bing:fr",
@@ -2167,7 +2201,7 @@
    "ref"
   ],
   "erowid":"Erowid, aussi appelé Erowid Center, est une organisation américaine à but non lucratif qui a pour but de fournir des informations sur les psychotropes naturels ou synthétiques, qu'ils soient légaux ou non, mais aussi sur les méthodes et pratiques censées permettre l'accès aux états modifiés de conscience comme la méditation ou le rêve lucide.",
-  "wikidata":"Wikidata est une base de connaissances libre éditée de manière collaborative et hébergée par la Fondation Wikimédia. Son contenu étant placé sous licence CC0, elle permet de centraliser les données utilisées par différents projets Wikimedia. Cette base est destinée à fournir une source commune de données objectives, telles que les dates de naissance ou bien le PIB des pays, qui pourront être utilisées dans tous les articles des différentes versions linguistiques de Wikipédia, une mise à jour de Wikidata pouvant être alors répercutée automatiquement sur l'ensemble des Wikipédias en différentes langues.",
+  "wikidata":"Wikidata est une base de connaissances librement améliorable, conçue pour centraliser les données utilisées par les différents projets du mouvement Wikimédia. Une mise à jour d'une fiche Wikidata se répercute automatiquement sur toutes les pages de projets Wikimédia qui y font appel. Plus largement, Wikidata est destiné à fournir une source commune de données objectives, telles que les dates de naissance de personnalités ou le produit intérieur brut des pays.",
   "duckduckgo":[
    "currency:fr",
    "ref"
@@ -2213,12 +2247,12 @@
   "library of congress":"La bibliothèque du Congrès, située à Washington, assure la fonction de bibliothèque de recherche du Congrès des États-Unis et, de facto, constitue la bibliothèque nationale américaine.",
   "metacpan":"Le Comprehensive Perl Archive Network, ou CPAN, est un site Web consacré au langage de programmation Perl. CPAN désigne également un module Perl servant à accéder à ce site. Son nom vient du Comprehensive TeX Archive Network, ou CTAN, son homologue consacré à TeX.",
   "mixcloud":"Mixcloud est une plate-forme collaborative de partage et d'écoute de musique en ligne spécialement dédiée aux sessions de mixage enregistrées en studio diffusées en radio ou en podcast. The Guardian et TED utilisent la plate-forme.",
-  "npm":"npm est le gestionnaire de paquets par défaut pour l'environnement d'exécution JavaScript Node.js de Node.js.",
+  "npm":"npm est le gestionnaire de paquets par défaut pour l'environnement d'exécution JavaScript Node.js.",
   "openstreetmap":"OpenStreetMap (OSM) est un projet collaboratif de cartographie en ligne qui vise à constituer une base de données géographiques libre du monde, en utilisant le système GPS et d'autres données libres. Il est mis en route en juillet 2004 par Steve Coast à l'University College de Londres.",
   "piratebay":"The Pirate Bay est un site web créé en 2003 en Suède, indexant des liens Magnets de fichiers numériques, permettant le partage de fichiers en pair à pair à l’aide du protocole de communication BitTorrent. Le site se finance par les dons et la publicité, il a été créé dans l’esprit d’une « culture libre ».",
   "pubmed":"MEDLINE est une base de données bibliographiques regroupant la littérature relative aux sciences biologiques et biomédicales. La base est gérée et mise à jour par la Bibliothèque américaine de médecine (NLM).",
   "pypi":"PyPI est le dépôt tiers officiel du langage de programmation Python. Son objectif est de doter la communauté des développeurs Python d'un catalogue complet recensant tous les paquets Python libres. Il est analogue au dépôt CPAN pour Perl.",
-  "qwant":"Qwant est un moteur de recherche français mis en ligne en février 2013. Qwant prétend garantir la vie privée de ses utilisateurs car il ne les trace pas à des fins publicitaires, ni ne revend de données personnelles. En outre le moteur se veut impartial dans l'affichage des résultats.",
+  "qwant":"Qwant est un moteur de recherche français mis en ligne en février 2013. Qwant annonce garantir la vie privée de ses utilisateurs en évitant de les tracer à des fins publicitaires et de revendre leurs données personnelles, et à être impartial dans l'affichage des résultats.",
   "qwant news":[
    "qwant:fr",
    "ref"
@@ -2251,23 +2285,23 @@
   "unsplash":"Unsplash est un site web dédié au partage de photos sous licence Unsplash. Il est basé à Montréal, la capitale économique du Québec.",
   "yahoo news":"Yahoo! Actualités est un service en ligne gratuit de Yahoo! qui présente des articles d'information en provenance de nombreuses sources et, notamment, d'agences de presse.",
   "youtube":"YouTube est un site web d'hébergement de vidéos et média social sur lequel les utilisateurs peuvent envoyer, regarder, commenter, évaluer et partager des vidéos en streaming. Il est créé en février 2005 par Steve Chen, Chad Hurley et Jawed Karim, trois anciens employés de PayPal, puis racheté par Google en octobre 2006 pour 1,65 milliard de dollars. Le service est situé à San Bruno, en Californie.",
-  "dailymotion":"Dailymotion est une entreprise française proposant, sur le site web du même nom, un service d'hébergement, de partage et de visionnage de vidéo en ligne.",
+  "dailymotion":"Dailymotion est une entreprise française, filiale du groupe Vivendi, proposant, sur le site web du même nom, un service d'hébergement, de partage et de visionnage de vidéo en ligne.",
   "vimeo":"Vimeo est un site web communautaire destiné au partage et au visionnage de vidéos réalisées par les utilisateurs. Ce site a été lancé en novembre 2004. Vimeo est une filiale du groupe américain IAC (InterActiveCorp).",
-  "wikibooks":"Wikibooks est un environnement d'apprentissage personnel (EAP) multilingue, géré en wiki grâce au moteur MediaWiki. Il consiste en un ensemble de manuels pédagogiques et techniques. Comme Wikipédia, il appartient à Wikimedia Foundation, Inc. et son contenu, librement améliorable, est protégé par la licence Creative Commons « Attribution - Partage dans les Mêmes Conditions ». En français, le nom Wikilivres est parfois utilisé.",
+  "wikibooks":"Wikibooks est un site web multilingue dont l'objectif est de créer et d'améliorer des ouvrages techniques et pédagogiques. Il est géré en wiki grâce au moteur MediaWiki.",
   "wikinews":"Wikinews est un site d'actualité multilingue, géré en wiki grâce au moteur MediaWiki. Comme Wikipédia, il appartient à Wikimedia Foundation, Inc. et son contenu, librement améliorable, est protégé par la licence Creative Commons « Attribution - Partage dans les Mêmes Conditions ».",
-  "wikiquote":"Wikiquote est un site multilingue proposant des recueils de citations, géré en wiki grâce au moteur MediaWiki. Comme Wikipédia, il appartient à Wikimedia Foundation, Inc. et son contenu, librement améliorable, est protégé par la licence Creative Commons « Attribution - Partage dans les Mêmes Conditions ».",
-  "wikisource":"Wikisource est une bibliothèque numérique de textes du domaine public, gérée en wiki grâce au moteur MediaWiki. Comme Wikipédia, elle appartient à Wikimedia Foundation, Inc.",
+  "wikiquote":"Wikiquote est un site web multilingue dont l'objectif est de créer et d'améliorer des recueils de citations. Il est géré en wiki grâce au moteur MediaWiki.",
+  "wikisource":"Wikisource est une bibliothèque numérique de textes du domaine public, gérée en wiki grâce au moteur MediaWiki. Comme Wikipédia, elle est hébergée par la fondation Wikimédia et son contenu est librement améliorable.",
   "wiktionary":"Le Wiktionnaire est un projet lexicographique de la Wikimedia Foundation dont l’objectif est de définir tous les mots de toutes les langues, dans toutes les langues. Il existe plus de 150 langues de rédaction. Le terme « Wiktionnaire » désigne la version en français de ce projet, Wiktionary étant le nom officiel en anglais. Il est géré en wiki dans le site web wiktionary.org et son contenu est librement réutilisable.",
-  "wikiversity":"Wikiversité est une communauté d'apprentissage et de recherche multilingue et ouverte à tous, gérée en wiki grâce au moteur MediaWiki. Comme Wikipédia, elle appartient à Wikimedia Foundation, Inc. et le contenu qu'elle produit, librement améliorable, est protégé par la licence Creative Commons « Attribution - Partage dans les Mêmes Conditions ».",
-  "wikivoyage":"Wikivoyage est un guide touristique multilingue, géré en wiki grâce au moteur MediaWiki. Depuis 2012, il appartient à Wikimedia Foundation, Inc. et son contenu, librement améliorable, est protégé par la licence Creative Commons « Attribution - Partage dans les Mêmes Conditions ».",
+  "wikiversity":"Wikiversité est une communauté d'apprentissage multilingue et ouverte à tous, gérée en wiki grâce au moteur MediaWiki.",
+  "wikivoyage":"Wikivoyage est un site web multilingue dont l'objectif est de créer et d'améliorer des guides touristiques. Il est géré en wiki grâce au moteur MediaWiki.",
   "wolframalpha":"Wolfram|Alpha est un outil de calcul en langage naturel développé par la société internationale Wolfram Research. Il s'agit d'un service internet qui répond directement à la saisie de questions factuelles en anglais par le calcul de la réponse à partir d'une base de données, au lieu de procurer une liste de documents ou de pages web pouvant contenir la réponse. Son lancement a été annoncé en mars 2009 par le physicien et mathématicien britannique Stephen Wolfram et il a été lancé le 16 mai 2009 à 3 h du matin.",
   "seznam":"Seznam est un portail web et un moteur de recherche tchèque. Il a été lancé en 1996. En novembre 2020, c'est le troisième site le plus visité en République tchèque.",
   "naver":"Naver est une plateforme en ligne sud-coréenne gérée par la société Naver Corporation. Le site a été créé en 1999 en tant que premier portail Web en Corée à développer et utiliser son propre moteur de recherche. Il a également été le premier opérateur au monde à introduire la fonction de recherche intégrée, qui compile les résultats de recherche de différentes catégories et les présente sur une seule page. Depuis, Naver a ajouté une multitude de nouveaux services telles que le courrier électronique et les nouvelles, puis aussi la première plateforme de questions-réponses en ligne Knowledge iN.",
   "rubygems":"RubyGems est un gestionnaire de paquets pour le langage de programmation Ruby qui fournit un format standard pour la distribution de programmes et de bibliothèques Ruby. Il permet de gérer facilement l'installation de gemmes et d'un serveur pour les distribuer. Il a été créé par Chad Fowler, Jim Weirich, David Alan Black, Paul Brannan et Richard Kilmer lors de la RubyConf 2004.",
   "peertube":"PeerTube est un logiciel libre d'hébergement de vidéo décentralisé permettant la diffusion en pair à pair, et un média social sur lequel les utilisateurs peuvent envoyer, regarder, commenter, évaluer et partager des vidéos en streaming. Il est créé en 2015 et est développé depuis 2017 par Framasoft. Il fonctionne sur le principe d'une fédération d'instances hébergées par des entités autonomes. Son objectif est de fournir une alternative aux plateformes centralisées telles que YouTube, Vimeo et plus récemment Twitch avec l'ajout du support de la diffusion en direct.",
-  "wikimini":"Wikimini est une encyclopédie en ligne écrite en grande partie par des enfants de 7 à 14 ans. Œuvre libre, elle est gérée en wiki grâce au moteur MediaWiki auquel a été ajoutée une interface plus colorée. Elle est écrite principalement en français, mais aussi en suédois.",
+  "wikimini":"Wikimini est un site web de médiation scientifique, en langue contrôlée, écrit en grande partie par des enfants de 7 à 14 ans. Œuvre libre, elle est gérée en wiki grâce au moteur MediaWiki auquel a été ajoutée une interface plus colorée. Elle est écrite principalement en français, mais aussi en suédois.",
   "wttr.in":[
-   "Prévisions météo pour: Des Moines, Iowa, United States",
+   "Prévisions météo pour: not found",
    "https://wttr.in"
   ],
   "brave":"Le moteur de recherche Brave, également appelé Brave Search, est un moteur de recherche créé par Brave Software en 2021 pour devenir une alternative aux géants du Web.",
@@ -2385,7 +2419,7 @@
    "https://www.bing.com/videos"
   ],
   "currency":"דקדקגו הוא מנוע חיפוש שמדגיש את הגנת פרטיות המשתמש ונמנע מיצירת \"בועת פילטר\" שמנחשת את אופי החיפושים הרלוונטיים למשתמש. דקדקגו נבדל ממנועי חיפוש אחרים בכך שהוא לא מתחקה אחר תוצאות המשתמשים, כמו גם, מאחזר את אותן תוצאות לכל המשתמשים שחיפשו מושג זהה ואינו נותן תוקף לשיקולים זרים בתוצאות החיפוש. יתרה מכך, דקדקגו מעדיף לאחזר מידע ממעט מקורות מידע איכותיים מאשר מהרבה מקורות מידע שאינם איכותיים. תוצאות החיפוש של דקדקגו הן קומפילציה של \"בערך 50\" מקורות מידע (duck.co). בין היתר, הוא מאחזר מידע מאתרי \"מיקור המונים\" כמו ויקיפדיה, ממנועי חיפוש אחרים כמו: Yandex, Yahoo!, Bing ו-Yummly ומזחלן הרשת שלו עצמו, דקדקבוט.",
-  "deezer":"דיזר הוא שירות הזרמת מוזיקה מבוסס אינטרנט. השירות מאפשר למשתמשים להאזין לתכנים מוזיקליים המפורסמים תחת חברות תקליטים שונות, ביניהן EMI, סוני, וורנר מיוזיק גרופ ויוניברסל מיוזיק גרופ, על מכשירים שונים באופן מקוון או לא מקוון. דיזר נוצר בפריז, צרפת על ידי דניאל מרלי בשנת 2007. השירות פועל בכ-185 מדינות ומחזיק ברישיון להזרמת כ-90 מיליון שירים בספרייתו. השירות מספק מעל ל-30,000 ערוצי רדיו, ורשומים אליו כ-20 מיליון משתמשים פעילים חודשיים, וכ-9 מיליון מנויים בתשלום, נכון לאפריל 2022. השירות, בנוסף לאתר האינטרנט, זמין עבור אנדרואיד, Kindle Fire HDX HDX, OS X, בלקברי, iOS, Windows Phone וסימביאן.",
+  "deezer":"דיזר הוא שירות הזרמת מוזיקה מבוסס אינטרנט. השירות מאפשר למשתמשים להאזין לתכנים מוזיקליים המפורסמים תחת חברות תקליטים שונות, ביניהן EMI, סוני, וורנר מיוזיק גרופ ויוניברסל מיוזיק גרופ, על מכשירים שונים באופן מקוון או לא מקוון. דיזר נוצר בפריז, צרפת על ידי דניאל מרלי וג'ונתן בנסאיה בשנת 2007. השירות פועל בכ-185 מדינות ומחזיק ברישיון להזרמת כ-90 מיליון שירים בספרייתו. השירות מספק מעל ל-30,000 ערוצי רדיו, ורשומים אליו כ-20 מיליון משתמשים פעילים חודשיים, וכ-9 מיליון מנויים בתשלום, נכון לאפריל 2022. השירות, בנוסף לאתר האינטרנט, זמין עבור אנדרואיד, Kindle Fire HDX HDX, OS X, בלקברי, iOS, Windows Phone וסימביאן.",
   "deviantart":"דיוויינטארט הוא שירות קהילתי לשיתוף תמונות ויצירות אומנות באינטרנט. החברה הוקמה ב-7 באוגוסט 2000. מטה החברה נמצא באזור הוליווד שבלוס אנג'לס, קליפורניה, ארצות הברית. בפברואר 2017 החברה נרכשה על ידי החברה הישראלית \"Wix\" תמורת 36 מיליון דולר.",
   "ddg definitions":[
    "currency:he",
@@ -2453,7 +2487,7 @@
   ],
   "rumble":"ראמבל היא פלטפורמת וידאו מקוונת קנדית, שנוסדה בשנת 2013. האתר הוקם על ידי כריס פבלובסקי, יזם טכנולוגי קנדי. האתר פופולרי בקרב יוצרי תוכן שמרניים.",
   "wttr.in":[
-   "Des Moines, Iowa, United States :ריוואה גזמ תיזחת",
+   "not found :ריוואה גזמ תיזחת",
    "https://wttr.in"
   ]
  },
@@ -2607,7 +2641,7 @@
   "wikivoyage":"A Wikivoyage a Wikimédia Alapítvány ingyenes internetes útikönyve. A neve a Wiki és a francia voyage szóból áll. Az új wikiprojekt 2013. január 15-én, a Wikipédia alapításának 12. évfordulója évében, napra a születésnapján startolt. 24 nyelven érhető el: angolul, németül, hollandul, oroszul, svédül, olaszul, portugálul, franciául, spanyolul, kínaiul, finnül, görögül, héberül, perzsául, lengyelül, románul, ukránul, vietnámiul, törökül, japánul, hindiül, pastuul, bengáliul és eszperantóul.",
   "naver":"A Naver dél-koreai internetes portál és keresőmotor, melyet 1999-ben hozott létre egy korábbi Samsung-alkalmazott. A Naver saját keresőmotort fejlesztett, ami kifejezetten koreai nyelvű tartalomra specializálódik. 2009-ben a keresőmotorok között az ötödik helyen szerepelt a világon, a Google, a Yahoo!, a Baidu és a Microsoft után. A Naver a koreai piac domináns keresője, a keresések mintegy 70%-át itt bonyolítják és mintegy 25 millió felhasználónak ez a kezdőoldala a böngészőben. A Woori Investment and Securities elemzése szerint a Google-nek azért nem sikerült megvetnie a lábát a koreai piacon a Naverrel szemben, mert túl kevés koreai nyelvű tartalmat szolgáltat.",
   "wttr.in":[
-   "Időjárás előrejelzés: Des Moines, Iowa, United States",
+   "Időjárás előrejelzés: not found",
    "https://wttr.in"
   ],
   "goo":"A goo egy japán internetes keresőmotor és webportál, amely összegyűjti és indexeli a japán nyelvű weboldalakat. A goot a japán NTT Resonant, az NTT Communications egyik leányvállalata működteti."
@@ -2643,7 +2677,7 @@
    "https://www.bing.com/news"
   ],
   "bing videos":[
-   "Carian pintar dari Bing memudahkan anda untuk menemui dengan segera apa yang anda cari dan memberi ganjaran kepada anda.",
+   "Pencarian cerdas dari Bing mempermudah Anda menemukan apa yang Anda cari dengan cepat dan memberikan hadiah.",
    "https://www.bing.com/videos"
   ],
   "bitbucket":"Bitbucket adalah sebuah layanan hosting yang berbasis web untuk kode sumber dan pembangunan proyek yang menggunakan Mercurial ataupun sistem kendali versi Git yang dimiliki oleh Atlassian. Bitbucket menawarkan paket akun komersial dan gratis. Akun gratis tersebut menawarkan sebuah layanan repositori dengan jumlah yang tidak terbatas sejak bulan September 2010. Bitbucket terintegrasi dengan perangkat lunak Atlassian lain seperti Jira, HipChat, Confluence dan Bamboo.",
@@ -2816,10 +2850,6 @@
   "hoogle":"Haskell è un linguaggio di programmazione puramente funzionale general-purpose creato da un apposito comitato alla fine degli anni ottanta principalmente per analizzare le caratteristiche dei linguaggi. È stato chiamato così in onore del matematico e logico statunitense Haskell Curry.",
   "imdb":"Internet Movie Database, comunemente indicato con l'acronimo IMDb, è un sito web di proprietà di Amazon.com che gestisce informazioni su film, attori, registi, personale di produzione, programmi televisivi, e anche videogiochi. Una versione a pagamento denominata IMDb Pro è disponibile per tutti coloro che pubblicizzano i propri lavori.",
   "ina":"L'Institut national de l'audiovisuel (INA), è un ente pubblico commerciale francese, incaricato di archiviare tutte le trasmissioni radiofoniche e audiovisive del paese, come fa la Bibliothèque nationale de France con i documenti scritti.",
-  "invidious":[
-   "frontend libero per YouTube",
-   "wikidata"
-  ],
   "jisho":[
    "dizionario online giapponese-inglese",
    "wikidata"
@@ -2947,7 +2977,7 @@
   "google news":"Google ニュース は、Googleが提供するニュースアグリゲーター。Googleのページランクに関連して、Googleの主任研究者である Krishna Bharat が2001年に開発したストーリーランクをベースとして始まった。人間はアグリゲーションのアルゴリズムを調節するだけで、掲載する記事の選択は全て自動的に行われる。2006年1月、Google News ベータ版が登場した。",
   "google videos":"Google ビデオ は、Googleの動画検索エンジンである。かつては無料の動画共有サイトであり、YouTubeのように選択した動画をリモートのウェブページに埋め込むためのHTMLコードを提供し、帯域幅が狭くストレージ容量も少ないウェブサイトで動画を豊富に利用できるものだった。",
   "google scholar":"Google Scholar（グーグル・スカラー）は、ウェブ検索サイトのGoogleの提供する検索サービスの一つ。主に学術用途での検索を対象としており、論文、学術誌、出版物の全文やメタデータにアクセスできる。Googleはそのデータベースのサイズを公開していないが、第三者機関の調査によれば、2014年5月時点、約1.6億の文章が含まれると推定される。",
-  "google play apps":"Google Play は、Googleによって提供される、主にAndroid及びChrome OS（2016年4月以降）向けデジタルコンテンツ（アプリケーション・映画・音楽・書籍など）の配信サービス。2012年3月6日にGoogleは「Android Market」を「Google Play」に改名し、「Google Play ブックス」「Google Play Music」といったサービスも合わせて誕生した。",
+  "google play apps":"Google Play は、Googleによって提供される、主にAndroid及びChromeOS（2016年4月以降）向けデジタルコンテンツ（アプリケーション・映画・音楽・書籍など）の配信サービス。2012年3月6日にGoogleは「Android Market」を「Google Play」に改名し、「Google Play ブックス」「Google Play Music」といったサービスも合わせて誕生した。",
   "google play movies":[
    "google play apps:ja",
    "ref"
@@ -2957,7 +2987,7 @@
   "library genesis":"Library GenesisまたはLibGenは、様々なトピックスに関する論文や書籍のためのサーチエンジンであり、 有料で配布されていたり、どこにおいてもデジタル化されていなかったりするコンテンツを無料でアクセス可能にしている。 特に、エルゼビアのScienceDirectウェブポータルで配布されているPDFファイルを収録している。",
   "library of congress":"アメリカ議会図書館 は、アメリカ合衆国の事実上の国立図書館。",
   "metacpan":"CPAN とは、Perlのライブラリ・モジュールやその他のPerlで書かれたソフトウェアを集めた巨大なアーカイブで、世界中のサーバにその内容がミラーリングされている。再利用性・汎用性の高いモジュールが登録されており、Perlプログラマができるだけ車輪の再発明をせずに済むための支援環境となっている。登録モジュールの検索システムも提供されているため、Perlプログラマは望む機能を持ったモジュールを容易に入手することができる。",
-  "npm":"npmとはパッケージ管理システムの一種。Node Package Managerの意。なおnpmとは「Node Package Manager」の頭文字を取ったものではなく、実際はバクロニムである。",
+  "npm":"npmとはJavaScriptのパッケージ管理システムの一種。Node Package Managerの意。",
   "openstreetmap":"オープンストリートマップ は自由に利用でき、なおかつ編集機能のある世界地図を作る共同作業プロジェクトである。GPS機能を持った携帯端末、空中写真やほかの無料機械からのデータをもとに作られていくのが基本だが、編集ツール上で道1本から手入力での追加も可能である。与えられた画像とベクトルデータセットはオープンデータベースライセンス (ODbL) 1.0のもと再利用可能である。",
   "piratebay":"パイレート・ベイ は、デジタルコンテンツのトレントファイルを検索できるインデックスサイトである。2003年に、スウェーデンの反著作権団体Piratbyrånによって設立された。利用者はマグネットリンクおよびトレントファイルの検索、ダウンロード、掲載が可能である。マグネットリンクとトレントファイルは、BitTorrentプロトコルを用いたP2Pファイル共有に使用される。",
   "pubmed":"MEDLINE（メドライン）またはMEDLARS Online は、医学を中心とする生命科学の文献情報を収集したオンラインデータベースである。1964年に米国国立医学図書館 が作成したコンピューター化医学文献データベース「MEDLARS」は、1971年10月27日にオンライン検索サービスが開始され、1997年にはPubMedの名でインターネットに無料公開された後、改良が重ねられて成長を続け、2007年現在、月に7000万回程度のアクセスがある世界で最もよく使用される生物医学系データベースである。",
@@ -2974,7 +3004,7 @@
    "ref"
   ],
   "startpage":"Startpage とはニューヨークとオランダを拠点にしているメタ検索エンジンで、1998年にデビッド・ボドニックが設立し2000年にオランダのサーフボード・ホールディングBVが買収した。2015年2月2日にIxquickと姉妹プロジェクトのStartpage.comは（28日平均で）1日直接クエリ数が5,700万に達したとしている。",
-  "youtube":"YouTube（ユーチューブ）は、アメリカ合衆国カリフォルニア州サンブルーノに本社を置くオンライン動画共有プラットフォーム。アクティブユーザー数は、2022年1月時点で25億6,200万人（うち定額制サービス契約者数は8000万人以上）であり、SNSとしては世界第2位。2005年2月にPayPalの元従業員であるチャド・ハーリー、スティーブ・チェン、ジョード・カリムの3人によって設立された。その後、2006年11月に16.5億米ドルでGoogleに買収され、現在は同社の子会社の1つとして運営されている。アレクサ・インターネットランキングによると、Google 検索に次いで2番目にアクセス数の多いウェブサイトである。",
+  "youtube":"YouTube（ユーチューブ）は、アメリカ合衆国カリフォルニア州サンブルーノに本社を置くオンライン動画共有プラットフォーム。アクティブユーザー数は、2022年1月時点で25億6,200万人（うち定額制サービス契約者数は8000万人以上）であり、ソーシャルメディアとしては世界第2位。2005年2月にPayPalの元従業員であるチャド・ハーリー、スティーブ・チェン、ジョード・カリムの3人によって設立された。その後、2006年11月に16.5億米ドルでGoogleに買収され、現在は同社の子会社の1つとして運営されている。アレクサ・インターネットランキングによると、Google 検索に次いで2番目にアクセス数の多いウェブサイトである。",
   "dailymotion":"Dailymotion（デイリーモーション）は、Vivendi S.A.傘下のDAILYMOTION SAが運営する、フランスの動画共有サービス。",
   "vimeo":"Vimeo は、クリエイター向け動画共有サイト。「video」（ビデオ）と「me」（私）の意味と、「movie」（映画）という言葉のアナグラムである。",
   "wikisource":"ウィキソース (Wikisource) は、ウィキメディア財団が運営するウィキを利用した自由に利用できるテキストを集めた電子図書館である。ウィキソースはプロジェクトの名前でもあり、またプロジェクトのインスタンス（実体）である個々のサイト（主に各言語版）もウィキソースと呼ばれ、複数のウィキソースが集まって大きなウィキソースプロジェクトを形成している。ウィキソースの目的はあらゆる形態のフリーテキストを、多数の言語および翻訳においても提供することである。元々は有用または重要な歴史的文書を保存するアーカイブとして着想され、今では幅広いコンテンツを扱うライブラリとなっている。",
@@ -3017,7 +3047,7 @@
   ],
   "currency":"덕덕고(영어: DuckDuckGo DDG[*])는 사용자의 개인정보를 수집하지 않는 검색 엔진이다. 덕덕고 검색 엔진의 일부는 오픈 소스이고, 펄을 사용하며 서버는 NGINX로 운영된다. 회사는 미국의 펜실베이니아주에 있으며, 직원은 약 30명이다. 회사의 이름은 외국 게임인 덕, 덕, 구스에서 유래되었다.",
   "deezer":"Deezer는 프랑스 온라인 음악 스트리밍 서비스이다. 이를 통해 사용자는 유니버설 뮤직 그룹, 소니 뮤직 및 워너 뮤직 그룹을 비롯한 음반사의 음악 콘텐츠를 들을 수 있다. 온라인 또는 오프라인으로 다양한 기기에서 팟캐스트로 청취할 수 있다.",
-  "deviantart":"디비언트아트(영어: DeviantArt)는 온라인 커뮤니티 사이트로, 사람들이 직접 그린 그림을 올린다. 2000년 8월 7일 설립되었으며, 본사는 미국 캘리포니아주 로스앤젤레스 할리우드에 있다. 세계구급의 자유 창작 사이트이며, 창작품의 종류도 가지가지여서 단순한 팬 아트나 팬 픽션부터, 예술 사진, 봉제 인형, 짧은 애니메이션 등등이 매일매일 올라오는 것이 특징이다. 2011년 7월 기준으로 매주 380만 명의 방문자 수를 기록하면서 소셜 네트워크 사이트 중 13위를 기록했다. 2015년 12월을 기준으로 2,600만명 이상의 회원이 가입했고, 251,000,000개의 사진이 올라왔다.",
+  "deviantart":"디비언트아트(영어: DeviantArt)는 온라인 커뮤니티 사이트로, 사람들이 직접 그린 그림을 올린다. 2000년 8월 7일 설립되었으며, 본사는 미국 캘리포니아주 로스앤젤레스 할리우드에 있다. 세계구급의 자유 창작 사이트이며, 창작품의 종류도 가지가지여서 단순한 팬 아트나 팬 픽션부터, 예술 사진, 봉제 인형, 짧은 애니메이션 등등이 매일매일 올라오는 것이 특징이다. 2011년 7월 기준으로 매주 380만 명의 방문자 수를 기록하면서 소셜 네트워크 사이트 중 13위를 기록했다. 2015년 12월을 기준으로 2,600만명 이상의 회원이 가입했고, 251,000,000개의 사진이 올라왔다. 2022년 11월 11일 그림 인공지능 DreamUp을 도입하였으며, 회원들의 반발로 이용자 수가 감소하였다.",
   "ddg definitions":[
    "currency:ko",
    "ref"
@@ -3112,7 +3142,7 @@
    "wikidata"
   ],
   "wttr.in":[
-   "일기 예보： Des Moines, Iowa, United States",
+   "일기 예보： not found",
    "https://wttr.in"
   ]
  },
@@ -3174,7 +3204,7 @@
   "wikiversity":"Vikiversitetas – Vikimedijos fondo projektas, pagrįstas MediaWiki technologija; vikisvetainė.",
   "wikivoyage":"Vikikelionės – internetinis projektas, kuriamas vikitechnologija bei pagrįstas MediaWiki programine įranga. Vikikelionės nuo 2013 m. sausio 15 d. yra oficialus Vikimedijos projektas.",
   "wttr.in":[
-   "Orų prognozė: Des Moines, Iowa, United States",
+   "Orų prognozė: not found",
    "https://wttr.in"
   ]
  },
@@ -3252,7 +3282,11 @@
   "wikisource":"Wikisource ir viens no Wikimedia Foundation projektiem, kas ir balstīts uz wiki programmatūru. Tā ir tiešsaistes bibliotēka ar brīva satura informāciju. Projekts tika izveidots 2003. gada 24. novembrī. Sākotnējais projekta nosaukums bija Project Sourceberg. 2004. gada 23. jūlijā nosaukums tika mainīts uz pašreizējo.",
   "wiktionary":"Vikivārdnīca ir brīva papildināma daudzvalodu vārdnīca, kura izveidota uz wiki bāzes. Tas ir viens no Wikimedia Foundation projektiem.",
   "wikiversity":"Wikiversity ir viens no Wikimedia Foundation projektiem, kas ir balstīts uz wiki programmatūru. Tas ir tiešsaistes brīva satura portāls ar mācību materiāliem un dažādām pamācībām. Projekts tika uzsākts 2006. gada augustā. Pašlaik šis projekts pieejams angliski un vēl vairāk nekā 10 citās valodās.",
-  "wikivoyage":"Wikivoyage ir viens no Wikimedia Foundation projektiem, kas ir balstīts uz wiki programmatūru. Tas ir tiešsaistes brīva satura portāls ar tūrisma materiāliem. Dibināts 2006. gadā, Wikimedia Foundation to pārņēma 2012. gadā. Pašlaik šis projekts pieejams angliski un vēl vairāk nekā 10 citās valodās."
+  "wikivoyage":"Wikivoyage ir viens no Wikimedia Foundation projektiem, kas ir balstīts uz wiki programmatūru. Tas ir tiešsaistes brīva satura portāls ar tūrisma materiāliem. Dibināts 2006. gadā, Wikimedia Foundation to pārņēma 2012. gadā. Pašlaik šis projekts pieejams angliski un vēl vairāk nekā 10 citās valodās.",
+  "wttr.in":[
+   "Laika ziņas: not found",
+   "https://wttr.in"
+  ]
  },
  "ml":{
   "apple app store":"ആപ്പിൾ ഇൻക്. അതിന്റെ ഐഒഎസ്, ഐപാഡ്ഒഎസ്(iPadOS) ഓപ്പറേറ്റിംഗ് സിസ്റ്റങ്ങൾക്കുള്ള മൊബൈൽ ആപ്പുകൾക്കായി വികസിപ്പിച്ച് പരിപാലിക്കുന്ന ഒരു ആപ്പ് സ്റ്റോർ പ്ലാറ്റ്‌ഫോമാണ് ആപ്പ് സ്റ്റോർ. ആപ്പിളിന്റെ ഐഒഎസ് സോഫ്റ്റ്‌വെയർ ഡെവലപ്‌മെന്റ് കിറ്റിൽ വികസിപ്പിച്ച അംഗീകൃത ആപ്പുകൾ ബ്രൗസ് ചെയ്യാനും ഡൗൺലോഡ് ചെയ്യാനും സ്റ്റോർ ഉപയോക്താക്കളെ അനുവദിക്കുന്നു. ഐഫോൺ, ഐപോഡ് ടച്ച്(iPod Touch), അല്ലെങ്കിൽ ഐപാഡ് എന്നിവയിൽ ആപ്പുകൾ ഡൗൺലോഡ് ചെയ്യാം, ചിലത് ഐഫോൺ ആപ്പുകളുടെ എക്സ്റ്റൻഷനുകളായി ആപ്പിൾ സ്മാർട്ട് വാച്ചിലേക്കോ നാലാം തലമുറയിലേക്കോ പുതിയ ആപ്പിൾ ടിവിയിലേക്കോ മാറ്റാം.",
@@ -3328,7 +3362,7 @@
   "wikiversity":"വിക്കിമീഡിയ ഫൗണ്ടേഷന്റെ വിക്കി അധിഷ്ഠിത സം‌രംഭങ്ങളിൽ ഒന്നാണ് വിക്കിവേഴ്സിറ്റി.ഇവിടെ സ്വതന്ത്ര പഠന സാമഗ്രികൾ പ്രവർത്തനങ്ങളും നടത്തുന്ന ഒരു പദ്ധതിയാണിത്. വിക്കിപീഡിയ പോലുള്ള വിജ്ഞാനകോശങ്ങളിൽ നിന്നു് വിഭിന്നമായി ഇവിടെ ഒരേ വിഷയത്തിൽ അധിഷ്ഠിതമായ നിരവധി പഠനസാമഗ്രികൾ വിവിധ പതിപ്പുകളിലായി ലഭിക്കുന്നു.",
   "wikivoyage":"ഒരു വിക്കിമീഡിയ സംരംഭമാണ് വിക്കിപര്യടനം(en:wikivoyage). സ്വതന്ത്ര യാത്രാപുസ്തകമാണിത്. ലോകത്തെമ്പാടുമുള്ള പ്രധാന ടൂറിസ്റ്റ് കേന്ദ്രങ്ങളിൽ എത്തിപ്പെടുന്ന ഒരാൾക്ക് ആവശ്യമായ എല്ലാ സംഗതികളും ഉൾക്കൊള്ളിച്ചുകൊണ്ട് വെബ് അടിസ്ഥാനത്തിൽ സേവനം ലഭ്യമാക്കുക എന്നതാണ് ഇതിന്റെ ലക്ഷ്യം.",
   "wttr.in":[
-   "കാലാവസ്ഥ റിപ്പോർട്ട്: Des Moines, Iowa, United States",
+   "കാലാവസ്ഥ റിപ്പോർട്ട്: not found",
    "https://wttr.in"
   ]
  },
@@ -3621,10 +3655,6 @@
    "ina:en",
    "ref"
   ],
-  "invidious":[
-   "invidious:en",
-   "ref"
-  ],
   "jisho":[
    "jisho:en",
    "ref"
@@ -3959,6 +3989,10 @@
    "tineye:nl-BE",
    "ref"
   ],
+  "etymonline":[
+   "etymonline:nl-BE",
+   "ref"
+  ],
   "flickr":[
    "flickr:nl-BE",
    "ref"
@@ -4152,7 +4186,7 @@
    "https://rubygems.org/"
   ],
   "wttr.in":[
-   "Weerbericht voor: Des Moines, Iowa, United States",
+   "Weerbericht voor: not found",
    "https://wttr.in"
   ]
  },
@@ -4292,7 +4326,7 @@
   ],
   "startpage":"Startpage to holenderska wyszukiwarka internetowa, nastawiona na prywatność. Strona umożliwia uzyskiwanie wyników wyszukiwania Google, jednocześnie chroniąc prywatność użytkowników, nie przechowując danych osobowych ani danych wyszukiwania i usuwając wszystkie skrypty śledzące. Startpage.com posiada również funkcję \"Anonymous View\", która umożliwia użytkownikom otwieranie wyników wyszukiwania za pośrednictwem serwera proxy w celu zwiększenia anonimowości. Ponieważ firma ma siedzibę w Holandii, jest chroniona przez holenderskie i unijne przepisy dotyczące prywatności, a zatem nie podlega programom nadzoru prowadzonym przez Stany Zjednoczone, takim jak PRISM.",
   "unsplash":"Unsplash – społecznościowa platforma internetowa przeznaczona do udostępniania fotografii.",
-  "youtube":"YouTube – serwis internetowy założony w lutym 2005 roku, który umożliwia bezpłatne umieszczanie, nadawanie na żywo, ocenianie i komentowanie filmów. Prezesem od 2014 roku jest Susan Wojcicki.",
+  "youtube":"YouTube – serwis internetowy założony w lutym 2005 roku, który umożliwia bezpłatne umieszczanie, nadawanie na żywo i komentowanie filmów. Od 2023 roku prezesem jest Neal Mohan.",
   "dailymotion":"Dailymotion – serwis internetowy umożliwiający prezentację filmów, wideoklipów lub własnych mini produkcji w internecie, reklamujący się hasłem Regarder, publier, partager, założony w Paryżu we Francji, jako odpowiedź na serwis YouTube. Domena dailymotion.com została zarejestrowana miesiąc po YouTube. Dailymotion jest dostępny w 18 różnych językach i 35 zlokalizowanych wersjach.",
   "vimeo":"Vimeo – serwis internetowy umożliwiający oglądanie i udostępnianie plików filmowych przez użytkowników. Strona została założona w listopadzie 2004 przez Zacha Kleina oraz Jakoba Lodwicka, który również wymyślił nazwę, która stanowi grę słów opartą na wyrazach video i me, jako odniesienie do faktu udostępniania plików stworzonych wyłącznie przez użytkowników. Serwis został kupiony przez InterActiveCorp w sierpniu 2006 roku.",
   "wikibooks":"Wikibooks – jeden z projektów Wikimedia Foundation, uruchomiony 10 lipca 2003 r. Jest to projekt siostrzany Wikipedii, który ma na celu poszerzanie i rozpowszechnianie darmowych, otwartych do edycji materiałów edukacyjnych, takich jak podręczniki szkolne, akademickie, poradniki, instrukcje obsługi oraz im podobne. Podręczniki Wikibooks, tak samo jak artykuły Wikipedii, są dostępne na zasadach Licencji Wolnej Dokumentacji GNU oraz Licencji Creative Commons; uznanie autorstwa – na tych samych warunkach 3.0.",
@@ -4310,6 +4344,10 @@
    "Słownik Języka Polskiego PWN online",
    "wikidata"
   ],
+  "wttr.in":[
+   "Pogoda w: not found",
+   "https://wttr.in"
+  ],
   "petalsearch":"Petal Search – wyszukiwarka internetowa rozwijana przez Huawei od drugiej połowy 2020 roku. Nazwa Petal w języku angielskim oznacza płatek i nawiązuje do płatków kwiatu w logo Huawei. Serwis był notowany w rankingu Alexa na miejscu 40362.",
   "petalsearch images":[
    "petalsearch:pl",
@@ -4357,6 +4395,7 @@
   "apple maps":"Apple Maps é um serviço de pesquisa e visualização de mapas desenvolvido pela Apple Inc. É o aplicativo de mapas padrão dos sistemas macOS, iOS, iPadOS e watchOS que fornece instruções de navegação e rotas. O serviço foi lançado em 19 de setembro de 2012 juntamente com o iOS 6, substituindo o Google Maps, tornando-se assim um serviço padrão nos sistemas da Apple.",
   "emojipedia":"Emojipedia é um site de referência de emoji que documenta o significado e o uso comum de caracteres emoji no Unicode Standard. Mais comumente descrito como uma enciclopédia emoji ou dicionário emoji, Emojipedia também publica artigos e fornece ferramentas para rastrear novos caracteres emoji, alterações de design e tendências de uso. É propriedade da Zedge desde 2021.",
   "tineye":"TinEye é um mecanismo de busca de imagens reversas desenvolvido e oferecido pela Idée, Inc., uma empresa sediada em Toronto, Ontário no Canadá. É o primeiro mecanismo de pesquisa de imagens na Web a usar a tecnologia de identificação de imagens em vez de palavras-chave, metadados ou marcas d'água. TinEye permite aos usuários pesquisar não usando palavras-chave, mas com imagens. Ao enviar uma imagem, o TinEye cria uma \"assinatura digital única e compacta ou impressão digital\" da imagem e a combina com outras imagens indexadas.",
+  "etymonline":"O Online Etymology Dictionary (Etymonline) é um dicionário online gratuito, escrito e compilado por Douglas R. Harper, que descreve as origens das palavras da língua inglesa.",
   "fdroid":"F-Droid é um loja de software para Android, tem uma função similar à da Google Play. O repositório principal, hospedado pelo projeto, contém apenas aplicativos gratuitos e de código aberto. Os aplicativos podem ser navegados, baixados e instalados a partir do site ou do F-Droid sem a necessidade de registro. As \"anti-features\" tais como publicidade, rastreamento de usuários ou dependência de software não livre são sinalizados nas descrições dos aplicativos.",
   "flickr":"O Flickr é um site da web de hospedagem e partilha de imagens como fotografias, desenhos e ilustrações, além de permitir novas maneiras de organizar as fotos e vídeos. Caracterizado como rede social, o site permite aos usuários criar álbuns para armazenar suas fotografias e contatar-se com usuários de diferentes locais do mundo. No início de 2005 o site foi adquirido pela Yahoo! Inc.",
   "genius":"Genius é uma empresa estadunidense de mídia digital originalmente fundada em agosto de 2009 por Tom Lehman, Ilan Zechory e Mahbod Moghadam. O site permite que os usuários forneçam anotações e interpretações de letras de músicas, explicações de notícias, fontes, poesia e documentos.",
@@ -4391,7 +4430,7 @@
    "ref"
   ],
   "openstreetmap":"OpenStreetMap (OSM) é um projeto de mapeamento colaborativo para criar um mapa livre e editável do mundo, inspirado por sites como a Wikipédia. Traduzindo para português o nome significa Mapa Aberto de Ruas. Ele fornece dados a centenas de sites na internet, aplicações de celular e outros dispositivos.",
-  "piratebay":"The Pirate Bay (TPB), autointitulado \"O tracker BitTorrent mais resiliente da galáxia\", contém magnet links, sendo também o índice para os arquivos .torrent. Um magnet link ou um arquivo .torrent, em conjunto com um cliente BitTorrent, proporciona ao cliente as informações necessárias para se copiar um arquivo ou conjunto de arquivos de outras pessoas que estão copiando ou compartindo o mesmo arquivo.",
+  "piratebay":"The Pirate Bay é um índice online de conteúdo digital de mídia de entretenimento e software. Fundado em 2003 pelo think tank sueco Piratbyrån, o Pirate Bay permite que os visitantes pesquisem, baixem e contribuam com magnet links e arquivos torrent, que facilitam o compartilhamento de arquivos peer-to-peer entre os usuários do protocolo BitTorrent.",
   "pubmed":"MEDLINE® é uma sigla em inglês para Sistema Online de Busca e Análise de Literatura Médica é a base de dados bibliográficos da Biblioteca Nacional de Medicina dos Estados Unidos da América. Contém mais de 18 milhões de referências a artigos de jornais científicos, com maior concentração em biomedicina, mas contém também artigos sobre enfermagem, veterinária, farmacologia, odontologia, entre outros. Uma característica marcante da MEDLINE é que os dados gravados no sistema são indexados com palavras-chave específicas de um sistema chamado MeSH.",
   "pypi":"O Python Package Index, abreviado como PyPI e também conhecido como Cheese Shop, é o repositório de software oficial de terceiros para Python. É análogo ao CPAN, o repositório para Perl. Alguns gerenciadores de pacotes, incluindo o pip, usam o PyPI como a fonte padrão para os pacotes e suas dependências. Mais de 113.000 pacotes Python podem ser acessados por meio do PyPI.",
   "qwant":"Qwant é um motor de busca desenvolvido pela empresa francesa de mesmo nome, que se anuncia com uma forte política de privacidade.",
@@ -4429,10 +4468,10 @@
   "1337x":"1337x é um site, fundado em 2007, que fornece arquivos torrent e links magnéticos para facilitar o compartilhamento de arquivos ponto-a-ponto usando o protocolo BitTorrent.",
   "naver":"Naver é um popular portal de busca da Coreia do Sul, com um market share superior a 70%, comparado com 2% do Google. O Naver foi lançado em junho de 1999 por ex-funcionários da Samsung, e estreou como o primeiro portal da Coreia do Sul a usar seu próprio motor de busca. Entre os recursos do Naver está a \"Comprehensive Search\", lançada em 2000, que fornece resultados de várias categorias em uma única página. Desde então, tem agregado novos serviços, como a \"Knowledge Search\", lançada em 2002. Ele também oferece serviços de Internet, incluindo um serviço de notícias, um serviço de e-mail, um serviço de busca de teses acadêmicas e um portal para crianças. Em 2005, Naver lançou Happybean, o primeiro portal de doações online do mundo, que permite aos usuários encontrar informações e fazer doações para mais de 20.000 organizações da sociedade civil e de assistência social.",
   "rubygems":"RubyGems é um gerenciador de pacotes para a linguagem de programação Ruby que provê um formato padrão para a distribuição de programas Ruby e bibliotecas em um formato auto-suficiente chamado de gem, uma ferramenta projetada para gerenciar facilmente a instalação de gems, e um servidor para distribui-los. RubyGems foi criado em volta de novembro de 2003 e agora faz parte da biblioteca padrão do Ruby versão 1.9 a diante.",
-  "peertube":"PeerTube é uma plataforma de vídeo livre, descentralizada e federada operada por ActivityPub e WebTorrent que usa tecnologia peer-to-peer para reduzir o estresse em servidores individuais ao assistir vídeos.",
+  "peertube":"PeerTube é uma plataforma de vídeo livre, descentralizada e federada operada por ActivityPub e WebTorrent que usa a tecnologia peer-to-peer para reduzir o estresse em servidores individuais ao assistir vídeos.",
   "rumble":"Rumble é uma plataforma de compartilhamento de vídeo canadense com sede em Toronto. O serviço foi fundado em 2013 por Chris Pavlovski, um empresário de tecnologia do Canadá. A contagem mensal de usuários do Rumble experimentou um rápido crescimento desde julho de 2020, houve um salto de 1,6 milhões de usuários mensais para 31,9 milhões no final do primeiro trimestre de 2021.",
   "wttr.in":[
-   "Previsão do tempo para: Des Moines, Iowa, United States",
+   "Previsão do tempo para: not found",
    "https://wttr.in"
   ],
   "brave":"O Brave Search é um mecanismo de pesquisa desenvolvido pela Brave Software, Inc. Em determinados países, ele é definido como o mecanismo de pesquisa padrão para usuários do navegador Brave.",
@@ -4539,6 +4578,10 @@
    "tineye:pt",
    "ref"
   ],
+  "etymonline":[
+   "etymonline:pt",
+   "ref"
+  ],
   "fdroid":[
    "fdroid:pt",
    "ref"
@@ -4834,7 +4877,7 @@
  },
  "ru":{
   "9gag":"9GAG — интернет-платформа и социальная сеть. Пользователи загружают и делятся контентом, сделанным ими лично или взятым с других сайтов. Офис 9GAG находится в Маунтин-Вью. Со дня основания, 12 апреля 2008, и до сегодняшнего дня сайт сильно вырос и заполучил больше 34 миллионов лайков в Фейсбуке, 8 миллионов читателей в Твиттере и 46 миллионов подписчиков в Инстаграме. Он является одним из самых успешных сайтов, входя в Топ-200 веб-сайтов и его стоимость примерно составляет 56 млн долларов США.",
-  "apple app store":"App Store — магазин приложений, раздел онлайн-магазина iTunes Store, содержащий различные приложения для мобильных смартфонов iPhone, плееров iPod Touch и планшетов iPad, а также для персональных компьютеров Mac и позволяющий их купить, либо скачать бесплатно.",
+  "apple app store":"App Store — магазин приложений, раздел онлайн-магазина iTunes Store, содержащий различные приложения для мобильных смартфонов iPhone, плееров iPod Touch и планшетов iPad, а также для персональных компьютеров Mac и позволяющий их купить либо скачать бесплатно.",
   "archive is":"archive.today — бесплатный сервис по архивированию веб-страниц, запущенный в 2012 году одноимённой некоммерческой организацией. Archive.today сохраняет содержание страниц, включая изображения, однако не поддерживает динамический контент. В отличие от портала Wayback Machine (WB) archive.today архивирует страницы по запросу пользователей и не использует поисковых роботов.",
   "artic":"Чикагский институт искусств — художественный музей и высшее учебное заведение в Чикаго, штат Иллинойс в США. Основные учебные специализации — архитектура и изобразительное искусство.",
   "arxiv":"arXiv.org — электронный архив с открытым доступом для научных статей и препринтов по физике, математике, астрономии, информатике, биологии, электротехнике, статистике, финансовой математике и экономике. Перед публикацией статьи не рецензируются, однако проходят первичную проверку модераторов.",
@@ -4907,10 +4950,6 @@
   "hoogle":"Haskell — стандартизированный чистый функциональный язык программирования общего назначения. Является одним из самых распространённых языков программирования с поддержкой отложенных вычислений. Система типов — полная, сильная, статическая, с автоматическим выводом типов, основанная на системе типов Хиндли — Милнера. Поскольку язык функциональный, то основная управляющая структура — это функция.",
   "imdb":"Internet Movie Database — веб-сайт с условно свободно редактируемой и крупнейшей в мире базой данных о кинематографе. По состоянию на январь 2021 года, в базе собрана информация о более чем 6,5 млн кинофильмов, телесериалов и отдельных их серий, а также о 10,4 млн персоналий, связанных с кино и телевидением, — актёрах, режиссёрах, сценаристах и других.",
   "ina":"Национальный институт аудиовизуала — общественное учреждение, имеющее промышленный и коммерческий характер.",
-  "invidious":[
-   "альтернативный фронтенд для YouTube",
-   "wikidata"
-  ],
   "kickass":"KickassTorrents — вебсайт, поисковик .torrent-файлов и magnet-ссылок. Основан в 2008 году. По состоянию на июль 2016 года сайт занимал 68 место по посещаемости в мире согласно глобальному рейтингу Alexa. Один из серверов ресурса размещен в США.",
   "library genesis":"Library Genesis — веб-сайт, поисковая система и онлайн-хранилище, предоставляющее бесплатный доступ к пиратским коллекциям и защищённым авторским правом произведениям, в основном научной тематики. LibGen также называют «теневой библиотекой». Портал был создан в 2008 году, предположительно, группой российских учёных. До 2011 года коллекция LibGen росла в основном благодаря копированию других российских интернет-архивов и интеграции около полумиллиона англоязычных работ интернет-библиотеки Library.nu, закрытой в 2012 году. Начиная с 2013 года коллекция LibGen пополняется за счет интеграции созданных издателями электронных текстовых репозиториев. До 2013 года большая часть коллекции была представлена на русском и английском языках, позднее начали добавлять работы и на немецком, итальянском, испанском и французском.",
   "library of congress":"Библиотека Конгресса — исследовательская библиотека, которая официально обслуживает Конгресс США и является де-факто национальной библиотекой США. Это старейшее федеральное учреждение культуры в Соединённых Штатах. Библиотека расположена в трёх зданиях в районе Капитолийского холма в Вашингтоне, округ Колумбия; она также поддерживает Национальный центр аудиовизуальной консервации в Калпепер (Виргиния). Функции библиотеки контролирует библиотекарь Конгресса, а её здания обслуживает архитектор Капитолия. Библиотека Конгресса претендует на звание самой большой библиотеки в мире. Её «коллекции универсальны, не ограничены предметом, форматом или национальной границей и включают исследовательские материалы со всех частей света и на более чем 450 языках».",
@@ -4947,7 +4986,7 @@
   ],
   "semantic scholar":"Semantic Scholar (англ. Semantic Scholar — поисковая интернет-платформа, разработанная в Институте искусственного интеллекта Аллена. Проект был запущен в 2015 году. Поиск научных публикаций производится с поддержкой искусственного интеллекта для статей в научных журналах. Поисковый сервис комбинирует машинное обучение, обработку естественного языка и машинного зрения, чтобы добавить слой семантического анализа к традиционным методам анализа цитирования. Semantic Scholar выделяет наиболее важные статьи, а также связи между ними.",
   "startpage":"Ixquick — метапоисковая система, основанная Дэвидом Бодникином в 1998 году в Нью-Йорке и в Нидерландах. С 2000 года принадлежит нидерландской компании Surfboard Holding BV.",
-  "yahoo news":"Yahoo! News — новостной веб-сайт, созданный Yahoo! как интернет-агрегатор новостей.Сайт был создан инженером-программистом Yahoo! по имени Брэд Клоси в августе 1996 года. Первоначально статьи поступали из новостных служб, таких как Associated Press, Reuters, Fox News, Al Jazeera, ABC News, USA Today, CNN и BBC News.",
+  "yahoo news":"Yahoo! News — новостной веб-сайт, созданный Yahoo! как интернет-агрегатор новостей. Сайт был создан инженером-программистом Yahoo! по имени Брэд Клоси в августе 1996 года. Первоначально статьи поступали из новостных служб, таких как Associated Press, Reuters, Fox News, Al Jazeera, ABC News, USA Today, CNN и BBC News.",
   "youtube":"YouTube — видеохостинг, предоставляющий пользователям услуги хранения, доставки и показа видео. YouTube стал популярнейшим видеохостингом и вторым сайтом в мире по количеству посетителей.",
   "dailymotion":"Dailymotion — французский видеохостинг. По состоянию на 2017 год Dailymotion является 114-м по посещаемости сайтом мира по версии Alexa Internet. Наибольшее количество посетителей сайта из Японии и США.",
   "vimeo":"Vimeo — американский видеохостинг со штаб-квартирой в Нью-Йорке. Запущенный в 2004 году, по состоянию на март 2018 года занимает 130-е место в глобальном рейтинге сайтов и 91-е место в рейтинге США по данным Alexa. Имеет мобильные приложения для платформ iOS, Android и Windows Phone.",
@@ -4966,7 +5005,7 @@
   "rumble":"Rumble — канадский видеохостинг и облачная система хранения, имеющий штаб-квартиры в канадском Торонто и Лонгбот-Ки. Основан в октябре 2013 года канадским предпринимателем Крисом Павловски.",
   "wordnik":"Wordnik (wordnik.com) — интернет-сайт, разрабатываемый одноименной некоммерческой организацией, представляющий собой онлайн-словарь английского языка и языковой ресурс для словарей и тезауруса. Часть контента, представленного Wordnik, основывается на известных печатных словарях английского языка, таких как Century Dictionary, American Heritage Dictionary, WordNet и GCIDE. Wordnik собрал корпус из миллиардов слов, которые используются на сайте для отображения примеров предложений, что позволяет ему предоставлять информацию о гораздо большем наборе слов, чем в обычном словаре. Wordnik использует как можно больше реальных примеров при определении слова.",
   "wttr.in":[
-   "Прогноз погоды: Des Moines, Iowa, United States",
+   "Прогноз погоды: not found",
    "https://wttr.in"
   ]
  },
@@ -5034,10 +5073,6 @@
   ],
   "hoogle":"Haskell je štandardizovaný funkcionálny programovací jazyk s voľnou sémantikou pomenovaný po logikovi Haskellovi Currym. Bol vytvorený v 90. rokoch 20. storočia. Posledným polooficiálnym štandardom je Haskell 98, ktorý definuje minimálnu a portabilnú verziu jazyka využiteľnú na výuku alebo ako základ ďalších rozšírení. Jazyk sa rýchlo vyvíja, predovšetkým vďaka svojim implementáciám Hugs a GHC, ktoré predstavujú súčasný de facto štandard.",
   "imdb":"Internet Movie Database, skr. IMDb je na internete dostupná databáza informácií o filmových hviezdach, filmoch, televíznych reláciach, reklamách a videohrách. Vlastní ju firma Amazon.com.",
-  "invidious":[
-   "alternatívny frontend pre YouTube",
-   "wikidata"
-  ],
   "library of congress":"Kongresová knižnica je velka fest najvacsia na svete",
   "openstreetmap":"OpenStreetMap je otvorený projekt, ktorého cieľom je tvorba voľných geografických dát, ako sú napríklad cestné mapy. Používa predovšetkým dáta z prijímačov GPS, ktoré sú následne kontrolované a editované. Je založený na kolektívnej spolupráci a na koncepcii Open source.",
   "piratebay":"The Pirate Bay je webová stránka, ktorá patrí medzi najväčšie databázy torrentov na internete. Stránka je na 88. mieste najnavštevovanejších stránok na svete. Po skonfiškovaní serverov, ktoré sa 31. mája 2006 lokalizovali vo Švédsku, si TPB získala pozornosť švédskych a medzinárodných médií.",
@@ -5280,7 +5315,7 @@
   "wolframalpha":"Wolfram Alpha är ett sökmotorliknande internetverktyg som utvecklats av Wolfram Research.",
   "peertube":"Peertube, i marknadsföringssyfte skrivet PeerTube, är en fritt licensierad, decentraliserad, Activitypub-federerad videoplattform som använder WebTorrent- och peer-to-peer-teknik för att minska belastningen på enskilda servrar när videor visas.",
   "wttr.in":[
-   "Väderleksprognos för: Des Moines, Iowa, United States",
+   "Väderleksprognos för: not found",
    "https://wttr.in"
   ]
  },
@@ -5355,7 +5390,7 @@
   "wolframalpha":"வொல்பிராம் அல்பா (Wolfram|Alpha) என்பது ஒரு கேள்விகளுக்குப் பதிலளிக்கும் இயந்திரம். இது மதமட்டிக்கா மென்பொருளை உருவாக்கிய வொல்பிராம் ஆய்வு நிறுவனத்தால் உருவாக்கப்பட்டது. கேள்விகள் இலக்கணப் பகுப்பாய்வு செய்யப்பட்டு, கணிக்கூடியவாறு ஒழுங்கமைக்கப்பட்ட தரவுகளைக் கொண்டு விடைகள் தருவிக்கப்படுகின்றன. துறைசார் கேள்விகளுக்கு இது துல்லியமான பதில்களைத் தரக்கூடியது.",
   "rubygems":"ரூபி செம்சு (RubyGems) என்பது ரூபி நிரலாக்க மொழிக்கான ஒரு பொது மேலாண்மைக் கருவி ஆகும். ரூபி நிரல்களையும் காப்பகங்களையும் விநியோகிப்பதற்கான தரப்படுத்தப்பட்ட முறை இதுவாகும். இதனைப் பயன்படுத்தி இவற்றை இலகுவாக நிறுவி மேலாண்மை செய்ய முடியும். ரூபி 1.9 மற்றும் அதன் பின்னர் வெளியிடப்பட்ட அனைத்து பதிவுகளிலும் ரூபி செம்சு ஒரு பகுதியாக உள்ளடக்கப்பட்டுள்ளது.",
   "wttr.in":[
-   "வானிலை அறிக்கை Des Moines, Iowa, United States",
+   "வானிலை அறிக்கை not found",
    "https://wttr.in"
   ]
  },
@@ -5396,7 +5431,7 @@
   "wikisource":"వికీసోర్స్ స్వేచ్ఛా నకలు హక్కుల రచనలను ప్రచురించుటకు సముదాయసభ్యులు సేకరించి, నిర్వహించుచున్న ఒక స్వేచ్ఛాయుత గ్రంథాలయము. దీనిని 2005 ఆగస్టు 19 న ప్రారంభమైంది. ప్రారంభంలో విశేషంగా కృషిచేసినవాడుకరులు అన్వేషి, రాజ్, రాజశేఖర్ (Rajasekhar1961), మల్లిన నరసింహారావు, తాడేపల్లి (Tadepally), వైఙాసత్య, రాకేశ్వర, సురేష్ (Sureshkvolam), సుజాత. అన్వేషి ఏప్రిల్ నుండి డిసెంబరు 2007 మధ్య శతకాలు, భగవద్గీత, వాల్మీకి రామాయణం మొదలగునవి వికీసోర్స్ లో చేర్చాడు. తరువాత వికీసోర్స్ కి కావలసిన మూసలు తెలుగుసేత, డాక్యుమెంటేషన్ పేజీలు తయారుచేయడం, రచనలు చేర్చడం మొదలగు మెరుగులుచేశాడు. ఫ్రూఫ్ రీడ్ ఎక్స్టెన్షన్ వాడుటకు చేసిన ప్రయత్నం మధ్యలో ఆగిపోయింది. 2012లో అది పూర్తి కావించబడింది. వైఙాసత్య దీనిలో తెలుగు నేరుగా టైపు చేసేసౌకర్యం కలిగించాడు, మొల్ల రామాయణం చేర్చటానికి కృషి చేసాడు.",
   "wiktionary":"విక్షనరీ, వికీపీడియా యొక్క సోదర వెబ్ సైట్. ఈ పదం వికి, డిక్షనరి పదాలను కలుపగా తయారయ్యినది. ఇది తెలుగు పదాలను వివిధమైన వ్యాకరణ, వాడుక, నానార్ధ, వ్యతిరేఖార్థ లాంటి వివరణలతో నిక్షిప్తం చేసే మాధ్యమము (నిఘంటువు). అయితే పుస్తక రూపంలో వుండే నిఘంటువులు మహా అయితే మూడు భాషలలో వుంటాయి. దీనిలో తెలుగు-తెలుగు, ఇంగ్లీషు-తెలుగుతో పాటు ఇతర విక్షనరీలోని సమాన అర్థం గల పదాలకు లింకులుండటంవలన, మీకు ప్రపంచంలోని వికీ భాషలన్నిటిలో సమాన అర్థంగల పదాలను తెలుసుకునే వీలుండటంతో, దీనిని బహుభాష నిఘంటువుగా పేర్కొనవచ్చు. తెలుగు వికీపీడియాలో లాగా, ఇందులో ఎవరైనా తెలుగు పదాలకు పేజీలను సృష్టించవచ్చు లేక మార్పులు చేయవచ్చు.",
   "wttr.in":[
-   "వాతావరణ సమాచారము: Des Moines, Iowa, United States",
+   "వాతావరణ సమాచారము: not found",
    "https://wttr.in"
   ]
  },
@@ -5437,7 +5472,7 @@
    "currency:th",
    "ref"
   ],
-  "flickr":"ฟลิคเกอร์ (Flickr) เป็น เว็บไซต์สำหรับเก็บรูปภาพดิจิตัล โดยอัปโหลดจากผู้ใช้งาน และสามารถแบ่งปันให้ผู้อื่นดูได้",
+  "flickr":"ฟลิคเกอร์ เป็น เว็บไซต์สำหรับเก็บรูปภาพดิจิทัล โดยอัปโหลดจากผู้ใช้งาน และสามารถแบ่งปันให้ผู้อื่นดูได้",
   "gentoo":[
    "gentoo:ru",
    "ref"
@@ -5479,7 +5514,7 @@
   "wikiversity":"วิกิวิทยาลัย เป็นโครงการหนึ่งของมูลนิธิวิกิมีเดีย โดยมีเป้าหมายรวบรวมความรู้ต่าง ๆ คล้ายมหาวิทยาลัย โดยวิกิวิทยาลัยยังไม่มีแบบภาษาไทย วิกิวิทยาลัยใช้ซอฟต์แวร์มีเดียวิกิซึ่งเป็นซอฟต์แวร์เดียวกันกับวิกิพีเดีย และเผยแพร่ภายใต้ GFDL และ CC-BY-SA",
   "wikivoyage":"วิกิท่องเที่ยว เป็นคู่มือท่องเที่ยวออนไลน์สำหรับแหล่งท่องเที่ยวและหัวข้อท่องเที่ยวที่เขียนโดยอาสาสมัคร ชื่อของโครงการนี้ในภาษาอังกฤษประกอบด้วย \"Wiki\" และ \"Voyage\" คำภาษาฝรังเศสที่หมายถึงการท่องเที่ยว การเดินทาง",
   "wttr.in":[
-   "รายงานสภาพอากาศ: Des Moines, Iowa, United States",
+   "รายงานสภาพอากาศ: not found",
    "https://wttr.in"
   ]
  },
@@ -5595,6 +5630,10 @@
   "seznam":"Seznam.cz, Çek Cumhuriyeti merkezli arama motoru ve internet portalıdır.",
   "mojeek":"Mojeek, internet sansürü barındırmayan, gizlilik dostu bir arama motorudur. İngiltere'de ortaya çıkan bir projedir ve Marc Smith Mojeek'in kurucusudur. Arama motoru C programlama dili ile yazılmıştır ve 2021 yılında 4 milyar sayfa hedefini geçmiştir.",
   "naver":"Naver, Güney Kore merkezli bir arama motoru ve internet portalıdır. Site, Haziran 1999 tarihinde eski Samsung çalışanları tarafından kurulmuş olup Güney Kore'nin kendi arama motoruna sahip ilk internet portalıdır. Günümüzde Naver Corporation bünyesinde faaliyet göstermektedir.",
+  "wttr.in":[
+   "Hava beklentisi: not found",
+   "https://wttr.in"
+  ],
   "petalsearch":"Petal Search, Huawei tarafından 2020 yılının ikinci yarısında geliştirilen, Huawei ve diğer mobil cihazlarda yanı sıra masaüstünde de kullanılabilen bir arama motorudur. Arama motoruna, Huawei'nin logosundaki çiçek yapraklarından ilham alınarak Petal ismi verildi.",
   "petalsearch images":[
    "petalsearch:tr",
@@ -5611,7 +5650,7 @@
   "arxiv":"arXiv.org — найбільший безкоштовний архів електронних публікацій наукових статей та їх препринтів. ArXiv підтримується бібліотекою Корнелльського університету під керівництвом науково-консультативної ради архіву та консультативної групи щодо стійкості архіву, а також за допомогою численних модераторів тем. Наукова тематика архіву включає астрономію, фізику, математику, інформатику, кількісну біологію, статистику та фінансову математику.",
   "bandcamp":"Bandcamp — приватна компанія, заснована колишніми співзасновниками сервісу Oddpost, а саме Ітаном Даймондом і Шоном Ґрюнберґером у 2007 році разом з програмістами Джо Голтом і Нілом Такером. У 2008 році компанія запустила інтернет-магазин музики, який є також платформою для розкрутки музикантів. Музиканти, що використовують Bandcamp, можуть скористатися інтерфейсом сайту, а також опублікувати власні композиції. Всі композиції доступні для користувачів безкоштовно, при цьому найчастіше надається можливість придбати альбом або конкретний трек за гнучкими цінами.",
   "wikipedia":"Вікіпе́дія — загальнодоступна вільна багатомовна онлайн-енциклопедія, якою опікується неприбуткова організація «Фонд Вікімедіа».",
-  "bing":"Bing — пошукова система, що належить компанії Microsoft. Цей пошуковий сервіс змінив попередні пошукові, що розроблялись корпорацією: MSN Search, Windows Live Search та пізніше Live Search. Bing виконує пошук тексту, зображень, відео або географічних об'єктів, які потім відображає на мапі. Сервіс працює на платформі ASP.NET.",
+  "bing":"Bing — пошукова система, що належить компанії Microsoft. Цей пошуковий сервіс змінив попередні пошукові, що розроблялись корпорацією: MSN Search, Windows Live Search та пізніше Live Search. Bing виконує пошук тексту, зображень, відео або географічних об'єктів, які потім показує на мапі. Сервіс працює на платформі ASP.NET.",
   "bing images":[
    "bing:uk",
    "ref"
@@ -5715,7 +5754,11 @@
    "система керування пакунками",
    "wikidata"
   ],
-  "peertube":"PeerTube — децентралізований, федеративний відеохостинг з відкритим початковим кодом, заснований на технологіях ActivityPub та WebTorrent. Створений в 2017 році розробником з ніком Chocobozzz, у подальшому підтримку розробки взяла на себе французька некомерційна організація Framasoft."
+  "peertube":"PeerTube — децентралізований, федеративний відеохостинг з відкритим початковим кодом, заснований на технологіях ActivityPub та WebTorrent. Створений в 2017 році розробником з ніком Chocobozzz, у подальшому підтримку розробки взяла на себе французька некомерційна організація Framasoft.",
+  "wttr.in":[
+   "Прогноз погоди для: not found",
+   "https://wttr.in"
+  ]
  },
  "vi":{
   "9gag":"9GAG là một trang web giải trí có trụ sở chính tại Hồng Kông với chủ đề là các hình ảnh do người dùng cung cấp cho phép người dùng tải lên và chia sẻ nội dung do chính người dùng tạo hoặc những nội dung khác từ các nền tảng mạng xã hội trực tuyến bên ngoài. Được ra mắt vào ngày 11 tháng 4 năm 2008, trang web đã đạt một tỷ lượt xem vào tháng 12 năm 2011 và đã trở nên phổ biến trên các nền tảng mạng xã hội trực tuyến như Facebook, Twitter và Instagram.",
@@ -5759,13 +5802,17 @@
    "ref"
   ],
   "apple maps":"Apple Maps là một dịch vụ ứng dụng và công nghệ bản đồ trực tuyến trên web miễn phí do Apple Inc. cung cấp. Ứng dụng này được cài mặt định trên các thiết bị chạy hệ điều hành iOS, OS X và watchOS. Apple Maps có thể hướng dẫn đường đi và thời gian lái xe dự kiến ​​cho ô tô, người đi bộ, hướng dẫn chuyển hướng giao thông công cộng.. Ngoài ra, ứng dụng còn có tính năng 3D Flyover xem dạng vệ tinh, cho phép người sử dụng có thể xoay bản đồ dạng 3D và xem ở các góc độ khác nhau.",
+  "etymonline":[
+   "Từ điển từ nguyên tiếng Anh trực tuyến",
+   "wikidata"
+  ],
   "flickr":"Flickr là một trang mạng và bộ dịch vụ web chia sẻ hình ảnh, và một nền tảng cộng đồng trực tuyến, được xem như một kiểu mẫu sớm nhất cho ứng dụng Web 2.0. Flickr được tạo bởi Ludicorp vào năm 2004. Qua vài lần thay đổi chủ sở hữu, trong đó nổi tiếng nhất là Yahoo!, SmugMug đã mua lại Flickr vào ngày 20 tháng 4 năm 2018 từ Verizon's Oath, công ty chủ quản của Yahoo!.",
   "gentoo":[
    "gentoo:ru",
    "ref"
   ],
   "github":"GitHub là một dịch vụ cung cấp kho lưu trữ mã nguồn Git dựa trên nền web cho các dự án phát triển phần mềm. GitHub cung cấp cả phiên bản trả tiền lẫn miễn phí cho các tài khoản. Các dự án mã nguồn mở sẽ được cung cấp kho lưu trữ miễn phí. Tính đến tháng 4 năm 2016, GitHub có hơn 14 triệu người sử dụng với hơn 35 triệu kho mã nguồn, làm cho nó trở thành máy chủ chứa mã nguồn lớn trên thế giới.",
-  "google":"Google Tìm kiếm là dịch vụ cung cấp chính và quan trọng nhất của công ty Google. Dịch vụ này cho phép người truy cập tìm kiếm thông tin về trên Internet bằng cách sử dụng công cụ tìm kiếm Google, bao gồm các trang Web, hình ảnh & nhiều thông tin khác.",
+  "google":"Google Tìm kiếm, cũng được gọi với tên tiếng Anh phổ biến là Google Search hay đơn giản là Google, là dịch vụ cung cấp chính và quan trọng nhất của công ty Google. Dịch vụ này cho phép người truy cập tìm kiếm thông tin về trên Internet bằng cách sử dụng công cụ tìm kiếm Google, bao gồm các trang Web, hình ảnh & nhiều thông tin khác.",
   "google images":"Google Images là một dịch vụ tìm kiếm được tạo ra bởi Google cho phép người dùng tìm hình ảnh trên các trang web. Tính năng này được hoàn thành vào tháng 12 năm 2001. Những từ khóa để tìm kiếm hình ảnh được dựa theo tên của file hình ảnh, đoạn văn bản chứa đường link đến tấm hình và những đoạn nằm gần bức ảnh. Khi tìm kiếm một tấm hình, một hình thu nhỏ của mỗi tấm hình khớp với từ khóa tìm kiếm sẽ được hiển thị. Khi nháp vào hình thu nhỏ, tấm hình sẽ được hiển thị trong một khung ở phía trên trang và trang web chứa tấm hình sẽ được hiển thị trong khung bên dưới, tạo sự dễ dàng để thấy được nơi mà tấm hình xuất hiện.",
   "google news":"Google News là một trang web tổng hợp tin tức tự động được cung cấp bởi Google. Ý tưởng ban đầu được hình thành từ việc xếp hạng trang web của Google, được phát triển bởi Krishna Bharat vào năm 2001, trưởng bộ phận Nghiên cứu của Google. Không ai được thay thế trang chủ hoặc nội dung của nó. Tất cả đều được thực hiện bằng các giải thuật tổng hợp tin. Google News trở thành bản chính thức vào tháng 1 năm 2006. Hiện trang đã có phiên bản tiếng Việt tại địa chỉ http://news.google.com.vn.",
   "google videos":"Google Video là một dịch vụ chia sẻ video trực tuyến của Google cho phép mọi người tải các đoạn clip của mình lên máy chủ của Google mà không tốn bất kỳ phí nào, đồng thời có thể chia sẻ cho mọi người hoặc bán các đoạn video clip của mình thông qua các cửa hàng online của Google Video.",
@@ -5807,7 +5854,7 @@
   "naver":"Naver là một nền tảng trực tuyến của Hàn Quốc được điều hành bởi Naver Corporation. Được ra mắt lần đầu năm 1999, Naver vốn là cổng thông tin điện tử đầu tiên tại Hàn Quốc, sau đó được phát triển thành một công cụ tìm kiếm riêng. Đây cũng là nhà điều hành đầu tiên trên thế giới ra mắt tính năng tìm kiếm toàn diện, tính toán các kết quả tìm kiếm từ nhiều danh mục tìm kiếm khác nhau và trình bày chúng trong một trang duy nhất. Từ đó, Naver cũng đã xây dựng thêm vô số các dịch vụ từ cơ bản như e-mail, tin tức đến nền tảng trực tuyến Q&A đầu tiên trên thế giới Knowledge iN.",
   "peertube":"PeerTube là một nền tảng chia sẻ video liên hợp, tự do và nguồn mở hoạt động với hình thức tự lưu trữ (self-hosting). Nền tảng này sử dụng giao thức ActivityPub và WebTorrent, một công nghệ P2P tiết kiệm tài nguyên cho các máy chủ cá nhân.",
   "wttr.in":[
-   "Báo cáo thời tiết: Des Moines, Iowa, United States",
+   "Báo cáo thời tiết: not found",
    "https://wttr.in"
   ]
  },
@@ -6009,7 +6056,7 @@
    "ref"
   ],
   "wttr.in":[
-   "天气预报： Des Moines, Iowa, United States",
+   "天气预报： not found",
    "https://wttr.in"
   ],
   "goo":[

searx/data/useragents.json

@@ -1,7 +1,7 @@
 {
     "versions": [
-        "110.0",
-        "109.0"
+        "111.0",
+        "110.0"
     ],
     "os": [
         "Windows NT 10.0; Win64; x64",

searx/enginelib/__init__.py

@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Engine related implementations
+
+.. note::
+
+   The long term goal is to modularize all relevant implementations to the
+   engines here in this Python package.  In addition to improved modularization,
+   this will also be necessary in part because the probability of circular
+   imports will increase due to the increased typification of implementations in
+   the future.
+
+   ToDo:
+
+   - move :py:obj:`searx.engines.load_engine` to a new module `searx.enginelib`.
+"""
+
+
+from __future__ import annotations
+from typing import Union, Dict, List, Callable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from searx.enginelib import traits
+
+
+class Engine:  # pylint: disable=too-few-public-methods
+    """Class of engine instances build from YAML settings.
+
+    Further documentation see :ref:`general engine configuration`.
+
+    .. hint::
+
+       This class is currently never initialized and only used for type hinting.
+    """
+
+    # Common options in the engine module
+
+    engine_type: str
+    """Type of the engine (:origin:`searx/search/processors`)"""
+
+    paging: bool
+    """Engine supports multiple pages."""
+
+    time_range_support: bool
+    """Engine supports search time range."""
+
+    safesearch: bool
+    """Engine supports SafeSearch"""
+
+    language_support: bool
+    """Engine supports languages (locales) search."""
+
+    language: str
+    """For an engine, when there is ``language: ...`` in the YAML settings the engine
+    does support only this one language:
+
+    .. code:: yaml
+
+      - name: google french
+        engine: google
+        language: fr
+    """
+
+    region: str
+    """For an engine, when there is ``region: ...`` in the YAML settings the engine
+    does support only this one region::
+
+    .. code:: yaml
+
+      - name: google belgium
+        engine: google
+        region: fr-BE
+    """
+
+    fetch_traits: Callable
+    """Function to to fetch engine's traits from origin."""
+
+    traits: traits.EngineTraits
+    """Traits of the engine."""
+
+    # settings.yml
+
+    categories: List[str]
+    """Tabs, in which the engine is working."""
+
+    name: str
+    """Name that will be used across SearXNG to define this engine.  In settings, on
+    the result page .."""
+
+    engine: str
+    """Name of the python file used to handle requests and responses to and from
+    this search engine (file name from :origin:`searx/engines` without
+    ``.py``)."""
+
+    enable_http: bool
+    """Enable HTTP (by default only HTTPS is enabled)."""
+
+    shortcut: str
+    """Code used to execute bang requests (``!foo``)"""
+
+    timeout: float
+    """Specific timeout for search-engine."""
+
+    display_error_messages: bool
+    """Display error messages on the web UI."""
+
+    proxies: dict
+    """Set proxies for a specific engine (YAML):
+
+    .. code:: yaml
+
+       proxies :
+         http:  socks5://proxy:port
+         https: socks5://proxy:port
+    """
+
+    disabled: bool
+    """To disable by default the engine, but not deleting it.  It will allow the
+    user to manually activate it in the settings."""
+
+    inactive: bool
+    """Remove the engine from the settings (*disabled & removed*)."""
+
+    about: dict
+    """Additional fileds describing the engine.
+
+    .. code:: yaml
+
+       about:
+          website: https://example.com
+          wikidata_id: Q306656
+          official_api_documentation: https://example.com/api-doc
+          use_official_api: true
+          require_api_key: true
+          results: HTML
+    """

searx/enginelib/traits.py

@@ -0,0 +1,250 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Engine's traits are fetched from the origin engines and stored in a JSON file
+in the *data folder*.  Most often traits are languages and region codes and
+their mapping from SearXNG's representation to the representation in the origin
+search engine.  For new traits new properties can be added to the class
+:py:class:`EngineTraits`.
+
+To load traits from the persistence :py:obj:`EngineTraitsMap.from_data` can be
+used.
+"""
+
+from __future__ import annotations
+import json
+import dataclasses
+from typing import Dict, Union, Callable, Optional, TYPE_CHECKING
+from typing_extensions import Literal, Self
+
+from searx import locales
+from searx.data import data_dir, ENGINE_TRAITS
+
+if TYPE_CHECKING:
+    from . import Engine
+
+
+class EngineTraitsEncoder(json.JSONEncoder):
+    """Encodes :class:`EngineTraits` to a serializable object, see
+    :class:`json.JSONEncoder`."""
+
+    def default(self, o):
+        """Return dictionary of a :class:`EngineTraits` object."""
+        if isinstance(o, EngineTraits):
+            return o.__dict__
+        return super().default(o)
+
+
+@dataclasses.dataclass
+class EngineTraits:
+    """The class is intended to be instantiated for each engine."""
+
+    regions: Dict[str, str] = dataclasses.field(default_factory=dict)
+    """Maps SearXNG's internal representation of a region to the one of the engine.
+
+    SearXNG's internal representation can be parsed by babel and the value is
+    send to the engine:
+
+    .. code:: python
+
+       regions ={
+           'fr-BE' : <engine's region name>,
+       }
+
+       for key, egnine_region regions.items():
+          searxng_region = babel.Locale.parse(key, sep='-')
+          ...
+    """
+
+    languages: Dict[str, str] = dataclasses.field(default_factory=dict)
+    """Maps SearXNG's internal representation of a language to the one of the engine.
+
+    SearXNG's internal representation can be parsed by babel and the value is
+    send to the engine:
+
+    .. code:: python
+
+       languages = {
+           'ca' : <engine's language name>,
+       }
+
+       for key, egnine_lang in languages.items():
+          searxng_lang = babel.Locale.parse(key)
+          ...
+    """
+
+    all_locale: Optional[str] = None
+    """To which locale value SearXNG's ``all`` language is mapped (shown a "Default
+    language").
+    """
+
+    data_type: Literal['traits_v1'] = 'traits_v1'
+    """Data type, default is 'traits_v1'.
+    """
+
+    custom: Dict[str, Dict] = dataclasses.field(default_factory=dict)
+    """A place to store engine's custom traits, not related to the SearXNG core
+
+    """
+
+    def get_language(self, searxng_locale: str, default=None):
+        """Return engine's language string that *best fits* to SearXNG's locale.
+
+        :param searxng_locale: SearXNG's internal representation of locale
+          selected by the user.
+
+        :param default: engine's default language
+
+        The *best fits* rules are implemented in
+        :py:obj:`locales.get_engine_locale`.  Except for the special value ``all``
+        which is determined from :py:obj`EngineTraits.all_language`.
+        """
+        if searxng_locale == 'all' and self.all_locale is not None:
+            return self.all_locale
+        return locales.get_engine_locale(searxng_locale, self.languages, default=default)
+
+    def get_region(self, searxng_locale: str, default=None):
+        """Return engine's region string that best fits to SearXNG's locale.
+
+        :param searxng_locale: SearXNG's internal representation of locale
+          selected by the user.
+
+        :param default: engine's default region
+
+        The *best fits* rules are implemented in
+        :py:obj:`locales.get_engine_locale`.  Except for the special value ``all``
+        which is determined from :py:obj`EngineTraits.all_language`.
+        """
+        if searxng_locale == 'all' and self.all_locale is not None:
+            return self.all_locale
+        return locales.get_engine_locale(searxng_locale, self.regions, default=default)
+
+    def is_locale_supported(self, searxng_locale: str) -> bool:
+        """A *locale* (SearXNG's internal representation) is considered to be supported
+        by the engine if the *region* or the *language* is supported by the
+        engine.  For verification the functions :py:func:`self.get_region` and
+        :py:func:`self.get_region` are used.
+        """
+        if self.data_type == 'traits_v1':
+            return bool(self.get_region(searxng_locale) or self.get_language(searxng_locale))
+
+        raise TypeError('engine traits of type %s is unknown' % self.data_type)
+
+    def copy(self):
+        """Create a copy of the dataclass object."""
+        return EngineTraits(**dataclasses.asdict(self))
+
+    @classmethod
+    def fetch_traits(cls, engine: Engine) -> Union[Self, None]:
+        """Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch
+        and set properties from the origin engine in the object ``engine_traits``.  If
+        function does not exists, ``None`` is returned.
+        """
+
+        fetch_traits = getattr(engine, 'fetch_traits', None)
+        engine_traits = None
+
+        if fetch_traits:
+            engine_traits = cls()
+            fetch_traits(engine_traits)
+        return engine_traits
+
+    def set_traits(self, engine: Engine):
+        """Set traits from self object in a :py:obj:`.Engine` namespace.
+
+        :param engine: engine instance build by :py:func:`searx.engines.load_engine`
+        """
+
+        if self.data_type == 'traits_v1':
+            self._set_traits_v1(engine)
+        else:
+            raise TypeError('engine traits of type %s is unknown' % self.data_type)
+
+    def _set_traits_v1(self, engine: Engine):
+        # For an engine, when there is `language: ...` in the YAML settings the engine
+        # does support only this one language (region)::
+        #
+        #   - name: google italian
+        #     engine: google
+        #     language: it
+        #     region: it-IT
+
+        traits = self.copy()
+
+        _msg = "settings.yml - engine: '%s' / %s: '%s' not supported"
+
+        languages = traits.languages
+        if hasattr(engine, 'language'):
+            if engine.language not in languages:
+                raise ValueError(_msg % (engine.name, 'language', engine.language))
+            traits.languages = {engine.language: languages[engine.language]}
+
+        regions = traits.regions
+        if hasattr(engine, 'region'):
+            if engine.region not in regions:
+                raise ValueError(_msg % (engine.name, 'region', engine.region))
+            traits.regions = {engine.region: regions[engine.region]}
+
+        engine.language_support = bool(traits.languages or traits.regions)
+
+        # set the copied & modified traits in engine's namespace
+        engine.traits = traits
+
+
+class EngineTraitsMap(Dict[str, EngineTraits]):
+    """A python dictionary to map :class:`EngineTraits` by engine name."""
+
+    ENGINE_TRAITS_FILE = (data_dir / 'engine_traits.json').resolve()
+    """File with persistence of the :py:obj:`EngineTraitsMap`."""
+
+    def save_data(self):
+        """Store EngineTraitsMap in in file :py:obj:`self.ENGINE_TRAITS_FILE`"""
+        with open(self.ENGINE_TRAITS_FILE, 'w', encoding='utf-8') as f:
+            json.dump(self, f, indent=2, sort_keys=True, cls=EngineTraitsEncoder)
+
+    @classmethod
+    def from_data(cls) -> Self:
+        """Instantiate :class:`EngineTraitsMap` object from :py:obj:`ENGINE_TRAITS`"""
+        obj = cls()
+        for k, v in ENGINE_TRAITS.items():
+            obj[k] = EngineTraits(**v)
+        return obj
+
+    @classmethod
+    def fetch_traits(cls, log: Callable) -> Self:
+        from searx import engines  # pylint: disable=cyclic-import, import-outside-toplevel
+
+        names = list(engines.engines)
+        names.sort()
+        obj = cls()
+
+        for engine_name in names:
+            engine = engines.engines[engine_name]
+
+            traits = EngineTraits.fetch_traits(engine)
+            if traits is not None:
+                log("%-20s: SearXNG languages --> %s " % (engine_name, len(traits.languages)))
+                log("%-20s: SearXNG regions   --> %s" % (engine_name, len(traits.regions)))
+                obj[engine_name] = traits
+
+        return obj
+
+    def set_traits(self, engine: Engine):
+        """Set traits in a :py:obj:`Engine` namespace.
+
+        :param engine: engine instance build by :py:func:`searx.engines.load_engine`
+        """
+
+        engine_traits = EngineTraits(data_type='traits_v1')
+        if engine.name in self.keys():
+            engine_traits = self[engine.name]
+
+        elif engine.engine in self.keys():
+            # The key of the dictionary traits_map is the *engine name*
+            # configured in settings.xml.  When multiple engines are configured
+            # in settings.yml to use the same origin engine (python module)
+            # these additional engines can use the languages from the origin
+            # engine.  For this use the configured ``engine: ...`` from
+            # settings.yml
+            engine_traits = self[engine.engine]
+
+        engine_traits.set_traits(engine)

searx/engines/__init__.py

@@ -11,24 +11,22 @@ usage::
 
 """
 
+from __future__ import annotations
+
 import sys
 import copy
-from typing import Dict, List, Optional
-
 from os.path import realpath, dirname
-from babel.localedata import locale_identifiers
-from searx import logger, settings
-from searx.data import ENGINES_LANGUAGES
-from searx.network import get
-from searx.utils import load_module, match_language, gen_useragent
 
+from typing import TYPE_CHECKING, Dict, Optional
+
+from searx import logger, settings
+from searx.utils import load_module
+
+if TYPE_CHECKING:
+    from searx.enginelib import Engine
 
 logger = logger.getChild('engines')
 ENGINE_DIR = dirname(realpath(__file__))
-BABEL_LANGS = [
-    lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]
-    for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())
-]
 ENGINE_DEFAULT_ARGS = {
     "engine_type": "online",
     "inactive": False,
@@ -36,8 +34,6 @@ ENGINE_DEFAULT_ARGS = {
     "timeout": settings["outgoing"]["request_timeout"],
     "shortcut": "-",
     "categories": ["general"],
-    "supported_languages": [],
-    "language_aliases": {},
     "paging": False,
     "safesearch": False,
     "time_range_support": False,
@@ -52,24 +48,6 @@ ENGINE_DEFAULT_ARGS = {
 OTHER_CATEGORY = 'other'
 
 
-class Engine:  # pylint: disable=too-few-public-methods
-    """This class is currently never initialized and only used for type hinting."""
-
-    name: str
-    engine: str
-    shortcut: str
-    categories: List[str]
-    supported_languages: List[str]
-    about: dict
-    inactive: bool
-    disabled: bool
-    language_support: bool
-    paging: bool
-    safesearch: bool
-    time_range_support: bool
-    timeout: float
-
-
 # Defaults for the namespace of an engine module, see :py:func:`load_engine`
 
 categories = {'general': []}
@@ -136,9 +114,15 @@ def load_engine(engine_data: dict) -> Optional[Engine]:
         return None
 
     update_engine_attributes(engine, engine_data)
-    set_language_attributes(engine)
     update_attributes_for_tor(engine)
 
+    # avoid cyclic imports
+    # pylint: disable=import-outside-toplevel
+    from searx.enginelib.traits import EngineTraitsMap
+
+    trait_map = EngineTraitsMap.from_data()
+    trait_map.set_traits(engine)
+
     if not is_engine_active(engine):
         return None
 
@@ -190,60 +174,6 @@ def update_engine_attributes(engine: Engine, engine_data):
             setattr(engine, arg_name, copy.deepcopy(arg_value))
 
 
-def set_language_attributes(engine: Engine):
-    # assign supported languages from json file
-    if engine.name in ENGINES_LANGUAGES:
-        engine.supported_languages = ENGINES_LANGUAGES[engine.name]
-
-    elif engine.engine in ENGINES_LANGUAGES:
-        # The key of the dictionary ENGINES_LANGUAGES is the *engine name*
-        # configured in settings.xml.  When multiple engines are configured in
-        # settings.yml to use the same origin engine (python module) these
-        # additional engines can use the languages from the origin engine.
-        # For this use the configured ``engine: ...`` from settings.yml
-        engine.supported_languages = ENGINES_LANGUAGES[engine.engine]
-
-    if hasattr(engine, 'language'):
-        # For an engine, when there is `language: ...` in the YAML settings, the
-        # engine supports only one language, in this case
-        # engine.supported_languages should contains this value defined in
-        # settings.yml
-        if engine.language not in engine.supported_languages:
-            raise ValueError(
-                "settings.yml - engine: '%s' / language: '%s' not supported" % (engine.name, engine.language)
-            )
-
-        if isinstance(engine.supported_languages, dict):
-            engine.supported_languages = {engine.language: engine.supported_languages[engine.language]}
-        else:
-            engine.supported_languages = [engine.language]
-
-    # find custom aliases for non standard language codes
-    for engine_lang in engine.supported_languages:
-        iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None)
-        if (
-            iso_lang
-            and iso_lang != engine_lang
-            and not engine_lang.startswith(iso_lang)
-            and iso_lang not in engine.supported_languages
-        ):
-            engine.language_aliases[iso_lang] = engine_lang
-
-    # language_support
-    engine.language_support = len(engine.supported_languages) > 0
-
-    # assign language fetching method if auxiliary method exists
-    if hasattr(engine, '_fetch_supported_languages'):
-        headers = {
-            'User-Agent': gen_useragent(),
-            'Accept-Language': "en-US,en;q=0.5",  # bing needs to set the English language
-        }
-        engine.fetch_supported_languages = (
-            # pylint: disable=protected-access
-            lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers))
-        )
-
-
 def update_attributes_for_tor(engine: Engine) -> bool:
     if using_tor_proxy(engine) and hasattr(engine, 'onion_url'):
         engine.search_url = engine.onion_url + getattr(engine, 'search_path', '')

searx/engines/archlinux.py

@@ -1,15 +1,32 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
 """
- Arch Linux Wiki
+Arch Linux Wiki
+~~~~~~~~~~~~~~~
+
+This implementation does not use a official API: Mediawiki provides API, but
+Arch Wiki blocks access to it.
 
- API: Mediawiki provides API, but Arch Wiki blocks access to it
 """
 
-from urllib.parse import urlencode, urljoin
-from lxml import html
+from typing import TYPE_CHECKING
+from urllib.parse import urlencode, urljoin, urlparse
+import lxml
+import babel
+
+from searx import network
 from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
+from searx.enginelib.traits import EngineTraits
+from searx.locales import language_tag
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
+
 
-# about
 about = {
     "website": 'https://wiki.archlinux.org/',
     "wikidata_id": 'Q101445877',
@@ -22,125 +39,113 @@ about = {
 # engine dependent config
 categories = ['it', 'software wikis']
 paging = True
-base_url = 'https://wiki.archlinux.org'
-
-# xpath queries
-xpath_results = '//ul[@class="mw-search-results"]/li'
-xpath_link = './/div[@class="mw-search-result-heading"]/a'
+main_wiki = 'wiki.archlinux.org'
 
 
-# cut 'en' from 'en-US', 'de' from 'de-CH', and so on
-def locale_to_lang_code(locale):
-    if locale.find('-') >= 0:
-        locale = locale.split('-')[0]
-    return locale
-
-
-# wikis for some languages were moved off from the main site, we need to make
-# requests to correct URLs to be able to get results in those languages
-lang_urls = {
-    # fmt: off
-    'all': {
-        'base': 'https://wiki.archlinux.org',
-        'search': '/index.php?title=Special:Search&offset={offset}&{query}'
-    },
-    'de': {
-        'base': 'https://wiki.archlinux.de',
-        'search': '/index.php?title=Spezial:Suche&offset={offset}&{query}'
-    },
-    'fr': {
-        'base': 'https://wiki.archlinux.fr',
-        'search': '/index.php?title=Spécial:Recherche&offset={offset}&{query}'
-    },
-    'ja': {
-        'base': 'https://wiki.archlinuxjp.org',
-        'search': '/index.php?title=特別:検索&offset={offset}&{query}'
-    },
-    'ro': {
-        'base': 'http://wiki.archlinux.ro',
-        'search': '/index.php?title=Special:Căutare&offset={offset}&{query}'
-    },
-    'tr': {
-        'base': 'http://archtr.org/wiki',
-        'search': '/index.php?title=Özel:Ara&offset={offset}&{query}'
-    }
-    # fmt: on
-}
-
-
-# get base & search URLs for selected language
-def get_lang_urls(language):
-    if language in lang_urls:
-        return lang_urls[language]
-    return lang_urls['all']
-
-
-# Language names to build search requests for
-# those languages which are hosted on the main site.
-main_langs = {
-    'ar': 'العربية',
-    'bg': 'Български',
-    'cs': 'Česky',
-    'da': 'Dansk',
-    'el': 'Ελληνικά',
-    'es': 'Español',
-    'he': 'עברית',
-    'hr': 'Hrvatski',
-    'hu': 'Magyar',
-    'it': 'Italiano',
-    'ko': '한국어',
-    'lt': 'Lietuviškai',
-    'nl': 'Nederlands',
-    'pl': 'Polski',
-    'pt': 'Português',
-    'ru': 'Русский',
-    'sl': 'Slovenský',
-    'th': 'ไทย',
-    'uk': 'Українська',
-    'zh': '简体中文',
-}
-supported_languages = dict(lang_urls, **main_langs)
-
-
-# do search-request
 def request(query, params):
-    # translate the locale (e.g. 'en-US') to language code ('en')
-    language = locale_to_lang_code(params['language'])
 
-    # if our language is hosted on the main site, we need to add its name
-    # to the query in order to narrow the results to that language
-    if language in main_langs:
-        query += ' (' + main_langs[language] + ')'
-
-    # prepare the request parameters
-    query = urlencode({'search': query})
+    sxng_lang = params['searxng_locale'].split('-')[0]
+    netloc = traits.custom['wiki_netloc'].get(sxng_lang, main_wiki)
+    title = traits.custom['title'].get(sxng_lang, 'Special:Search')
+    base_url = 'https://' + netloc + '/index.php?'
     offset = (params['pageno'] - 1) * 20
 
-    # get request URLs for our language of choice
-    urls = get_lang_urls(language)
-    search_url = urls['base'] + urls['search']
+    if netloc == main_wiki:
+        eng_lang: str = traits.get_language(sxng_lang, 'English')
+        query += ' (' + eng_lang + ')'
+    elif netloc == 'wiki.archlinuxcn.org':
+        base_url = 'https://' + netloc + '/wzh/index.php?'
 
-    params['url'] = search_url.format(query=query, offset=offset)
+    args = {
+        'search': query,
+        'title': title,
+        'limit': 20,
+        'offset': offset,
+        'profile': 'default',
+    }
 
+    params['url'] = base_url + urlencode(args)
     return params
 
 
-# get response from search-request
 def response(resp):
-    # get the base URL for the language in which request was made
-    language = locale_to_lang_code(resp.search_params['language'])
-    base_url = get_lang_urls(language)['base']
 
     results = []
+    dom = lxml.html.fromstring(resp.text)
 
-    dom = html.fromstring(resp.text)
+    # get the base URL for the language in which request was made
+    sxng_lang = resp.search_params['searxng_locale'].split('-')[0]
+    netloc = traits.custom['wiki_netloc'].get(sxng_lang, main_wiki)
+    base_url = 'https://' + netloc + '/index.php?'
 
-    # parse results
-    for result in eval_xpath_list(dom, xpath_results):
-        link = eval_xpath_getindex(result, xpath_link, 0)
-        href = urljoin(base_url, link.attrib.get('href'))
-        title = extract_text(link)
-
-        results.append({'url': href, 'title': title})
+    for result in eval_xpath_list(dom, '//ul[@class="mw-search-results"]/li'):
+        link = eval_xpath_getindex(result, './/div[@class="mw-search-result-heading"]/a', 0)
+        content = extract_text(result.xpath('.//div[@class="searchresult"]'))
+        results.append(
+            {
+                'url': urljoin(base_url, link.get('href')),
+                'title': extract_text(link),
+                'content': content,
+            }
+        )
 
     return results
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages from Archlinix-Wiki.  The location of the Wiki address of a
+    language is mapped in a :py:obj:`custom field
+    <searx.enginelib.traits.EngineTraits.custom>` (``wiki_netloc``).  Depending
+    on the location, the ``title`` argument in the request is translated.
+
+    .. code:: python
+
+       "custom": {
+         "wiki_netloc": {
+           "de": "wiki.archlinux.de",
+            # ...
+           "zh": "wiki.archlinuxcn.org"
+         }
+         "title": {
+           "de": "Spezial:Suche",
+            # ...
+           "zh": "Special:\u641c\u7d22"
+         },
+       },
+
+    """
+
+    engine_traits.custom['wiki_netloc'] = {}
+    engine_traits.custom['title'] = {}
+
+    title_map = {
+        'de': 'Spezial:Suche',
+        'fa': 'ویژه:جستجو',
+        'ja': '特別:検索',
+        'zh': 'Special:搜索',
+    }
+
+    resp = network.get('https://wiki.archlinux.org/')
+    if not resp.ok:
+        print("ERROR: response from wiki.archlinix.org is not OK.")
+
+    dom = lxml.html.fromstring(resp.text)
+    for a in eval_xpath_list(dom, "//a[@class='interlanguage-link-target']"):
+
+        sxng_tag = language_tag(babel.Locale.parse(a.get('lang'), sep='-'))
+        # zh_Hans --> zh
+        sxng_tag = sxng_tag.split('_')[0]
+
+        netloc = urlparse(a.get('href')).netloc
+        if netloc != 'wiki.archlinux.org':
+            title = title_map.get(sxng_tag)
+            if not title:
+                print("ERROR: title tag from %s (%s) is unknown" % (netloc, sxng_tag))
+                continue
+            engine_traits.custom['wiki_netloc'][sxng_tag] = netloc
+            engine_traits.custom['title'][sxng_tag] = title
+
+        eng_tag = extract_text(eval_xpath_list(a, ".//span"))
+        engine_traits.languages[sxng_tag] = eng_tag
+
+    engine_traits.languages['en'] = 'English'

searx/engines/bing.py

@@ -1,16 +1,53 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""Bing (Web)
+"""This is the implementation of the Bing-WEB engine. Some of this
+implementations are shared by other engines:
+
+- :ref:`bing images engine`
+- :ref:`bing news engine`
+- :ref:`bing videos engine`
+
+On the `preference page`_ Bing offers a lot of languages an regions (see section
+'Search results languages' and 'Country/region').  However, the abundant choice
+does not correspond to reality, where Bing has a full-text indexer only for a
+limited number of languages.  By example: you can select a language like Māori
+but you never get a result in this language.
+
+What comes a bit closer to the truth are the `search-APIs`_ but they don`t seem
+to be completely correct either (if you take a closer look you will find some
+inaccuracies there too):
+
+- :py:obj:`searx.engines.bing.bing_traits_url`
+- :py:obj:`searx.engines.bing_videos.bing_traits_url`
+- :py:obj:`searx.engines.bing_images.bing_traits_url`
+- :py:obj:`searx.engines.bing_news.bing_traits_url`
+
+.. _preference page: https://www.bing.com/account/general
+.. _search-APIs: https://learn.microsoft.com/en-us/bing/search-apis/
 
-- https://github.com/searx/searx/issues/2019#issuecomment-648227442
 """
-# pylint: disable=too-many-branches
+# pylint: disable=too-many-branches, invalid-name
 
+from typing import TYPE_CHECKING
+import datetime
 import re
-from urllib.parse import urlencode, urlparse, parse_qs
+import uuid
+from urllib.parse import urlencode
 from lxml import html
-from searx.utils import eval_xpath, extract_text, eval_xpath_list, match_language, eval_xpath_getindex
-from searx.network import multi_requests, Request
+import babel
+import babel.languages
+
+from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_getindex
+from searx import network
+from searx.locales import language_tag, region_tag
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
 about = {
     "website": 'https://www.bing.com',
@@ -21,56 +58,124 @@ about = {
     "results": 'HTML',
 }
 
+send_accept_language_header = True
+"""Bing tries to guess user's language and territory from the HTTP
+Accept-Language.  Optional the user can select a search-language (can be
+different to the UI language) and a region (market code)."""
+
 # engine dependent config
 categories = ['general', 'web']
 paging = True
-time_range_support = False
-safesearch = False
-send_accept_language_header = True
-supported_languages_url = 'https://www.bing.com/account/general'
-language_aliases = {}
+time_range_support = True
+safesearch = True
+safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'}  # cookie: ADLT=STRICT
 
-# search-url
-base_url = 'https://www.bing.com/'
+base_url = 'https://www.bing.com/search'
+"""Bing (Web) search URL"""
 
-# initial query:     https://www.bing.com/search?q=foo&search=&form=QBLH
-inital_query = 'search?{query}&search=&form=QBLH'
-
-# following queries: https://www.bing.com/search?q=foo&search=&first=11&FORM=PERE
-page_query = 'search?{query}&search=&first={offset}&FORM=PERE'
+bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes'
+"""Bing (Web) search API description"""
 
 
 def _get_offset_from_pageno(pageno):
     return (pageno - 1) * 10 + 1
 
 
+def set_bing_cookies(params, engine_language, engine_region, SID):
+
+    # set cookies
+    # -----------
+
+    params['cookies']['_EDGE_V'] = '1'
+
+    # _EDGE_S: F=1&SID=3A5253BD6BCA609509B741876AF961CA&mkt=zh-tw
+    _EDGE_S = [
+        'F=1',
+        'SID=%s' % SID,
+        'mkt=%s' % engine_region.lower(),
+        'ui=%s' % engine_language.lower(),
+    ]
+    params['cookies']['_EDGE_S'] = '&'.join(_EDGE_S)
+    logger.debug("cookie _EDGE_S=%s", params['cookies']['_EDGE_S'])
+
+    # "_EDGE_CD": "m=zh-tw",
+
+    _EDGE_CD = [  # pylint: disable=invalid-name
+        'm=%s' % engine_region.lower(),  # search region: zh-cn
+        'u=%s' % engine_language.lower(),  # UI: en-us
+    ]
+
+    params['cookies']['_EDGE_CD'] = '&'.join(_EDGE_CD) + ';'
+    logger.debug("cookie _EDGE_CD=%s", params['cookies']['_EDGE_CD'])
+
+    SRCHHPGUSR = [  # pylint: disable=invalid-name
+        'SRCHLANG=%s' % engine_language,
+        # Trying to set ADLT cookie here seems not to have any effect, I assume
+        # there is some age verification by a cookie (and/or session ID) needed,
+        # to disable the SafeSearch.
+        'ADLT=%s' % safesearch_types.get(params['safesearch'], 'DEMOTE'),
+    ]
+    params['cookies']['SRCHHPGUSR'] = '&'.join(SRCHHPGUSR)
+    logger.debug("cookie SRCHHPGUSR=%s", params['cookies']['SRCHHPGUSR'])
+
+
 def request(query, params):
+    """Assemble a Bing-Web request."""
 
-    offset = _get_offset_from_pageno(params.get('pageno', 1))
+    engine_region = traits.get_region(params['searxng_locale'], 'en-US')
+    engine_language = traits.get_language(params['searxng_locale'], 'en')
 
-    # logger.debug("params['pageno'] --> %s", params.get('pageno'))
-    # logger.debug("          offset --> %s", offset)
+    SID = uuid.uuid1().hex.upper()
+    CVID = uuid.uuid1().hex.upper()
 
-    search_string = page_query
-    if offset == 1:
-        search_string = inital_query
+    set_bing_cookies(params, engine_language, engine_region, SID)
 
-    if params['language'] == 'all':
-        lang = 'EN'
-    else:
-        lang = match_language(params['language'], supported_languages, language_aliases)
+    # build URL query
+    # ---------------
 
-    query = 'language:{} {}'.format(lang.split('-')[0].upper(), query)
+    # query term
+    page = int(params.get('pageno', 1))
+    query_params = {
+        # fmt: off
+        'q': query,
+        'pq': query,
+        'cvid': CVID,
+        'qs': 'n',
+        'sp': '-1'
+        # fmt: on
+    }
 
-    search_path = search_string.format(query=urlencode({'q': query}), offset=offset)
-
-    if offset > 1:
-        referer = base_url + inital_query.format(query=urlencode({'q': query}))
+    # page
+    if page > 1:
+        referer = base_url + '?' + urlencode(query_params)
         params['headers']['Referer'] = referer
         logger.debug("headers.Referer --> %s", referer)
 
-    params['url'] = base_url + search_path
-    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+    query_params['first'] = _get_offset_from_pageno(page)
+
+    if page == 2:
+        query_params['FORM'] = 'PERE'
+    elif page > 2:
+        query_params['FORM'] = 'PERE%s' % (page - 2)
+
+    filters = ''
+    if params['time_range']:
+        query_params['filt'] = 'custom'
+
+        if params['time_range'] == 'day':
+            filters = 'ex1:"ez1"'
+        elif params['time_range'] == 'week':
+            filters = 'ex1:"ez2"'
+        elif params['time_range'] == 'month':
+            filters = 'ex1:"ez3"'
+        elif params['time_range'] == 'year':
+            epoch_1970 = datetime.date(1970, 1, 1)
+            today_no = (datetime.date.today() - epoch_1970).days
+            filters = 'ex1:"ez5_%s_%s"' % (today_no - 365, today_no)
+
+    params['url'] = base_url + '?' + urlencode(query_params)
+    if filters:
+        params['url'] = params['url'] + '&filters=' + filters
     return params
 
 
@@ -107,7 +212,8 @@ def response(resp):
             url_cite = extract_text(eval_xpath(result, './/div[@class="b_attribution"]/cite'))
             # Bing can shorten the URL either at the end or in the middle of the string
             if (
-                url_cite.startswith('https://')
+                url_cite
+                and url_cite.startswith('https://')
                 and '…' not in url_cite
                 and '...' not in url_cite
                 and '›' not in url_cite
@@ -127,9 +233,9 @@ def response(resp):
 
     # resolve all Bing redirections in parallel
     request_list = [
-        Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve
+        network.Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve
     ]
-    response_list = multi_requests(request_list)
+    response_list = network.multi_requests(request_list)
     for i, redirect_response in enumerate(response_list):
         if not isinstance(redirect_response, Exception):
             results[url_to_resolve_index[i]]['url'] = redirect_response.headers['location']
@@ -157,27 +263,71 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-Web."""
 
-    lang_tags = set()
+    xpath_market_codes = '//table[1]/tbody/tr/td[3]'
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
+
+    _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes)
+
+
+def _fetch_traits(engine_traits: EngineTraits, url: str, xpath_language_codes: str, xpath_market_codes: str):
+
+    # insert alias to map from a language (zh) to a language + script (zh_Hans)
+    engine_traits.languages['zh'] = 'zh-hans'
+
+    resp = network.get(url)
+
+    if not resp.ok:
+        print("ERROR: response from peertube is not OK.")
 
     dom = html.fromstring(resp.text)
-    lang_links = eval_xpath(dom, '//div[@id="language-section"]//li')
 
-    for _li in lang_links:
+    map_lang = {'jp': 'ja'}
+    for td in eval_xpath(dom, xpath_language_codes):
+        eng_lang = td.text
 
-        href = eval_xpath(_li, './/@href')[0]
-        (_scheme, _netloc, _path, _params, query, _fragment) = urlparse(href)
-        query = parse_qs(query, keep_blank_values=True)
+        if eng_lang in ('en-gb', 'pt-br'):
+            # language 'en' is already in the list and a language 'en-gb' can't
+            # be handled in SearXNG, same with pt-br which is covered by pt-pt.
+            continue
 
-        # fmt: off
-        setlang = query.get('setlang', [None, ])[0]
-        # example: 'mn-Cyrl-MN' --> '['mn', 'Cyrl-MN']
-        lang, nation = (setlang.split('-', maxsplit=1) + [None,])[:2]  # fmt: skip
-        # fmt: on
+        babel_lang = map_lang.get(eng_lang, eng_lang).replace('-', '_')
+        try:
+            sxng_tag = language_tag(babel.Locale.parse(babel_lang))
+        except babel.UnknownLocaleError:
+            print("ERROR: language (%s) is unknown by babel" % (eng_lang))
+            continue
+        conflict = engine_traits.languages.get(sxng_tag)
+        if conflict:
+            if conflict != eng_lang:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_lang))
+            continue
+        engine_traits.languages[sxng_tag] = eng_lang
 
-        tag = lang + '-' + nation if nation else lang
-        lang_tags.add(tag)
+    map_region = {
+        'en-ID': 'id_ID',
+        'no-NO': 'nb_NO',
+    }
 
-    return list(lang_tags)
+    for td in eval_xpath(dom, xpath_market_codes):
+        eng_region = td.text
+        babel_region = map_region.get(eng_region, eng_region).replace('-', '_')
+
+        if eng_region == 'en-WW':
+            engine_traits.all_locale = eng_region
+            continue
+
+        try:
+            sxng_tag = region_tag(babel.Locale.parse(babel_region))
+        except babel.UnknownLocaleError:
+            print("ERROR: region (%s) is unknown by babel" % (eng_region))
+            continue
+        conflict = engine_traits.regions.get(sxng_tag)
+        if conflict:
+            if conflict != eng_region:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_region))
+            continue
+        engine_traits.regions[sxng_tag] = eng_region

searx/engines/bing_images.py

@@ -1,20 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""Bing (Images)
-
+"""Bing-Images: description see :py:obj:`searx.engines.bing`.
 """
+# pylint: disable=invalid-name
 
-from json import loads
+
+from typing import TYPE_CHECKING
+import uuid
+import json
 from urllib.parse import urlencode
 
 from lxml import html
 
-from searx.utils import match_language
-from searx.engines.bing import language_aliases
-from searx.engines.bing import (  # pylint: disable=unused-import
-    _fetch_supported_languages,
-    supported_languages_url,
+from searx.enginelib.traits import EngineTraits
+from searx.engines.bing import (
+    set_bing_cookies,
+    _fetch_traits,
 )
+from searx.engines.bing import send_accept_language_header  # pylint: disable=unused-import
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
@@ -31,77 +41,92 @@ categories = ['images', 'web']
 paging = True
 safesearch = True
 time_range_support = True
-send_accept_language_header = True
-supported_languages_url = 'https://www.bing.com/account/general'
-number_of_results = 28
 
-# search-url
-base_url = 'https://www.bing.com/'
-search_string = (
+base_url = 'https://www.bing.com/images/async'
+"""Bing (Images) search URL"""
+
+bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-image-search/reference/market-codes'
+"""Bing (Images) search API description"""
+
+time_map = {
     # fmt: off
-    'images/search'
-    '?{query}'
-    '&count={count}'
-    '&first={first}'
-    '&tsc=ImageHoverTitle'
+    'day': 60 * 24,
+    'week': 60 * 24 * 7,
+    'month': 60 * 24 * 31,
+    'year': 60 * 24 * 365,
     # fmt: on
-)
-time_range_string = '&qft=+filterui:age-lt{interval}'
-time_range_dict = {'day': '1440', 'week': '10080', 'month': '43200', 'year': '525600'}
-
-# safesearch definitions
-safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'}
+}
 
 
-# do search-request
 def request(query, params):
-    offset = ((params['pageno'] - 1) * number_of_results) + 1
+    """Assemble a Bing-Image request."""
 
-    search_path = search_string.format(query=urlencode({'q': query}), count=number_of_results, first=offset)
+    engine_region = traits.get_region(params['searxng_locale'], 'en-US')
+    engine_language = traits.get_language(params['searxng_locale'], 'en')
 
-    language = match_language(params['language'], supported_languages, language_aliases).lower()
+    SID = uuid.uuid1().hex.upper()
+    set_bing_cookies(params, engine_language, engine_region, SID)
 
-    params['cookies']['SRCHHPGUSR'] = 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
+    # build URL query
+    # - example: https://www.bing.com/images/async?q=foo&first=155&count=35
 
-    params['cookies']['_EDGE_S'] = 'mkt=' + language + '&ui=' + language + '&F=1'
+    query_params = {
+        # fmt: off
+        'q': query,
+        'async' : 'content',
+        # to simplify the page count lets use the default of 35 images per page
+        'first' : (int(params.get('pageno', 1)) - 1) * 35 + 1,
+        'count' : 35,
+        # fmt: on
+    }
 
-    params['url'] = base_url + search_path
-    if params['time_range'] in time_range_dict:
-        params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']])
+    # time range
+    # - example: one year (525600 minutes) 'qft=+filterui:age-lt525600'
+
+    if params['time_range']:
+        query_params['qft'] = 'filterui:age-lt%s' % time_map[params['time_range']]
+
+    params['url'] = base_url + '?' + urlencode(query_params)
 
     return params
 
 
-# get response from search-request
 def response(resp):
-    results = []
+    """Get response from Bing-Images"""
 
+    results = []
     dom = html.fromstring(resp.text)
 
-    # parse results
-    for result in dom.xpath('//div[@class="imgpt"]'):
-        img_format = result.xpath('./div[contains(@class, "img_info")]/span/text()')[0]
-        # Microsoft seems to experiment with this code so don't make the path too specific,
-        # just catch the text section for the first anchor in img_info assuming this to be
-        # the originating site.
-        source = result.xpath('./div[contains(@class, "img_info")]//a/text()')[0]
+    for result in dom.xpath('//ul[contains(@class, "dgControl_list")]/li'):
 
-        m = loads(result.xpath('./a/@m')[0])
+        metadata = result.xpath('.//a[@class="iusc"]/@m')
+        if not metadata:
+            continue
 
-        # strip 'Unicode private use area' highlighting, they render to Tux
-        # the Linux penguin and a standing diamond on my machine...
-        title = m.get('t', '').replace('\ue000', '').replace('\ue001', '')
+        metadata = json.loads(result.xpath('.//a[@class="iusc"]/@m')[0])
+        title = ' '.join(result.xpath('.//div[@class="infnmpt"]//a/text()')).strip()
+        img_format = ' '.join(result.xpath('.//div[@class="imgpt"]/div/span/text()')).strip()
+        source = ' '.join(result.xpath('.//div[@class="imgpt"]//div[@class="lnkw"]//a/text()')).strip()
         results.append(
             {
                 'template': 'images.html',
-                'url': m['purl'],
-                'thumbnail_src': m['turl'],
-                'img_src': m['murl'],
-                'content': '',
+                'url': metadata['purl'],
+                'thumbnail_src': metadata['turl'],
+                'img_src': metadata['murl'],
+                'content': metadata['desc'],
                 'title': title,
                 'source': source,
                 'img_format': img_format,
             }
         )
-
     return results
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-News."""
+
+    xpath_market_codes = '//table[1]/tbody/tr/td[3]'
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
+
+    _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes)

searx/engines/bing_news.py

@@ -1,24 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""Bing (News)
+"""Bing-News: description see :py:obj:`searx.engines.bing`.
 """
 
-from urllib.parse import (
-    urlencode,
-    urlparse,
-    parse_qsl,
-    quote,
-)
-from datetime import datetime
-from dateutil import parser
-from lxml import etree
-from lxml.etree import XPath
-from searx.utils import match_language, eval_xpath_getindex
-from searx.engines.bing import (  # pylint: disable=unused-import
-    language_aliases,
-    _fetch_supported_languages,
-    supported_languages_url,
+# pylint: disable=invalid-name
+
+from typing import TYPE_CHECKING
+import uuid
+from urllib.parse import urlencode
+
+from lxml import html
+
+from searx.enginelib.traits import EngineTraits
+from searx.engines.bing import (
+    set_bing_cookies,
+    _fetch_traits,
 )
+from searx.engines.bing import send_accept_language_header  # pylint: disable=unused-import
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
+
 
 # about
 about = {
@@ -34,108 +40,111 @@ about = {
 categories = ['news']
 paging = True
 time_range_support = True
-send_accept_language_header = True
+time_map = {
+    'day': '4',
+    'week': '8',
+    'month': '9',
+}
+"""A string '4' means *last hour*. We use *last hour* for ``day`` here since the
+difference of *last day* and *last week* in the result list is just marginally.
+"""
 
-# search-url
-base_url = 'https://www.bing.com/'
-search_string = 'news/search?{query}&first={offset}&format=RSS'
-search_string_with_time = 'news/search?{query}&first={offset}&qft=interval%3d"{interval}"&format=RSS'
-time_range_dict = {'day': '7', 'week': '8', 'month': '9'}
+base_url = 'https://www.bing.com/news/infinitescrollajax'
+"""Bing (News) search URL"""
 
+bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-news-search/reference/market-codes'
+"""Bing (News) search API description"""
 
-def url_cleanup(url_string):
-    """remove click"""
-
-    parsed_url = urlparse(url_string)
-    if parsed_url.netloc == 'www.bing.com' and parsed_url.path == '/news/apiclick.aspx':
-        query = dict(parse_qsl(parsed_url.query))
-        url_string = query.get('url', None)
-    return url_string
-
-
-def image_url_cleanup(url_string):
-    """replace the http://*bing.com/th?id=... by https://www.bing.com/th?id=..."""
-
-    parsed_url = urlparse(url_string)
-    if parsed_url.netloc.endswith('bing.com') and parsed_url.path == '/th':
-        query = dict(parse_qsl(parsed_url.query))
-        url_string = "https://www.bing.com/th?id=" + quote(query.get('id'))
-    return url_string
-
-
-def _get_url(query, language, offset, time_range):
-    if time_range in time_range_dict:
-        search_path = search_string_with_time.format(
-            # fmt: off
-            query = urlencode({
-                'q': query,
-                'setmkt': language
-            }),
-            offset = offset,
-            interval = time_range_dict[time_range]
-            # fmt: on
-        )
-    else:
-        # e.g. setmkt=de-de&setlang=de
-        search_path = search_string.format(
-            # fmt: off
-            query = urlencode({
-                'q': query,
-                'setmkt': language
-            }),
-            offset = offset
-            # fmt: on
-        )
-    return base_url + search_path
+mkt_alias = {
+    'zh': 'en-WW',
+    'zh-CN': 'en-WW',
+}
+"""Bing News has an official market code 'zh-CN' but we won't get a result with
+this market code.  For 'zh' and 'zh-CN' we better use the *Worldwide aggregate*
+market code (en-WW).
+"""
 
 
 def request(query, params):
+    """Assemble a Bing-News request."""
 
-    if params['time_range'] and params['time_range'] not in time_range_dict:
-        return params
+    sxng_locale = params['searxng_locale']
+    engine_region = traits.get_region(mkt_alias.get(sxng_locale, sxng_locale), traits.all_locale)
+    engine_language = traits.get_language(sxng_locale, 'en')
 
-    offset = (params['pageno'] - 1) * 10 + 1
-    if params['language'] == 'all':
-        language = 'en-US'
-    else:
-        language = match_language(params['language'], supported_languages, language_aliases)
-    params['url'] = _get_url(query, language, offset, params['time_range'])
+    SID = uuid.uuid1().hex.upper()
+    set_bing_cookies(params, engine_language, engine_region, SID)
+
+    # build URL query
+    #
+    # example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
+
+    query_params = {
+        # fmt: off
+        'q': query,
+        'InfiniteScroll': 1,
+        # to simplify the page count lets use the default of 10 images per page
+        'first' : (int(params.get('pageno', 1)) - 1) * 10 + 1,
+        # fmt: on
+    }
+
+    if params['time_range']:
+        # qft=interval:"7"
+        query_params['qft'] = 'qft=interval="%s"' % time_map.get(params['time_range'], '9')
+
+    params['url'] = base_url + '?' + urlencode(query_params)
 
     return params
 
 
 def response(resp):
-
+    """Get response from Bing-Video"""
     results = []
-    rss = etree.fromstring(resp.content)
-    namespaces = rss.nsmap
 
-    for item in rss.xpath('./channel/item'):
-        # url / title / content
-        url = url_cleanup(eval_xpath_getindex(item, './link/text()', 0, default=None))
-        title = eval_xpath_getindex(item, './title/text()', 0, default=url)
-        content = eval_xpath_getindex(item, './description/text()', 0, default='')
+    if not resp.ok or not resp.text:
+        return results
 
-        # publishedDate
-        publishedDate = eval_xpath_getindex(item, './pubDate/text()', 0, default=None)
-        try:
-            publishedDate = parser.parse(publishedDate, dayfirst=False)
-        except TypeError:
-            publishedDate = datetime.now()
-        except ValueError:
-            publishedDate = datetime.now()
+    dom = html.fromstring(resp.text)
 
-        # thumbnail
-        thumbnail = eval_xpath_getindex(item, XPath('./News:Image/text()', namespaces=namespaces), 0, default=None)
-        if thumbnail is not None:
-            thumbnail = image_url_cleanup(thumbnail)
+    for newsitem in dom.xpath('//div[contains(@class, "newsitem")]'):
+
+        url = newsitem.xpath('./@url')[0]
+        title = ' '.join(newsitem.xpath('.//div[@class="caption"]//a[@class="title"]/text()')).strip()
+        content = ' '.join(newsitem.xpath('.//div[@class="snippet"]/text()')).strip()
+        thumbnail = None
+        author = newsitem.xpath('./@data-author')[0]
+        metadata = ' '.join(newsitem.xpath('.//div[@class="source"]/span/text()')).strip()
+
+        img_src = newsitem.xpath('.//a[@class="imagelink"]//img/@src')
+        if img_src:
+            thumbnail = 'https://www.bing.com/' + img_src[0]
 
-        # append result
-        if thumbnail is not None:
         results.append(
-                {'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content, 'img_src': thumbnail}
+            {
+                'url': url,
+                'title': title,
+                'content': content,
+                'img_src': thumbnail,
+                'author': author,
+                'metadata': metadata,
+            }
         )
-        else:
-            results.append({'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content})
 
     return results
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-News.
+
+    The :py:obj:`description <searx.engines.bing_news.bing_traits_url>` of the
+    first table says *"query parameter when calling the Video Search API."*
+    .. thats why I use the 4. table "News Category API markets" for the
+    ``xpath_market_codes``.
+
+    """
+
+    xpath_market_codes = '//table[4]/tbody/tr/td[3]'
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
+
+    _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes)

searx/engines/bing_videos.py

@@ -1,21 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""Bing (Videos)
-
+"""Bing-Videos: description see :py:obj:`searx.engines.bing`.
 """
+# pylint: disable=invalid-name
 
-from json import loads
+from typing import TYPE_CHECKING
+import uuid
+import json
 from urllib.parse import urlencode
 
 from lxml import html
 
-from searx.utils import match_language
-from searx.engines.bing import language_aliases
-
-from searx.engines.bing import (  # pylint: disable=unused-import
-    _fetch_supported_languages,
-    supported_languages_url,
+from searx.enginelib.traits import EngineTraits
+from searx.engines.bing import (
+    set_bing_cookies,
+    _fetch_traits,
 )
+from searx.engines.bing import send_accept_language_header  # pylint: disable=unused-import
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
+
 
 about = {
     "website": 'https://www.bing.com/videos',
@@ -26,65 +35,76 @@ about = {
     "results": 'HTML',
 }
 
+# engine dependent config
 categories = ['videos', 'web']
 paging = True
 safesearch = True
 time_range_support = True
-send_accept_language_header = True
-number_of_results = 28
 
-base_url = 'https://www.bing.com/'
-search_string = (
+base_url = 'https://www.bing.com/videos/asyncv2'
+"""Bing (Videos) async search URL."""
+
+bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-video-search/reference/market-codes'
+"""Bing (Video) search API description"""
+
+time_map = {
     # fmt: off
-    'videos/search'
-    '?{query}'
-    '&count={count}'
-    '&first={first}'
-    '&scope=video'
-    '&FORM=QBLH'
+    'day': 60 * 24,
+    'week': 60 * 24 * 7,
+    'month': 60 * 24 * 31,
+    'year': 60 * 24 * 365,
     # fmt: on
-)
-time_range_string = '&qft=+filterui:videoage-lt{interval}'
-time_range_dict = {'day': '1440', 'week': '10080', 'month': '43200', 'year': '525600'}
-
-# safesearch definitions
-safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'}
+}
 
 
-# do search-request
 def request(query, params):
-    offset = ((params['pageno'] - 1) * number_of_results) + 1
+    """Assemble a Bing-Video request."""
 
-    search_path = search_string.format(query=urlencode({'q': query}), count=number_of_results, first=offset)
+    engine_region = traits.get_region(params['searxng_locale'], 'en-US')
+    engine_language = traits.get_language(params['searxng_locale'], 'en')
 
-    # safesearch cookie
-    params['cookies']['SRCHHPGUSR'] = 'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
+    SID = uuid.uuid1().hex.upper()
+    set_bing_cookies(params, engine_language, engine_region, SID)
 
-    # language cookie
-    language = match_language(params['language'], supported_languages, language_aliases).lower()
-    params['cookies']['_EDGE_S'] = 'mkt=' + language + '&F=1'
+    # build URL query
+    #
+    # example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
 
-    # query and paging
-    params['url'] = base_url + search_path
+    query_params = {
+        # fmt: off
+        'q': query,
+        'async' : 'content',
+        # to simplify the page count lets use the default of 35 images per page
+        'first' : (int(params.get('pageno', 1)) - 1) * 35 + 1,
+        'count' : 35,
+        # fmt: on
+    }
 
     # time range
-    if params['time_range'] in time_range_dict:
-        params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']])
+    #
+    # example: one week (10080 minutes) '&qft= filterui:videoage-lt10080'  '&form=VRFLTR'
+
+    if params['time_range']:
+        query_params['form'] = 'VRFLTR'
+        query_params['qft'] = ' filterui:videoage-lt%s' % time_map[params['time_range']]
+
+    params['url'] = base_url + '?' + urlencode(query_params)
 
     return params
 
 
-# get response from search-request
 def response(resp):
+    """Get response from Bing-Video"""
     results = []
 
     dom = html.fromstring(resp.text)
 
-    for result in dom.xpath('//div[@class="dg_u"]/div[contains(@class, "mc_vtvc")]'):
-        metadata = loads(result.xpath('.//div[@class="vrhdata"]/@vrhm')[0])
+    for result in dom.xpath('//div[@class="dg_u"]//div[contains(@id, "mc_vtvc_video")]'):
+        metadata = json.loads(result.xpath('.//div[@class="vrhdata"]/@vrhm')[0])
         info = ' - '.join(result.xpath('.//div[@class="mc_vtvc_meta_block"]//span/text()')).strip()
         content = '{0} - {1}'.format(metadata['du'], info)
-        thumbnail = '{0}th?id={1}'.format(base_url, metadata['thid'])
+        thumbnail = result.xpath('.//div[contains(@class, "mc_vtvc_th")]//img/@src')[0]
+
         results.append(
             {
                 'url': metadata['murl'],
@@ -96,3 +116,13 @@ def response(resp):
         )
 
     return results
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-Videos."""
+
+    xpath_market_codes = '//table[1]/tbody/tr/td[3]'
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
+
+    _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes)

searx/engines/dailymotion.py

@@ -1,17 +1,35 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Dailymotion (Videos)
+# lint: pylint
+"""
+Dailymotion (Videos)
+~~~~~~~~~~~~~~~~~~~~
+
+.. _REST GET: https://developers.dailymotion.com/tools/
+.. _Global API Parameters: https://developers.dailymotion.com/api/#global-parameters
+.. _Video filters API: https://developers.dailymotion.com/api/#video-filters
+.. _Fields selection: https://developers.dailymotion.com/api/#fields-selection
 
 """
 
-from typing import Set
+from typing import TYPE_CHECKING
+
 from datetime import datetime, timedelta
 from urllib.parse import urlencode
 import time
 import babel
 
 from searx.exceptions import SearxEngineAPIException
-from searx.network import raise_for_httperror
+from searx import network
 from searx.utils import html_to_text
+from searx.locales import region_tag, language_tag
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
@@ -37,11 +55,24 @@ time_delta_dict = {
 }
 
 safesearch = True
-safesearch_params = {2: '&is_created_for_kids=true', 1: '&is_created_for_kids=true', 0: ''}
+safesearch_params = {
+    2: {'is_created_for_kids': 'true'},
+    1: {'is_created_for_kids': 'true'},
+    0: {},
+}
+"""True if this video is "Created for Kids" / intends to target an audience
+under the age of 16 (``is_created_for_kids`` in `Video filters API`_ )
+"""
 
-# search-url
-# - https://developers.dailymotion.com/tools/
-# - https://www.dailymotion.com/doc/api/obj-video.html
+family_filter_map = {
+    2: 'true',
+    1: 'true',
+    0: 'false',
+}
+"""By default, the family filter is turned on. Setting this parameter to
+``false`` will stop filtering-out explicit content from searches and global
+contexts (``family_filter`` in `Global API Parameters`_ ).
+"""
 
 result_fields = [
     'allow_embed',
@@ -53,27 +84,21 @@ result_fields = [
     'thumbnail_360_url',
     'id',
 ]
-search_url = (
-    'https://api.dailymotion.com/videos?'
-    'fields={fields}&password_protected={password_protected}&private={private}&sort={sort}&limit={limit}'
-).format(
-    fields=','.join(result_fields),
-    password_protected='false',
-    private='false',
-    sort='relevance',
-    limit=number_of_results,
-)
+"""`Fields selection`_, by default, a few fields are returned. To request more
+specific fields, the ``fields`` parameter is used with the list of fields
+SearXNG needs in the response to build a video result list.
+"""
+
+search_url = 'https://api.dailymotion.com/videos?'
+"""URL to retrieve a list of videos.
+
+- `REST GET`_
+- `Global API Parameters`_
+- `Video filters API`_
+"""
+
 iframe_src = "https://www.dailymotion.com/embed/video/{video_id}"
-
-# The request query filters by 'languages' & 'country', therefore instead of
-# fetching only languages we need to fetch locales.
-supported_languages_url = 'https://api.dailymotion.com/locales'
-supported_languages_iso639: Set[str] = set()
-
-
-def init(_engine_settings):
-    global supported_languages_iso639
-    supported_languages_iso639 = set([language.split('_')[0] for language in supported_languages])
+"""URL template to embed video in SearXNG's result list."""
 
 
 def request(query, params):
@@ -81,34 +106,42 @@ def request(query, params):
     if not query:
         return False
 
-    language = params['language']
-    if language == 'all':
-        language = 'en-US'
-    locale = babel.Locale.parse(language, sep='-')
+    eng_region = traits.get_region(params['searxng_locale'], 'en_US')
+    eng_lang = traits.get_language(params['searxng_locale'], 'en')
 
-    language_iso639 = locale.language
-    if locale.language not in supported_languages_iso639:
-        language_iso639 = 'en'
-
-    query_args = {
+    args = {
         'search': query,
-        'languages': language_iso639,
+        'family_filter': family_filter_map.get(params['safesearch'], 'false'),
+        'thumbnail_ratio': 'original',  # original|widescreen|square
+        # https://developers.dailymotion.com/api/#video-filters
+        'languages': eng_lang,
         'page': params['pageno'],
+        'password_protected': 'false',
+        'private': 'false',
+        'sort': 'relevance',
+        'limit': number_of_results,
+        'fields': ','.join(result_fields),
     }
 
-    if locale.territory:
-        localization = locale.language + '_' + locale.territory
-        if localization in supported_languages:
-            query_args['country'] = locale.territory
+    args.update(safesearch_params.get(params['safesearch'], {}))
+
+    # Don't add localization and country arguments if the user does select a
+    # language (:de, :en, ..)
+
+    if len(params['searxng_locale'].split('-')) > 1:
+        # https://developers.dailymotion.com/api/#global-parameters
+        args['localization'] = eng_region
+        args['country'] = eng_region.split('_')[1]
+        # Insufficient rights for the `ams_country' parameter of route `GET /videos'
+        # 'ams_country': eng_region.split('_')[1],
 
     time_delta = time_delta_dict.get(params["time_range"])
     if time_delta:
         created_after = datetime.now() - time_delta
-        query_args['created_after'] = datetime.timestamp(created_after)
+        args['created_after'] = datetime.timestamp(created_after)
 
-    query_str = urlencode(query_args)
-    params['url'] = search_url + '&' + query_str + safesearch_params.get(params['safesearch'], '')
-    params['raise_for_httperror'] = False
+    query_str = urlencode(args)
+    params['url'] = search_url + query_str
 
     return params
 
@@ -123,7 +156,7 @@ def response(resp):
     if 'error' in search_res:
         raise SearxEngineAPIException(search_res['error'].get('message'))
 
-    raise_for_httperror(resp)
+    network.raise_for_httperror(resp)
 
     # parse results
     for res in search_res.get('list', []):
@@ -167,7 +200,53 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
-    response_json = resp.json()
-    return [item['locale'] for item in response_json['list']]
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch locales & languages from dailymotion.
+
+    Locales fetched from `api/locales <https://api.dailymotion.com/locales>`_.
+    There are duplications in the locale codes returned from Dailymotion which
+    can be ignored::
+
+      en_EN --> en_GB, en_US
+      ar_AA --> ar_EG, ar_AE, ar_SA
+
+    The language list `api/languages <https://api.dailymotion.com/languages>`_
+    contains over 7000 *languages* codes (see PR1071_).  We use only those
+    language codes that are used in the locales.
+
+    .. _PR1071: https://github.com/searxng/searxng/pull/1071
+
+    """
+
+    resp = network.get('https://api.dailymotion.com/locales')
+    if not resp.ok:
+        print("ERROR: response from dailymotion/locales is not OK.")
+
+    for item in resp.json()['list']:
+        eng_tag = item['locale']
+        if eng_tag in ('en_EN', 'ar_AA'):
+            continue
+        try:
+            sxng_tag = region_tag(babel.Locale.parse(eng_tag))
+        except babel.UnknownLocaleError:
+            print("ERROR: item unknown --> %s" % item)
+            continue
+
+        conflict = engine_traits.regions.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+        engine_traits.regions[sxng_tag] = eng_tag
+
+    locale_lang_list = [x.split('_')[0] for x in engine_traits.regions.values()]
+
+    resp = network.get('https://api.dailymotion.com/languages')
+    if not resp.ok:
+        print("ERROR: response from dailymotion/languages is not OK.")
+
+    for item in resp.json()['list']:
+        eng_tag = item['code']
+        if eng_tag in locale_lang_list:
+            sxng_tag = language_tag(babel.Locale.parse(eng_tag))
+            engine_traits.languages[sxng_tag] = eng_tag

searx/engines/demo_offline.py

@@ -63,7 +63,7 @@ def search(query, request_params):
     for row in result_list:
         entry = {
             'query': query,
-            'language': request_params['language'],
+            'language': request_params['searxng_locale'],
             'value': row.get("value"),
             # choose a result template or comment out to use the *default*
             'template': 'key-value.html',

searx/engines/duckduckgo.py

@@ -1,71 +1,220 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""DuckDuckGo Lite
+"""
+DuckDuckGo Lite
+~~~~~~~~~~~~~~~
 """
 
-from json import loads
-
-from lxml.html import fromstring
+from typing import TYPE_CHECKING
+import re
+from urllib.parse import urlencode
+import json
+import babel
+import lxml.html
 
+from searx import (
+    network,
+    locales,
+    redislib,
+    external_bang,
+)
+from searx import redisdb
 from searx.utils import (
-    dict_subset,
     eval_xpath,
     eval_xpath_getindex,
     extract_text,
-    match_language,
 )
-from searx.network import get
+from searx.enginelib.traits import EngineTraits
+from searx.exceptions import SearxEngineAPIException
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
-# about
 about = {
     "website": 'https://lite.duckduckgo.com/lite/',
     "wikidata_id": 'Q12805',
-    "official_api_documentation": 'https://duckduckgo.com/api',
     "use_official_api": False,
     "require_api_key": False,
     "results": 'HTML',
 }
 
+send_accept_language_header = True
+"""DuckDuckGo-Lite tries to guess user's prefered language from the HTTP
+``Accept-Language``.  Optional the user can select a region filter (but not a
+language).
+"""
+
 # engine dependent config
 categories = ['general', 'web']
 paging = True
-supported_languages_url = 'https://duckduckgo.com/util/u588.js'
 time_range_support = True
-send_accept_language_header = True
+safesearch = True  # user can't select but the results are filtered
 
-language_aliases = {
-    'ar-SA': 'ar-XA',
-    'es-419': 'es-XL',
-    'ja': 'jp-JP',
-    'ko': 'kr-KR',
-    'sl-SI': 'sl-SL',
-    'zh-TW': 'tzh-TW',
-    'zh-HK': 'tzh-HK',
-}
+url = 'https://lite.duckduckgo.com/lite/'
+# url_ping = 'https://duckduckgo.com/t/sl_l'
 
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
+form_data = {'v': 'l', 'api': 'd.js', 'o': 'json'}
 
-# search-url
-url = 'https://lite.duckduckgo.com/lite/'
-url_ping = 'https://duckduckgo.com/t/sl_l'
 
-# match query's language to a region code that duckduckgo will accept
-def get_region_code(lang, lang_list=None):
-    if lang == 'all':
-        return None
+def cache_vqd(query, value):
+    """Caches a ``vqd`` value from a query.
 
-    lang_code = match_language(lang, lang_list or [], language_aliases, 'wt-WT')
-    lang_parts = lang_code.split('-')
+    The vqd value depends on the query string and is needed for the follow up
+    pages or the images loaded by a XMLHttpRequest:
 
-    # country code goes first
-    return lang_parts[1].lower() + '-' + lang_parts[0].lower()
+    - DuckDuckGo Web: `https://links.duckduckgo.com/d.js?q=...&vqd=...`
+    - DuckDuckGo Images: `https://duckduckgo.com/i.js??q=...&vqd=...`
+
+    """
+    c = redisdb.client()
+    if c:
+        logger.debug("cache vqd value: %s", value)
+        key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query)
+        c.set(key, value, ex=600)
+
+
+def get_vqd(query, headers):
+    """Returns the ``vqd`` that fits to the *query*.  If there is no ``vqd`` cached
+    (:py:obj:`cache_vqd`) the query is sent to DDG to get a vqd value from the
+    response.
+
+    """
+    value = None
+    c = redisdb.client()
+    if c:
+        key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query)
+        value = c.get(key)
+        if value:
+            value = value.decode('utf-8')
+            logger.debug("re-use cached vqd value: %s", value)
+            return value
+
+    query_url = 'https://duckduckgo.com/?{query}&iar=images'.format(query=urlencode({'q': query}))
+    res = network.get(query_url, headers=headers)
+    content = res.text
+    if content.find('vqd=\'') == -1:
+        raise SearxEngineAPIException('Request failed')
+    value = content[content.find('vqd=\'') + 5 :]
+    value = value[: value.find('\'')]
+    logger.debug("new vqd value: %s", value)
+    cache_vqd(query, value)
+    return value
+
+
+def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
+    """Get DuckDuckGo's language identifier from SearXNG's locale.
+
+    DuckDuckGo defines its lanaguages by region codes (see
+    :py:obj:`fetch_traits`).
+
+    To get region and language of a DDG service use:
+
+    .. code: python
+
+       eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+       eng_lang = get_ddg_lang(traits, params['searxng_locale'])
+
+    It might confuse, but the ``l`` value of the cookie is what SearXNG calls
+    the *region*:
+
+    .. code:: python
+
+        # !ddi paris :es-AR --> {'ad': 'es_AR', 'ah': 'ar-es', 'l': 'ar-es'}
+        params['cookies']['ad'] = eng_lang
+        params['cookies']['ah'] = eng_region
+        params['cookies']['l'] = eng_region
+
+    .. hint::
+
+       `DDG-lite <https://lite.duckduckgo.com/lite>`__ does not offer a language
+       selection to the user, only a region can be selected by the user
+       (``eng_region`` from the example above).  DDG-lite stores the selected
+       region in a cookie::
+
+         params['cookies']['kl'] = eng_region  # 'ar-es'
+
+    """
+    return eng_traits.custom['lang_region'].get(sxng_locale, eng_traits.get_language(sxng_locale, default))
+
+
+ddg_reg_map = {
+    'tw-tzh': 'zh_TW',
+    'hk-tzh': 'zh_HK',
+    'ct-ca': 'skip',  # ct-ca and es-ca both map to ca_ES
+    'es-ca': 'ca_ES',
+    'id-en': 'id_ID',
+    'no-no': 'nb_NO',
+    'jp-jp': 'ja_JP',
+    'kr-kr': 'ko_KR',
+    'xa-ar': 'ar_SA',
+    'sl-sl': 'sl_SI',
+    'th-en': 'th_TH',
+    'vn-en': 'vi_VN',
+}
+
+ddg_lang_map = {
+    # use ar --> ar_EG (Egypt's arabic)
+    "ar_DZ": 'lang_region',
+    "ar_JO": 'lang_region',
+    "ar_SA": 'lang_region',
+    # use bn --> bn_BD
+    'bn_IN': 'lang_region',
+    # use de --> de_DE
+    'de_CH': 'lang_region',
+    # use en --> en_US,
+    'en_AU': 'lang_region',
+    'en_CA': 'lang_region',
+    'en_GB': 'lang_region',
+    # Esperanto
+    'eo_XX': 'eo',
+    # use es --> es_ES,
+    'es_AR': 'lang_region',
+    'es_CL': 'lang_region',
+    'es_CO': 'lang_region',
+    'es_CR': 'lang_region',
+    'es_EC': 'lang_region',
+    'es_MX': 'lang_region',
+    'es_PE': 'lang_region',
+    'es_UY': 'lang_region',
+    'es_VE': 'lang_region',
+    # use fr --> rf_FR
+    'fr_CA': 'lang_region',
+    'fr_CH': 'lang_region',
+    'fr_BE': 'lang_region',
+    # use nl --> nl_NL
+    'nl_BE': 'lang_region',
+    # use pt --> pt_PT
+    'pt_BR': 'lang_region',
+    # skip these languages
+    'od_IN': 'skip',
+    'io_XX': 'skip',
+    'tokipona_XX': 'skip',
+}
 
 
 def request(query, params):
 
+    # quote ddg bangs
+    query_parts = []
+    # for val in re.split(r'(\s+)', query):
+    for val in re.split(r'(\s+)', query):
+        if not val.strip():
+            continue
+        if val.startswith('!') and external_bang.get_node(external_bang.EXTERNAL_BANGS, val[1:]):
+            val = f"'{val}'"
+        query_parts.append(val)
+    query = ' '.join(query_parts)
+
+    eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+    # eng_lang = get_ddg_lang(traits, params['searxng_locale'])
+
     params['url'] = url
     params['method'] = 'POST'
-
     params['data']['q'] = query
 
     # The API is not documented, so we do some reverse engineering and emulate
@@ -88,23 +237,19 @@ def request(query, params):
         params['data']['s'] = offset
         params['data']['dc'] = offset + 1
 
+    # request needs a vqd argument
+    params['data']['vqd'] = get_vqd(query, params["headers"])
+
     # initial page does not have additional data in the input form
     if params['pageno'] > 1:
-        # request the second page (and more pages) needs 'o' and 'api' arguments
-        params['data']['o'] = 'json'
-        params['data']['api'] = 'd.js'
 
-    # initial page does not have additional data in the input form
-    if params['pageno'] > 2:
-        # request the third page (and more pages) some more arguments
-        params['data']['nextParams'] = ''
-        params['data']['v'] = ''
-        params['data']['vqd'] = ''
+        params['data']['o'] = form_data.get('o', 'json')
+        params['data']['api'] = form_data.get('api', 'd.js')
+        params['data']['nextParams'] = form_data.get('nextParams', '')
+        params['data']['v'] = form_data.get('v', 'l')
 
-    region_code = get_region_code(params['language'], supported_languages)
-    if region_code:
-        params['data']['kl'] = region_code
-        params['cookies']['kl'] = region_code
+    params['data']['kl'] = eng_region
+    params['cookies']['kl'] = eng_region
 
     params['data']['df'] = ''
     if params['time_range'] in time_range_dict:
@@ -116,26 +261,40 @@ def request(query, params):
     return params
 
 
-# get response from search-request
 def response(resp):
 
-    headers_ping = dict_subset(resp.request.headers, ['User-Agent', 'Accept-Encoding', 'Accept', 'Cookie'])
-    get(url_ping, headers=headers_ping)
-
     if resp.status_code == 303:
         return []
 
     results = []
-    doc = fromstring(resp.text)
+    doc = lxml.html.fromstring(resp.text)
 
     result_table = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table')
-    if not len(result_table) >= 3:
+
+    if len(result_table) == 2:
+        # some locales (at least China) does not have a "next page" button and
+        # the layout of the HTML tables is different.
+        result_table = result_table[1]
+    elif not len(result_table) >= 3:
         # no more results
         return []
+    else:
         result_table = result_table[2]
+        # update form data from response
+        form = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table//input/..')
+        if len(form):
+
+            form = form[0]
+            form_data['v'] = eval_xpath(form, '//input[@name="v"]/@value')[0]
+            form_data['api'] = eval_xpath(form, '//input[@name="api"]/@value')[0]
+            form_data['o'] = eval_xpath(form, '//input[@name="o"]/@value')[0]
+            logger.debug('form_data: %s', form_data)
+
+            value = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
+            query = resp.search_params['data']['q']
+            cache_vqd(query, value)
 
     tr_rows = eval_xpath(result_table, './/tr')
-
     # In the last <tr> is the form of the 'previous/next page' links
     tr_rows = tr_rows[:-1]
 
@@ -172,15 +331,105 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages & regions from DuckDuckGo.
 
-    # response is a js file with regions as an embedded object
-    response_page = resp.text
-    response_page = response_page[response_page.find('regions:{') + 8 :]
-    response_page = response_page[: response_page.find('}') + 1]
+    SearXNG's ``all`` locale maps DuckDuckGo's "Alle regions" (``wt-wt``).
+    DuckDuckGo's language "Browsers prefered language" (``wt_WT``) makes no
+    sense in a SearXNG request since SearXNG's ``all`` will not add a
+    ``Accept-Language`` HTTP header.  The value in ``engine_traits.all_locale``
+    is ``wt-wt`` (the region).
 
-    regions_json = loads(response_page)
-    supported_languages = map((lambda x: x[3:] + '-' + x[:2].upper()), regions_json.keys())
+    Beside regions DuckDuckGo also defines its lanaguages by region codes.  By
+    example these are the english languages in DuckDuckGo:
 
-    return list(supported_languages)
+    - en_US
+    - en_AU
+    - en_CA
+    - en_GB
+
+    The function :py:obj:`get_ddg_lang` evaluates DuckDuckGo's language from
+    SearXNG's locale.
+
+    """
+    # pylint: disable=too-many-branches, too-many-statements
+    # fetch regions
+
+    engine_traits.all_locale = 'wt-wt'
+
+    # updated from u588 to u661 / should be updated automatically?
+    resp = network.get('https://duckduckgo.com/util/u661.js')
+
+    if not resp.ok:
+        print("ERROR: response from DuckDuckGo is not OK.")
+
+    pos = resp.text.find('regions:{') + 8
+    js_code = resp.text[pos:]
+    pos = js_code.find('}') + 1
+    regions = json.loads(js_code[:pos])
+
+    for eng_tag, name in regions.items():
+
+        if eng_tag == 'wt-wt':
+            engine_traits.all_locale = 'wt-wt'
+            continue
+
+        region = ddg_reg_map.get(eng_tag)
+        if region == 'skip':
+            continue
+
+        if not region:
+            eng_territory, eng_lang = eng_tag.split('-')
+            region = eng_lang + '_' + eng_territory.upper()
+
+        try:
+            sxng_tag = locales.region_tag(babel.Locale.parse(region))
+        except babel.UnknownLocaleError:
+            print("ERROR: %s (%s) -> %s is unknown by babel" % (name, eng_tag, region))
+            continue
+
+        conflict = engine_traits.regions.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+        engine_traits.regions[sxng_tag] = eng_tag
+
+    # fetch languages
+
+    engine_traits.custom['lang_region'] = {}
+
+    pos = resp.text.find('languages:{') + 10
+    js_code = resp.text[pos:]
+    pos = js_code.find('}') + 1
+    js_code = '{"' + js_code[1:pos].replace(':', '":').replace(',', ',"')
+    languages = json.loads(js_code)
+
+    for eng_lang, name in languages.items():
+
+        if eng_lang == 'wt_WT':
+            continue
+
+        babel_tag = ddg_lang_map.get(eng_lang, eng_lang)
+        if babel_tag == 'skip':
+            continue
+
+        try:
+
+            if babel_tag == 'lang_region':
+                sxng_tag = locales.region_tag(babel.Locale.parse(eng_lang))
+                engine_traits.custom['lang_region'][sxng_tag] = eng_lang
+                continue
+
+            sxng_tag = locales.language_tag(babel.Locale.parse(babel_tag))
+
+        except babel.UnknownLocaleError:
+            print("ERROR: language %s (%s) is unknown by babel" % (name, eng_lang))
+            continue
+
+        conflict = engine_traits.languages.get(sxng_tag)
+        if conflict:
+            if conflict != eng_lang:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_lang))
+            continue
+        engine_traits.languages[sxng_tag] = eng_lang

searx/engines/duckduckgo_definitions.py

@@ -1,22 +1,33 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""DuckDuckGo (Instant Answer API)
+"""
+DuckDuckGo Instant Answer API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `DDG-API <https://duckduckgo.com/api>`__ is no longer documented but from
+reverse engineering we can see that some services (e.g. instant answers) still
+in use from the DDG search engine.
+
+As far we can say the *instant answers* API does not support languages, or at
+least we could not find out how language support should work.  It seems that
+most of the features are based on English terms.
 
 """
 
-import json
+from typing import TYPE_CHECKING
+
 from urllib.parse import urlencode, urlparse, urljoin
 from lxml import html
 
 from searx.data import WIKIDATA_UNITS
-from searx.engines.duckduckgo import language_aliases
-from searx.engines.duckduckgo import (  # pylint: disable=unused-import
-    _fetch_supported_languages,
-    supported_languages_url,
-)
-from searx.utils import extract_text, html_to_text, match_language, get_string_replaces_function
+from searx.utils import extract_text, html_to_text, get_string_replaces_function
 from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
 
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
 # about
 about = {
     "website": 'https://duckduckgo.com/',
@@ -37,7 +48,7 @@ replace_http_by_https = get_string_replaces_function({'http:': 'https:'})
 
 
 def is_broken_text(text):
-    """duckduckgo may return something like "<a href="xxxx">http://somewhere Related website<a/>"
+    """duckduckgo may return something like ``<a href="xxxx">http://somewhere Related website<a/>``
 
     The href URL is broken, the "Related website" may contains some HTML.
 
@@ -62,8 +73,6 @@ def result_to_text(text, htmlResult):
 
 def request(query, params):
     params['url'] = URL.format(query=urlencode({'q': query}))
-    language = match_language(params['language'], supported_languages, language_aliases)
-    language = language.split('-')[0]
     return params
 
 
@@ -71,7 +80,7 @@ def response(resp):
     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
     results = []
 
-    search_res = json.loads(resp.text)
+    search_res = resp.json()
 
     # search_res.get('Entity') possible values (not exhaustive) :
     # * continent / country / department / location / waterfall
@@ -235,7 +244,7 @@ def unit_to_str(unit):
 
 
 def area_to_str(area):
-    """parse {'unit': 'http://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}"""
+    """parse ``{'unit': 'https://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}``"""
     unit = unit_to_str(area.get('unit'))
     if unit is not None:
         try:

searx/engines/duckduckgo_images.py

@@ -1,26 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """
- DuckDuckGo (Images)
+DuckDuckGo Images
+~~~~~~~~~~~~~~~~~
 """
 
-from json import loads
+from typing import TYPE_CHECKING
 from urllib.parse import urlencode
-from searx.exceptions import SearxEngineAPIException
-from searx.engines.duckduckgo import get_region_code
-from searx.engines.duckduckgo import (  # pylint: disable=unused-import
-    _fetch_supported_languages,
-    supported_languages_url,
+
+from searx.engines.duckduckgo import fetch_traits  # pylint: disable=unused-import
+from searx.engines.duckduckgo import (
+    get_ddg_lang,
+    get_vqd,
 )
-from searx.network import get
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
     "website": 'https://duckduckgo.com/',
     "wikidata_id": 'Q12805',
-    "official_api_documentation": {
-        'url': 'https://duckduckgo.com/api',
-        'comment': 'but images are not supported',
-    },
     "use_official_api": False,
     "require_api_key": False,
     "results": 'JSON (site requires js to get images)',
@@ -32,70 +36,64 @@ paging = True
 safesearch = True
 send_accept_language_header = True
 
-# search-url
-images_url = 'https://duckduckgo.com/i.js?{query}&s={offset}&p={safesearch}&o=json&vqd={vqd}'
-site_url = 'https://duckduckgo.com/?{query}&iar=images&iax=1&ia=images'
+safesearch_cookies = {0: '-2', 1: None, 2: '1'}
+safesearch_args = {0: '1', 1: None, 2: '1'}
 
 
-# run query in site to get vqd number needed for requesting images
-# TODO: find a way to get this number without an extra request (is it a hash of the query?)
-def get_vqd(query, headers):
-    query_url = site_url.format(query=urlencode({'q': query}))
-    res = get(query_url, headers=headers)
-    content = res.text
-    if content.find('vqd=\'') == -1:
-        raise SearxEngineAPIException('Request failed')
-    vqd = content[content.find('vqd=\'') + 5 :]
-    vqd = vqd[: vqd.find('\'')]
-    return vqd
-
-
-# do search-request
 def request(query, params):
-    # to avoid running actual external requests when testing
-    if 'is_test' not in params:
-        vqd = get_vqd(query, params['headers'])
-    else:
-        vqd = '12345'
 
-    offset = (params['pageno'] - 1) * 50
+    eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+    eng_lang = get_ddg_lang(traits, params['searxng_locale'])
 
-    safesearch = params['safesearch'] - 1
+    args = {
+        'q': query,
+        'o': 'json',
+        # 'u': 'bing',
+        'l': eng_region,
+        'vqd': get_vqd(query, params["headers"]),
+    }
 
-    region_code = get_region_code(params['language'], lang_list=supported_languages)
-    if region_code:
-        params['url'] = images_url.format(
-            query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd
-        )
-    else:
-        params['url'] = images_url.format(query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
+    if params['pageno'] > 1:
+        args['s'] = (params['pageno'] - 1) * 100
+
+    params['cookies']['ad'] = eng_lang  # zh_CN
+    params['cookies']['ah'] = eng_region  # "us-en,de-de"
+    params['cookies']['l'] = eng_region  # "hk-tzh"
+    logger.debug("cookies: %s", params['cookies'])
+
+    safe_search = safesearch_cookies.get(params['safesearch'])
+    if safe_search is not None:
+        params['cookies']['p'] = safe_search  # "-2", "1"
+    safe_search = safesearch_args.get(params['safesearch'])
+    if safe_search is not None:
+        args['p'] = safe_search  # "-1", "1"
+
+    args = urlencode(args)
+    params['url'] = 'https://duckduckgo.com/i.js?{args}&f={f}'.format(args=args, f=',,,,,')
+
+    params['headers']['Accept'] = 'application/json, text/javascript, */*; q=0.01'
+    params['headers']['Referer'] = 'https://duckduckgo.com/'
+    params['headers']['X-Requested-With'] = 'XMLHttpRequest'
+    logger.debug("headers: %s", params['headers'])
 
     return params
 
 
-# get response from search-request
 def response(resp):
     results = []
+    res_json = resp.json()
 
-    content = resp.text
-    res_json = loads(content)
-
-    # parse results
     for result in res_json['results']:
-        title = result['title']
-        url = result['url']
-        thumbnail = result['thumbnail']
-        image = result['image']
-
-        # append result
         results.append(
             {
                 'template': 'images.html',
-                'title': title,
+                'title': result['title'],
                 'content': '',
-                'thumbnail_src': thumbnail,
-                'img_src': image,
-                'url': url,
+                'thumbnail_src': result['thumbnail'],
+                'img_src': result['image'],
+                'url': result['url'],
+                'img_format': '%s x %s' % (result['width'], result['height']),
+                'source': result['source'],
             }
         )
 

searx/engines/duckduckgo_weather.py

@@ -1,13 +1,29 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""DuckDuckGo Weather"""
+"""
+DuckDuckGo Weather
+~~~~~~~~~~~~~~~~~~
+"""
 
+from typing import TYPE_CHECKING
 from json import loads
 from urllib.parse import quote
 
 from datetime import datetime
 from flask_babel import gettext
 
+from searx.engines.duckduckgo import fetch_traits  # pylint: disable=unused-import
+from searx.engines.duckduckgo import get_ddg_lang
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
+
+
 about = {
     "website": 'https://duckduckgo.com/',
     "wikidata_id": 'Q12805',
@@ -17,9 +33,11 @@ about = {
     "results": "JSON",
 }
 
-categories = ["others"]
+send_accept_language_header = True
 
-url = "https://duckduckgo.com/js/spice/forecast/{query}/{lang}"
+# engine dependent config
+categories = ["others"]
+URL = "https://duckduckgo.com/js/spice/forecast/{query}/{lang}"
 
 
 def generate_condition_table(condition):
@@ -72,8 +90,17 @@ def generate_day_table(day):
 
 
 def request(query, params):
-    params["url"] = url.format(query=quote(query), lang=params['language'].split('-')[0])
 
+    eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+    eng_lang = get_ddg_lang(traits, params['searxng_locale'])
+
+    # !ddw paris :es-AR --> {'ad': 'es_AR', 'ah': 'ar-es', 'l': 'ar-es'}
+    params['cookies']['ad'] = eng_lang
+    params['cookies']['ah'] = eng_region
+    params['cookies']['l'] = eng_region
+    logger.debug("cookies: %s", params['cookies'])
+
+    params["url"] = URL.format(query=quote(query), lang=eng_lang.split('_')[0])
     return params
 
 

searx/engines/flickr_noapi.py

@@ -1,14 +1,22 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- Flickr (Images)
+# lint: pylint
+"""Flickr (Images)
+
 """
 
-from json import loads
+from typing import TYPE_CHECKING
+
+import json
 from time import time
 import re
 from urllib.parse import urlencode
 from searx.utils import ecma_unescape, html_to_text
 
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
 # about
 about = {
     "website": 'https://www.flickr.com',
@@ -19,23 +27,24 @@ about = {
     "results": 'HTML',
 }
 
+# engine dependent config
 categories = ['images']
-
-url = 'https://www.flickr.com/'
-search_url = url + 'search?{query}&page={page}'
-time_range_url = '&min_upload_date={start}&max_upload_date={end}'
-photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
-modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M)
-image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
-
 paging = True
 time_range_support = True
+safesearch = False
+
 time_range_dict = {
     'day': 60 * 60 * 24,
     'week': 60 * 60 * 24 * 7,
     'month': 60 * 60 * 24 * 7 * 4,
     'year': 60 * 60 * 24 * 7 * 52,
 }
+image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'm', 'n', 't', 'q', 's')
+
+search_url = 'https://www.flickr.com/search?{query}&page={page}'
+time_range_url = '&min_upload_date={start}&max_upload_date={end}'
+photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
+modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M)
 
 
 def build_flickr_url(user_id, photo_id):
@@ -55,51 +64,59 @@ def request(query, params):
     return params
 
 
-def response(resp):
+def response(resp):  # pylint: disable=too-many-branches
     results = []
 
     matches = modelexport_re.search(resp.text)
-
     if matches is None:
         return results
 
     match = matches.group(1)
-    model_export = loads(match)
+    model_export = json.loads(match)
 
     if 'legend' not in model_export:
         return results
-
     legend = model_export['legend']
 
     # handle empty page
     if not legend or not legend[0]:
         return results
 
-    for index in legend:
-        photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][int(index[4])]
+    for x, index in enumerate(legend):
+        if len(index) != 8:
+            logger.debug("skip legend enty %s : %s", x, index)
+            continue
+
+        photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][index[4]][index[5]][int(index[6])][
+            index[7]
+        ]
         author = ecma_unescape(photo.get('realname', ''))
-        source = ecma_unescape(photo.get('username', '')) + ' @ Flickr'
+        source = ecma_unescape(photo.get('username', ''))
+        if source:
+            source += ' @ Flickr'
         title = ecma_unescape(photo.get('title', ''))
         content = html_to_text(ecma_unescape(photo.get('description', '')))
         img_src = None
+
         # From the biggest to the lowest format
+        size_data = None
         for image_size in image_sizes:
-            if image_size in photo['sizes']:
-                img_src = photo['sizes'][image_size]['url']
-                img_format = (
-                    'jpg ' + str(photo['sizes'][image_size]['width']) + 'x' + str(photo['sizes'][image_size]['height'])
-                )
+            if image_size in photo['sizes']['data']:
+                size_data = photo['sizes']['data'][image_size]['data']
                 break
 
-        if not img_src:
-            logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
+        if not size_data:
+            logger.debug('cannot find valid image size: {0}'.format(repr(photo['sizes']['data'])))
             continue
 
+        img_src = size_data['url']
+        img_format = f"{size_data['width']} x {size_data['height']}"
+
         # For a bigger thumbnail, keep only the url_z, not the url_n
-        if 'n' in photo['sizes']:
-            thumbnail_src = photo['sizes']['n']['url']
-        elif 'z' in photo['sizes']:
-            thumbnail_src = photo['sizes']['z']['url']
+        if 'n' in photo['sizes']['data']:
+            thumbnail_src = photo['sizes']['data']['n']['data']['url']
+        elif 'z' in photo['sizes']['data']:
+            thumbnail_src = photo['sizes']['data']['z']['data']['url']
         else:
             thumbnail_src = img_src
 

searx/engines/gentoo.py

@@ -25,6 +25,7 @@ base_url = 'https://wiki.gentoo.org'
 # xpath queries
 xpath_results = '//ul[@class="mw-search-results"]/li'
 xpath_link = './/div[@class="mw-search-result-heading"]/a'
+xpath_content = './/div[@class="searchresult"]'
 
 
 # cut 'en' from 'en-US', 'de' from 'de-CH', and so on
@@ -77,8 +78,6 @@ main_langs = {
     'uk': 'Українська',
     'zh': '简体中文',
 }
-supported_languages = dict(lang_urls, **main_langs)
-
 
 # do search-request
 def request(query, params):
@@ -118,7 +117,8 @@ def response(resp):
         link = result.xpath(xpath_link)[0]
         href = urljoin(base_url, link.attrib.get('href'))
         title = extract_text(link)
+        content = extract_text(result.xpath(xpath_content))
 
-        results.append({'url': href, 'title': title})
+        results.append({'url': href, 'title': title, 'content': content})
 
     return results

searx/engines/gigablast.py

@@ -39,6 +39,9 @@ extra_param_ts = 0
 # after how many seconds extra_param expire
 extra_param_expiration_delay = 3000
 
+gb_userid = ''
+gb_code = ''
+
 
 def fetch_extra_param(query_args, headers):
 
@@ -71,6 +74,10 @@ def fetch_extra_param(query_args, headers):
 def request(query, params):  # pylint: disable=unused-argument
     query_args = dict(c='main', q=query, dr=1, showgoodimages=0)
 
+    if gb_userid and gb_code:
+        query_args['userid'] = gb_userid
+        query_args['code'] = gb_code
+
     if params['language'] and params['language'] != 'all':
         query_args['qlangcountry'] = params['language']
         query_args['qlang'] = params['language'].split('-')[0]

searx/engines/google.py

@@ -1,34 +1,39 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the google WEB engine.  Some of this
-implementations are shared by other engines:
+"""This is the implementation of the Google WEB engine.  Some of this
+implementations (manly the :py:obj:`get_google_info`) are shared by other
+engines:
 
 - :ref:`google images engine`
 - :ref:`google news engine`
 - :ref:`google videos engine`
-
-The google WEB engine itself has a special setup option:
-
-.. code:: yaml
-
-  - name: google
-    ...
-    use_mobile_ui: false
-
-``use_mobile_ui``: (default: ``false``)
-  Enables to use *mobile endpoint* to bypass the google blocking (see
-  :issue:`159`).  On the mobile UI of Google Search, the button :guilabel:`More
-  results` is not affected by Google rate limiting and we can still do requests
-  while actively blocked by the original Google search.  By activate
-  ``use_mobile_ui`` this behavior is simulated by adding the parameter
-  ``async=use_ac:true,_fmt:pc`` to the :py:func:`request`.
+- :ref:`google scholar engine`
+- :ref:`google autocomplete`
 
 """
 
+from typing import TYPE_CHECKING
+
+import re
 from urllib.parse import urlencode
 from lxml import html
-from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
+import babel
+import babel.core
+import babel.languages
+
+from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
+from searx.locales import language_tag, region_tag, get_offical_locales
+from searx import network
 from searx.exceptions import SearxEngineCaptchaException
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
+
 
 # about
 about = {
@@ -45,64 +50,6 @@ categories = ['general', 'web']
 paging = True
 time_range_support = True
 safesearch = True
-send_accept_language_header = True
-use_mobile_ui = False
-supported_languages_url = 'https://www.google.com/preferences?#languages'
-
-# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
-google_domains = {
-    'BG': 'google.bg',  # Bulgaria
-    'CZ': 'google.cz',  # Czech Republic
-    'DE': 'google.de',  # Germany
-    'DK': 'google.dk',  # Denmark
-    'AT': 'google.at',  # Austria
-    'CH': 'google.ch',  # Switzerland
-    'GR': 'google.gr',  # Greece
-    'AU': 'google.com.au',  # Australia
-    'CA': 'google.ca',  # Canada
-    'GB': 'google.co.uk',  # United Kingdom
-    'ID': 'google.co.id',  # Indonesia
-    'IE': 'google.ie',  # Ireland
-    'IN': 'google.co.in',  # India
-    'MY': 'google.com.my',  # Malaysia
-    'NZ': 'google.co.nz',  # New Zealand
-    'PH': 'google.com.ph',  # Philippines
-    'SG': 'google.com.sg',  # Singapore
-    'US': 'google.com',  # United States (google.us) redirects to .com
-    'ZA': 'google.co.za',  # South Africa
-    'AR': 'google.com.ar',  # Argentina
-    'CL': 'google.cl',  # Chile
-    'ES': 'google.es',  # Spain
-    'MX': 'google.com.mx',  # Mexico
-    'EE': 'google.ee',  # Estonia
-    'FI': 'google.fi',  # Finland
-    'BE': 'google.be',  # Belgium
-    'FR': 'google.fr',  # France
-    'IL': 'google.co.il',  # Israel
-    'HR': 'google.hr',  # Croatia
-    'HU': 'google.hu',  # Hungary
-    'IT': 'google.it',  # Italy
-    'JP': 'google.co.jp',  # Japan
-    'KR': 'google.co.kr',  # South Korea
-    'LT': 'google.lt',  # Lithuania
-    'LV': 'google.lv',  # Latvia
-    'NO': 'google.no',  # Norway
-    'NL': 'google.nl',  # Netherlands
-    'PL': 'google.pl',  # Poland
-    'BR': 'google.com.br',  # Brazil
-    'PT': 'google.pt',  # Portugal
-    'RO': 'google.ro',  # Romania
-    'RU': 'google.ru',  # Russia
-    'SK': 'google.sk',  # Slovakia
-    'SI': 'google.si',  # Slovenia
-    'SE': 'google.se',  # Sweden
-    'TH': 'google.co.th',  # Thailand
-    'TR': 'google.com.tr',  # Turkey
-    'UA': 'google.com.ua',  # Ukraine
-    'CN': 'google.com.hk',  # There is no google.cn, we use .com.hk for zh-CN
-    'HK': 'google.com.hk',  # Hong Kong
-    'TW': 'google.com.tw',  # Taiwan
-}
 
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
 
@@ -112,50 +59,50 @@ filter_mapping = {0: 'off', 1: 'medium', 2: 'high'}
 # specific xpath variables
 # ------------------------
 
-results_xpath = './/div[@data-sokoban-container]'
+results_xpath = './/div[contains(@jscontroller, "SC7lYd")]'
 title_xpath = './/a/h3[1]'
 href_xpath = './/a[h3]/@href'
-content_xpath = './/div[@data-content-feature=1]'
-
-# google *sections* are no usual *results*, we ignore them
-g_section_with_header = './g-section-with-header'
-
+content_xpath = './/div[@data-sncf]'
 
 # Suggestions are links placed in a *card-section*, we extract only the text
 # from the links not the links itself.
 suggestion_xpath = '//div[contains(@class, "EIaa9b")]//a'
 
+# UI_ASYNC = 'use_ac:true,_fmt:html' # returns a HTTP 500 when user search for
+#                                    # celebrities like '!google natasha allegri'
+#                                    # or '!google chris evans'
+UI_ASYNC = 'use_ac:true,_fmt:prog'
+"""Format of the response from UI's async request."""
 
-def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
-    """Composing various language properties for the google engines.
+
+def get_google_info(params, eng_traits):
+    """Composing various (language) properties for the google engines (:ref:`google
+    API`).
 
     This function is called by the various google engines (:ref:`google web
     engine`, :ref:`google images engine`, :ref:`google news engine` and
     :ref:`google videos engine`).
 
-    :param dict param: request parameters of the engine
+    :param dict param: Request parameters of the engine.  At least
+        a ``searxng_locale`` key should be in the dictionary.
 
-    :param list lang_list: list of supported languages of the engine
-        :py:obj:`ENGINES_LANGUAGES[engine-name] <searx.data.ENGINES_LANGUAGES>`
-
-    :param dict lang_list: custom aliases for non standard language codes
-        (used when calling :py:func:`searx.utils.match_language`)
-
-    :param bool supported_any_language: When a language is not specified, the
-        language interpretation is left up to Google to decide how the search
-        results should be delivered.  This argument is ``True`` for the google
-        engine and ``False`` for the other engines (google-images, -news,
-        -scholar, -videos).
+    :param eng_traits: Engine's traits fetched from google preferences
+        (:py:obj:`searx.enginelib.traits.EngineTraits`)
 
     :rtype: dict
     :returns:
         Py-Dictionary with the key/value pairs:
 
         language:
-            Return value from :py:func:`searx.utils.match_language`
+            The language code that is used by google (e.g. ``lang_en`` or
+            ``lang_zh-TW``)
 
         country:
-            The country code (e.g. US, AT, CA, FR, DE ..)
+            The country code that is used by google (e.g. ``US`` or ``TW``)
+
+        locale:
+            A instance of :py:obj:`babel.core.Locale` build from the
+            ``searxng_locale`` value.
 
         subdomain:
             Google subdomain :py:obj:`google_domains` that fits to the country
@@ -165,52 +112,67 @@ def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
             Py-Dictionary with additional request arguments (can be passed to
             :py:func:`urllib.parse.urlencode`).
 
+            - ``hl`` parameter: specifies the interface language of user interface.
+            - ``lr`` parameter: restricts search results to documents written in
+              a particular language.
+            - ``cr`` parameter: restricts search results to documents
+              originating in a particular country.
+            - ``ie`` parameter: sets the character encoding scheme that should
+              be used to interpret the query string ('utf8').
+            - ``oe`` parameter: sets the character encoding scheme that should
+              be used to decode the XML result ('utf8').
+
         headers:
             Py-Dictionary with additional HTTP headers (can be passed to
             request's headers)
+
+            - ``Accept: '*/*``
+
     """
+
     ret_val = {
         'language': None,
         'country': None,
         'subdomain': None,
         'params': {},
         'headers': {},
+        'cookies': {},
+        'locale': None,
     }
 
-    # language ...
+    sxng_locale = params.get('searxng_locale', 'all')
+    try:
+        locale = babel.Locale.parse(sxng_locale, sep='-')
+    except babel.core.UnknownLocaleError:
+        locale = None
 
-    _lang = params['language']
-    _any_language = _lang.lower() == 'all'
-    if _any_language:
-        _lang = 'en-US'
-    language = match_language(_lang, lang_list, custom_aliases)
-    ret_val['language'] = language
+    eng_lang = eng_traits.get_language(sxng_locale, 'lang_en')
+    lang_code = eng_lang.split('_')[-1]  # lang_zh-TW --> zh-TW / lang_en --> en
+    country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
 
-    # country ...
+    # Test zh_hans & zh_hant --> in the topmost links in the result list of list
+    # TW and HK you should a find wiktionary.org zh_hant link.  In the result
+    # list of zh-CN should not be no hant link instead you should find
+    # zh.m.wikipedia.org/zh somewhere in the top.
 
-    _l = _lang.split('-')
-    if len(_l) == 2:
-        country = _l[1]
-    else:
-        country = _l[0].upper()
-        if country == 'EN':
-            country = 'US'
+    # '!go 日 :zh-TW' --> https://zh.m.wiktionary.org/zh-hant/%E6%97%A5
+    # '!go 日 :zh-CN' --> https://zh.m.wikipedia.org/zh/%E6%97%A5
+
+    ret_val['language'] = eng_lang
     ret_val['country'] = country
-
-    # subdomain ...
-
-    ret_val['subdomain'] = 'www.' + google_domains.get(country.upper(), 'google.com')
-
-    # params & headers
-
-    lang_country = '%s-%s' % (language, country)  # (en-US, en-EN, de-DE, de-AU, fr-FR ..)
+    ret_val['locale'] = locale
+    ret_val['subdomain'] = eng_traits.custom['supported_domains'].get(country.upper(), 'www.google.com')
 
     # hl parameter:
-    #   https://developers.google.com/custom-search/docs/xml_results#hlsp The
-    # Interface Language:
+    #   The hl parameter specifies the interface language (host language) of
+    #   your user interface. To improve the performance and the quality of your
+    #   search results, you are strongly encouraged to set this parameter
+    #   explicitly.
+    #   https://developers.google.com/custom-search/docs/xml_results#hlsp
+    # The Interface Language:
     #   https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
 
-    ret_val['params']['hl'] = lang_list.get(lang_country, language)
+    ret_val['params']['hl'] = lang_code
 
     # lr parameter:
     #   The lr (language restrict) parameter restricts search results to
@@ -218,22 +180,72 @@ def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
     #   https://developers.google.com/custom-search/docs/xml_results#lrsp
     #   Language Collection Values:
     #   https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
-
-    if _any_language and supported_any_language:
-
-        # interpretation is left up to Google (based on whoogle)
     #
-        # - add parameter ``source=lnt``
-        # - don't use parameter ``lr``
-        # - don't add a ``Accept-Language`` HTTP header.
+    # To select 'all' languages an empty 'lr' value is used.
+    #
+    # Different to other google services, Google Schloar supports to select more
+    # than one language. The languages are seperated by a pipe '|' (logical OR).
+    # By example: &lr=lang_zh-TW%7Clang_de selects articles written in
+    # traditional chinese OR german language.
 
-        ret_val['params']['source'] = 'lnt'
+    ret_val['params']['lr'] = eng_lang
+    if sxng_locale == 'all':
+        ret_val['params']['lr'] = ''
 
-    else:
+    # cr parameter:
+    #   The cr parameter restricts search results to documents originating in a
+    #   particular country.
+    #   https://developers.google.com/custom-search/docs/xml_results#crsp
 
-        # restricts search results to documents written in a particular
-        # language.
-        ret_val['params']['lr'] = "lang_" + lang_list.get(lang_country, language)
+    ret_val['params']['cr'] = 'country' + country
+    if sxng_locale == 'all':
+        ret_val['params']['cr'] = ''
+
+    # gl parameter: (mandatory by Geeogle News)
+    #   The gl parameter value is a two-letter country code. For WebSearch
+    #   results, the gl parameter boosts search results whose country of origin
+    #   matches the parameter value. See the Country Codes section for a list of
+    #   valid values.
+    #   Specifying a gl parameter value in WebSearch requests should improve the
+    #   relevance of results. This is particularly true for international
+    #   customers and, even more specifically, for customers in English-speaking
+    #   countries other than the United States.
+    #   https://developers.google.com/custom-search/docs/xml_results#glsp
+
+    ret_val['params']['gl'] = country
+
+    # ie parameter:
+    #   The ie parameter sets the character encoding scheme that should be used
+    #   to interpret the query string. The default ie value is latin1.
+    #   https://developers.google.com/custom-search/docs/xml_results#iesp
+
+    ret_val['params']['ie'] = 'utf8'
+
+    # oe parameter:
+    #   The oe parameter sets the character encoding scheme that should be used
+    #   to decode the XML result. The default oe value is latin1.
+    #   https://developers.google.com/custom-search/docs/xml_results#oesp
+
+    ret_val['params']['oe'] = 'utf8'
+
+    # num parameter:
+    #   The num parameter identifies the number of search results to return.
+    #   The default num value is 10, and the maximum value is 20. If you request
+    #   more than 20 results, only 20 results will be returned.
+    #   https://developers.google.com/custom-search/docs/xml_results#numsp
+
+    # HINT: seems to have no effect (tested in google WEB & Images)
+    # ret_val['params']['num'] = 20
+
+    # HTTP headers
+
+    ret_val['headers']['Accept'] = '*/*'
+
+    # Cookies
+
+    # - https://github.com/searxng/searxng/pull/1679#issuecomment-1235432746
+    # - https://github.com/searxng/searxng/issues/1555
+    ret_val['cookies']['CONSENT'] = "YES+"
 
     return ret_val
 
@@ -245,33 +257,34 @@ def detect_google_sorry(resp):
 
 def request(query, params):
     """Google search request"""
-
+    # pylint: disable=line-too-long
     offset = (params['pageno'] - 1) * 10
-
-    lang_info = get_lang_info(params, supported_languages, language_aliases, True)
-
-    additional_parameters = {}
-    if use_mobile_ui:
-        additional_parameters = {
-            'asearch': 'arc',
-            'async': 'use_ac:true,_fmt:prog',
-        }
+    google_info = get_google_info(params, traits)
 
     # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
     query_url = (
         'https://'
-        + lang_info['subdomain']
+        + google_info['subdomain']
         + '/search'
         + "?"
         + urlencode(
             {
                 'q': query,
-                **lang_info['params'],
-                'ie': "utf8",
-                'oe': "utf8",
-                'start': offset,
+                **google_info['params'],
                 'filter': '0',
-                **additional_parameters,
+                'start': offset,
+                # 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
+                # 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
+                # 'cs' : 1,
+                # 'sa': 'N',
+                # 'yv': 3,
+                # 'prmd': 'vin',
+                # 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
+                # 'sa': 'N',
+                # 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
+                # formally known as use_mobile_ui
+                'asearch': 'arc',
+                'async': UI_ASYNC,
             }
         )
     )
@@ -282,25 +295,38 @@ def request(query, params):
         query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
     params['url'] = query_url
 
-    params['cookies']['CONSENT'] = "YES+"
-    params['headers'].update(lang_info['headers'])
-    if use_mobile_ui:
-        params['headers']['Accept'] = '*/*'
-    else:
-        params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
-
+    params['cookies'] = google_info['cookies']
+    params['headers'].update(google_info['headers'])
     return params
 
 
+# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
+# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
+RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);')
+
+
+def _parse_data_images(dom):
+    data_image_map = {}
+    for img_id, data_image in RE_DATA_IMAGE.findall(dom.text_content()):
+        end_pos = data_image.rfind('=')
+        if end_pos > 0:
+            data_image = data_image[: end_pos + 1]
+        data_image_map[img_id] = data_image
+    logger.debug('data:image objects --> %s', list(data_image_map.keys()))
+    return data_image_map
+
+
 def response(resp):
     """Get response from google's search request"""
-
+    # pylint: disable=too-many-branches, too-many-statements
     detect_google_sorry(resp)
 
     results = []
 
     # convert the text to dom
     dom = html.fromstring(resp.text)
+    data_image_map = _parse_data_images(dom)
+
     # results --> answer
     answer_list = eval_xpath(dom, '//div[contains(@class, "LGOjhe")]')
     if answer_list:
@@ -309,25 +335,9 @@ def response(resp):
     else:
         logger.debug("did not find 'answer'")
 
-        # results --> number_of_results
-        if not use_mobile_ui:
-            try:
-                _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0)
-                _digit = ''.join([n for n in _txt if n.isdigit()])
-                number_of_results = int(_digit)
-                results.append({'number_of_results': number_of_results})
-            except Exception as e:  # pylint: disable=broad-except
-                logger.debug("did not 'number_of_results'")
-                logger.error(e, exc_info=True)
-
     # parse results
 
-    for result in eval_xpath_list(dom, results_xpath):
-
-        # google *sections*
-        if extract_text(eval_xpath(result, g_section_with_header)):
-            logger.debug("ignoring <g-section-with-header>")
-            continue
+    for result in eval_xpath_list(dom, results_xpath):  # pylint: disable=too-many-nested-blocks
 
         try:
             title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None)
@@ -336,16 +346,30 @@ def response(resp):
                 logger.debug('ignoring item from the result_xpath list: missing title')
                 continue
             title = extract_text(title_tag)
+
             url = eval_xpath_getindex(result, href_xpath, 0, None)
             if url is None:
+                logger.debug('ignoring item from the result_xpath list: missing url of title "%s"', title)
                 continue
-            content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True)
-            if content is None:
+
+            content_nodes = eval_xpath(result, content_xpath)
+            content = extract_text(content_nodes)
+
+            if not content:
                 logger.debug('ignoring item from the result_xpath list: missing content of title "%s"', title)
                 continue
 
-            logger.debug('add link to results: %s', title)
-            results.append({'url': url, 'title': title, 'content': content})
+            img_src = content_nodes[0].xpath('.//img/@src')
+            if img_src:
+                img_src = img_src[0]
+                if img_src.startswith('data:image'):
+                    img_id = content_nodes[0].xpath('.//img/@id')
+                    if img_id:
+                        img_src = data_image_map.get(img_id[0])
+            else:
+                img_src = None
+
+            results.append({'url': url, 'title': title, 'content': content, 'img_src': img_src})
 
         except Exception as e:  # pylint: disable=broad-except
             logger.error(e, exc_info=True)
@@ -361,15 +385,107 @@ def response(resp):
 
 
 # get supported languages from their site
-def _fetch_supported_languages(resp):
-    ret_val = {}
+
+
+skip_countries = [
+    # official language of google-country not in google-languages
+    'AL',  # Albanien (sq)
+    'AZ',  # Aserbaidschan  (az)
+    'BD',  # Bangladesch (bn)
+    'BN',  # Brunei Darussalam (ms)
+    'BT',  # Bhutan (dz)
+    'ET',  # Äthiopien (am)
+    'GE',  # Georgien (ka, os)
+    'GL',  # Grönland (kl)
+    'KH',  # Kambodscha (km)
+    'LA',  # Laos (lo)
+    'LK',  # Sri Lanka (si, ta)
+    'ME',  # Montenegro (sr)
+    'MK',  # Nordmazedonien (mk, sq)
+    'MM',  # Myanmar (my)
+    'MN',  # Mongolei (mn)
+    'MV',  # Malediven (dv) // dv_MV is unknown by babel
+    'MY',  # Malaysia (ms)
+    'NP',  # Nepal (ne)
+    'TJ',  # Tadschikistan (tg)
+    'TM',  # Turkmenistan (tk)
+    'UZ',  # Usbekistan (uz)
+]
+
+
+def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
+    """Fetch languages from Google."""
+    # pylint: disable=import-outside-toplevel, too-many-branches
+
+    engine_traits.custom['supported_domains'] = {}
+
+    resp = network.get('https://www.google.com/preferences')
+    if not resp.ok:
+        raise RuntimeError("Response from Google's preferences is not OK.")
+
     dom = html.fromstring(resp.text)
 
-    radio_buttons = eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]')
+    # supported language codes
 
-    for x in radio_buttons:
-        name = x.get("data-name")
-        code = x.get("value").split('_')[-1]
-        ret_val[code] = {"name": name}
+    lang_map = {'no': 'nb'}
+    for x in eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]'):
 
-    return ret_val
+        eng_lang = x.get("value").split('_')[-1]
+        try:
+            locale = babel.Locale.parse(lang_map.get(eng_lang, eng_lang), sep='-')
+        except babel.UnknownLocaleError:
+            print("ERROR: %s -> %s is unknown by babel" % (x.get("data-name"), eng_lang))
+            continue
+        sxng_lang = language_tag(locale)
+
+        conflict = engine_traits.languages.get(sxng_lang)
+        if conflict:
+            if conflict != eng_lang:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_lang, conflict, eng_lang))
+            continue
+        engine_traits.languages[sxng_lang] = 'lang_' + eng_lang
+
+    # alias languages
+    engine_traits.languages['zh'] = 'lang_zh-CN'
+
+    # supported region codes
+
+    for x in eval_xpath_list(dom, '//*[@name="region"]/..//input[@name="region"]'):
+        eng_country = x.get("value")
+
+        if eng_country in skip_countries:
+            continue
+        if eng_country == 'ZZ':
+            engine_traits.all_locale = 'ZZ'
+            continue
+
+        sxng_locales = get_offical_locales(eng_country, engine_traits.languages.keys(), regional=True)
+
+        if not sxng_locales:
+            print("ERROR: can't map from google country %s (%s) to a babel region." % (x.get('data-name'), eng_country))
+            continue
+
+        for sxng_locale in sxng_locales:
+            engine_traits.regions[region_tag(sxng_locale)] = eng_country
+
+    # alias regions
+    engine_traits.regions['zh-CN'] = 'HK'
+
+    # supported domains
+
+    if add_domains:
+        resp = network.get('https://www.google.com/supported_domains')
+        if not resp.ok:
+            raise RuntimeError("Response from https://www.google.com/supported_domains is not OK.")
+
+        for domain in resp.text.split():
+            domain = domain.strip()
+            if not domain or domain in [
+                '.google.com',
+            ]:
+                continue
+            region = domain.split('.')[-1].upper()
+            engine_traits.custom['supported_domains'][region] = 'www' + domain
+            if region == 'HK':
+                # There is no google.cn, we use .com.hk for zh-CN
+                engine_traits.custom['supported_domains']['CN'] = 'www' + domain

searx/engines/google_images.py

@@ -1,31 +1,38 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the google images engine using the google
-internal API used the Google Go Android app.
+"""This is the implementation of the Google Images engine using the internal
+Google API used by the Google Go Android app.
 
 This internal API offer results in
 
-- JSON (_fmt:json)
-- Protobuf (_fmt:pb)
-- Protobuf compressed? (_fmt:pc)
-- HTML (_fmt:html)
-- Protobuf encoded in JSON (_fmt:jspb).
+- JSON (``_fmt:json``)
+- Protobuf_ (``_fmt:pb``)
+- Protobuf_ compressed? (``_fmt:pc``)
+- HTML (``_fmt:html``)
+- Protobuf_ encoded in JSON (``_fmt:jspb``).
 
+.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
 """
 
+from typing import TYPE_CHECKING
+
 from urllib.parse import urlencode
 from json import loads
 
+from searx.engines.google import fetch_traits  # pylint: disable=unused-import
 from searx.engines.google import (
-    get_lang_info,
+    get_google_info,
     time_range_dict,
     detect_google_sorry,
 )
 
-# pylint: disable=unused-import
-from searx.engines.google import supported_languages_url, _fetch_supported_languages
+if TYPE_CHECKING:
+    import logging
+    from searx.enginelib.traits import EngineTraits
+
+    logger: logging.Logger
+    traits: EngineTraits
 
-# pylint: enable=unused-import
 
 # about
 about = {
@@ -40,7 +47,6 @@ about = {
 # engine dependent config
 categories = ['images', 'web']
 paging = True
-use_locale_domain = True
 time_range_support = True
 safesearch = True
 send_accept_language_header = True
@@ -51,20 +57,18 @@ filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
 def request(query, params):
     """Google-Image search request"""
 
-    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
+    google_info = get_google_info(params, traits)
 
     query_url = (
         'https://'
-        + lang_info['subdomain']
+        + google_info['subdomain']
         + '/search'
         + "?"
         + urlencode(
             {
                 'q': query,
                 'tbm': "isch",
-                **lang_info['params'],
-                'ie': "utf8",
-                'oe': "utf8",
+                **google_info['params'],
                 'asearch': 'isch',
                 'async': '_fmt:json,p:1,ijn:' + str(params['pageno']),
             }
@@ -77,9 +81,8 @@ def request(query, params):
         query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
     params['url'] = query_url
 
-    params['headers'].update(lang_info['headers'])
-    params['headers']['User-Agent'] = 'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12; US) gzip'
-    params['headers']['Accept'] = '*/*'
+    params['cookies'] = google_info['cookies']
+    params['headers'].update(google_info['headers'])
     return params
 
 
@@ -111,7 +114,11 @@ def response(resp):
 
         copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
         if copyright_notice:
-            result_item['source'] += ' / ' + copyright_notice
+            result_item['source'] += ' | ' + copyright_notice
+
+        freshness_date = item["result"].get("freshness_date")
+        if freshness_date:
+            result_item['source'] += ' | ' + freshness_date
 
         file_size = item.get('gsa', {}).get('file_size')
         if file_size:

searx/engines/google_news.py

@@ -1,24 +1,38 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the google news engine.  The google news API
-ignores some parameters from the common :ref:`google API`:
+"""This is the implementation of the Google News engine.
 
-- num_ : the number of search results is ignored
+Google News has a different region handling compared to Google WEB.
+
+- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
+- the hl_ argument has to be set correctly (and different to Google WEB)
+- the gl_ argument is mandatory
+
+If one of this argument is not set correctly, the request is redirected to
+CONSENT dialog::
+
+  https://consent.google.com/m?continue=
+
+The google news API ignores some parameters from the common :ref:`google API`:
+
+- num_ : the number of search results is ignored / there is no paging all
+  results for a query term are in the first response.
 - save_ : is ignored / Google-News results are always *SafeSearch*
 
+.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
+.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
 .. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
 .. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
-
 """
 
-# pylint: disable=invalid-name
+from typing import TYPE_CHECKING
 
-import binascii
-import re
 from urllib.parse import urlencode
-from base64 import b64decode
+import base64
 from lxml import html
+import babel
 
+from searx import locales
 from searx.utils import (
     eval_xpath,
     eval_xpath_list,
@@ -26,18 +40,19 @@ from searx.utils import (
     extract_text,
 )
 
-# pylint: disable=unused-import
+from searx.engines.google import fetch_traits as _fetch_traits  # pylint: disable=unused-import
 from searx.engines.google import (
-    supported_languages_url,
-    _fetch_supported_languages,
-)
-
-# pylint: enable=unused-import
-
-from searx.engines.google import (
-    get_lang_info,
+    get_google_info,
     detect_google_sorry,
 )
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
@@ -49,70 +64,77 @@ about = {
     "results": 'HTML',
 }
 
-# compared to other google engines google-news has a different time range
-# support.  The time range is included in the search term.
-time_range_dict = {
-    'day': 'when:1d',
-    'week': 'when:7d',
-    'month': 'when:1m',
-    'year': 'when:1y',
-}
-
 # engine dependent config
-
 categories = ['news']
 paging = False
-use_locale_domain = True
-time_range_support = True
+time_range_support = False
 
 # Google-News results are always *SafeSearch*. Option 'safesearch' is set to
 # False here, otherwise checker will report safesearch-errors::
 #
 #  safesearch : results are identitical for safesearch=0 and safesearch=2
-safesearch = False
-send_accept_language_header = True
+safesearch = True
+# send_accept_language_header = True
 
 
 def request(query, params):
     """Google-News search request"""
 
-    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
+    sxng_locale = params.get('searxng_locale', 'en-US')
+    ceid = locales.get_engine_locale(sxng_locale, traits.custom['ceid'], default='US:en')
+    google_info = get_google_info(params, traits)
+    google_info['subdomain'] = 'news.google.com'  # google news has only one domain
 
-    # google news has only one domain
-    lang_info['subdomain'] = 'news.google.com'
+    ceid_region, ceid_lang = ceid.split(':')
+    ceid_lang, ceid_suffix = (
+        ceid_lang.split('-')
+        + [
+            None,
+        ]
+    )[:2]
 
-    ceid = "%s:%s" % (lang_info['country'], lang_info['language'])
+    google_info['params']['hl'] = ceid_lang
 
-    # google news redirects en to en-US
-    if lang_info['params']['hl'] == 'en':
-        lang_info['params']['hl'] = 'en-US'
+    if ceid_suffix and ceid_suffix not in ['Hans', 'Hant']:
 
-    # Very special to google-news compared to other google engines, the time
-    # range is included in the search term.
-    if params['time_range']:
-        query += ' ' + time_range_dict[params['time_range']]
+        if ceid_region.lower() == ceid_lang:
+            google_info['params']['hl'] = ceid_lang + '-' + ceid_region
+        else:
+            google_info['params']['hl'] = ceid_lang + '-' + ceid_suffix
+
+    elif ceid_region.lower() != ceid_lang:
+
+        if ceid_region in ['AT', 'BE', 'CH', 'IL', 'SA', 'IN', 'BD', 'PT']:
+            google_info['params']['hl'] = ceid_lang
+        else:
+            google_info['params']['hl'] = ceid_lang + '-' + ceid_region
+
+    google_info['params']['lr'] = 'lang_' + ceid_lang.split('-')[0]
+    google_info['params']['gl'] = ceid_region
 
     query_url = (
         'https://'
-        + lang_info['subdomain']
-        + '/search'
-        + "?"
-        + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'gl': lang_info['country']})
+        + google_info['subdomain']
+        + "/search?"
+        + urlencode(
+            {
+                'q': query,
+                **google_info['params'],
+            }
+        )
+        # ceid includes a ':' character which must not be urlencoded
         + ('&ceid=%s' % ceid)
-    )  # ceid includes a ':' character which must not be urlencoded
+    )
+
     params['url'] = query_url
-
-    params['cookies']['CONSENT'] = "YES+"
-    params['headers'].update(lang_info['headers'])
-    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
-
+    params['cookies'] = google_info['cookies']
+    params['headers'].update(google_info['headers'])
     return params
 
 
 def response(resp):
     """Get response from google's search request"""
     results = []
-
     detect_google_sorry(resp)
 
     # convert the text to dom
@@ -120,40 +142,23 @@ def response(resp):
 
     for result in eval_xpath_list(dom, '//div[@class="xrnccd"]'):
 
-        # The first <a> tag in the <article> contains the link to the
-        # article The href attribute of the <a> is a google internal link,
-        # we can't use.  The real link is hidden in the jslog attribute:
-        #
-        #   <a ...
-        #      jslog="95014; 4:https://www.cnn.com/.../index.html; track:click"
-        #      href="./articles/CAIiENu3nGS...?hl=en-US&amp;gl=US&amp;ceid=US%3Aen"
-        #      ... />
+        # The first <a> tag in the <article> contains the link to the article
+        # The href attribute of the <a> tag is a google internal link, we have
+        # to decode
 
-        jslog = eval_xpath_getindex(result, './article/a/@jslog', 0)
-        url = re.findall('http[^;]*', jslog)
-        if url:
-            url = url[0]
-        else:
-            # The real URL is base64 encoded in the json attribute:
-            # jslog="95014; 5:W251bGwsbnVsbCxudW...giXQ==; track:click"
-            jslog = jslog.split(";")[1].split(':')[1].strip()
-            try:
-                padding = (4 - (len(jslog) % 4)) * "="
-                jslog = b64decode(jslog + padding)
-            except binascii.Error:
-                # URL can't be read, skip this result
-                continue
+        href = eval_xpath_getindex(result, './article/a/@href', 0)
+        href = href.split('?')[0]
+        href = href.split('/')[-1]
+        href = base64.urlsafe_b64decode(href + '====')
+        href = href[href.index(b'http') :].split(b'\xd2')[0]
+        href = href.decode()
 
-            # now we have : b'[null, ... null,"https://www.cnn.com/.../index.html"]'
-            url = re.findall('http[^;"]*', str(jslog))[0]
-
-        # the first <h3> tag in the <article> contains the title of the link
         title = extract_text(eval_xpath(result, './article/h3[1]'))
 
         # The pub_date is mostly a string like 'yesertday', not a real
         # timezone date or time.  Therefore we can't use publishedDate.
-        pub_date = extract_text(eval_xpath(result, './article/div[1]/div[1]/time'))
-        pub_origin = extract_text(eval_xpath(result, './article/div[1]/div[1]/a'))
+        pub_date = extract_text(eval_xpath(result, './article//time'))
+        pub_origin = extract_text(eval_xpath(result, './article//a[@data-n-tid]'))
 
         content = ' / '.join([x for x in [pub_origin, pub_date] if x])
 
@@ -165,7 +170,7 @@ def response(resp):
 
         results.append(
             {
-                'url': url,
+                'url': href,
                 'title': title,
                 'content': content,
                 'img_src': img_src,
@@ -174,3 +179,127 @@ def response(resp):
 
     # return results
     return results
+
+
+ceid_list = [
+    'AE:ar',
+    'AR:es-419',
+    'AT:de',
+    'AU:en',
+    'BD:bn',
+    'BE:fr',
+    'BE:nl',
+    'BG:bg',
+    'BR:pt-419',
+    'BW:en',
+    'CA:en',
+    'CA:fr',
+    'CH:de',
+    'CH:fr',
+    'CL:es-419',
+    'CN:zh-Hans',
+    'CO:es-419',
+    'CU:es-419',
+    'CZ:cs',
+    'DE:de',
+    'EG:ar',
+    'ES:es',
+    'ET:en',
+    'FR:fr',
+    'GB:en',
+    'GH:en',
+    'GR:el',
+    'HK:zh-Hant',
+    'HU:hu',
+    'ID:en',
+    'ID:id',
+    'IE:en',
+    'IL:en',
+    'IL:he',
+    'IN:bn',
+    'IN:en',
+    'IN:hi',
+    'IN:ml',
+    'IN:mr',
+    'IN:ta',
+    'IN:te',
+    'IT:it',
+    'JP:ja',
+    'KE:en',
+    'KR:ko',
+    'LB:ar',
+    'LT:lt',
+    'LV:en',
+    'LV:lv',
+    'MA:fr',
+    'MX:es-419',
+    'MY:en',
+    'NA:en',
+    'NG:en',
+    'NL:nl',
+    'NO:no',
+    'NZ:en',
+    'PE:es-419',
+    'PH:en',
+    'PK:en',
+    'PL:pl',
+    'PT:pt-150',
+    'RO:ro',
+    'RS:sr',
+    'RU:ru',
+    'SA:ar',
+    'SE:sv',
+    'SG:en',
+    'SI:sl',
+    'SK:sk',
+    'SN:fr',
+    'TH:th',
+    'TR:tr',
+    'TW:zh-Hant',
+    'TZ:en',
+    'UA:ru',
+    'UA:uk',
+    'UG:en',
+    'US:en',
+    'US:es-419',
+    'VE:es-419',
+    'VN:vi',
+    'ZA:en',
+    'ZW:en',
+]
+"""List of region/language combinations supported by Google News.  Values of the
+``ceid`` argument of the Google News REST API."""
+
+
+_skip_values = [
+    'ET:en',  # english (ethiopia)
+    'ID:en',  # english (indonesia)
+    'LV:en',  # english (latvia)
+]
+
+_ceid_locale_map = {'NO:no': 'nb-NO'}
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    _fetch_traits(engine_traits, add_domains=False)
+
+    engine_traits.custom['ceid'] = {}
+
+    for ceid in ceid_list:
+        if ceid in _skip_values:
+            continue
+
+        region, lang = ceid.split(':')
+        x = lang.split('-')
+        if len(x) > 1:
+            if x[1] not in ['Hant', 'Hans']:
+                lang = x[0]
+
+        sxng_locale = _ceid_locale_map.get(ceid, lang + '-' + region)
+        try:
+            locale = babel.Locale.parse(sxng_locale, sep='-')
+        except babel.UnknownLocaleError:
+            print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
+            continue
+
+        engine_traits.custom['ceid'][locales.region_tag(locale)] = ceid

searx/engines/google_scholar.py

@@ -1,19 +1,18 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""Google (Scholar)
+"""This is the implementation of the Google Scholar engine.
 
-For detailed description of the *REST-full* API see: `Query Parameter
-Definitions`_.
-
-.. _Query Parameter Definitions:
-   https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
+Compared to other Google services the Scholar engine has a simple GET REST-API
+and there does not exists `async` API.  Even though the API slightly vintage we
+can make use of the :ref:`google API` to assemble the arguments of the GET
+request.
 """
 
-# pylint: disable=invalid-name
+from typing import TYPE_CHECKING
+from typing import Optional
 
 from urllib.parse import urlencode
 from datetime import datetime
-from typing import Optional
 from lxml import html
 
 from searx.utils import (
@@ -23,19 +22,21 @@ from searx.utils import (
     extract_text,
 )
 
+from searx.exceptions import SearxEngineCaptchaException
+
+from searx.engines.google import fetch_traits  # pylint: disable=unused-import
 from searx.engines.google import (
-    get_lang_info,
+    get_google_info,
     time_range_dict,
-    detect_google_sorry,
 )
+from searx.enginelib.traits import EngineTraits
 
-# pylint: disable=unused-import
-from searx.engines.google import (
-    supported_languages_url,
-    _fetch_supported_languages,
-)
+if TYPE_CHECKING:
+    import logging
 
-# pylint: enable=unused-import
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
@@ -51,53 +52,62 @@ about = {
 categories = ['science', 'scientific publications']
 paging = True
 language_support = True
-use_locale_domain = True
 time_range_support = True
 safesearch = False
 send_accept_language_header = True
 
 
-def time_range_url(params):
-    """Returns a URL query component for a google-Scholar time range based on
-    ``params['time_range']``.  Google-Scholar does only support ranges in years.
-    To have any effect, all the Searx ranges (*day*, *week*, *month*, *year*)
-    are mapped to *year*.  If no range is set, an empty string is returned.
-    Example::
+def time_range_args(params):
+    """Returns a dictionary with a time range arguments based on
+    ``params['time_range']``.
+
+    Google Scholar supports a detailed search by year.  Searching by *last
+    month* or *last week* (as offered by SearXNG) is uncommon for scientific
+    publications and is not supported by Google Scholar.
+
+    To limit the result list when the users selects a range, all the SearXNG
+    ranges (*day*, *week*, *month*, *year*) are mapped to *year*.  If no range
+    is set an empty dictionary of arguments is returned.  Example;  when
+    user selects a time range (current year minus one in 2022):
+
+    .. code:: python
+
+        { 'as_ylo' : 2021 }
 
-        &as_ylo=2019
     """
-    # as_ylo=2016&as_yhi=2019
-    ret_val = ''
+    ret_val = {}
     if params['time_range'] in time_range_dict:
-        ret_val = urlencode({'as_ylo': datetime.now().year - 1})
-    return '&' + ret_val
+        ret_val['as_ylo'] = datetime.now().year - 1
+    return ret_val
+
+
+def detect_google_captcha(dom):
+    """In case of CAPTCHA Google Scholar open its own *not a Robot* dialog and is
+    not redirected to ``sorry.google.com``.
+    """
+    if eval_xpath(dom, "//form[@id='gs_captcha_f']"):
+        raise SearxEngineCaptchaException()
 
 
 def request(query, params):
     """Google-Scholar search request"""
 
-    offset = (params['pageno'] - 1) * 10
-    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
-
+    google_info = get_google_info(params, traits)
     # subdomain is: scholar.google.xy
-    lang_info['subdomain'] = lang_info['subdomain'].replace("www.", "scholar.")
+    google_info['subdomain'] = google_info['subdomain'].replace("www.", "scholar.")
 
-    query_url = (
-        'https://'
-        + lang_info['subdomain']
-        + '/scholar'
-        + "?"
-        + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'start': offset})
-    )
+    args = {
+        'q': query,
+        **google_info['params'],
+        'start': (params['pageno'] - 1) * 10,
+        'as_sdt': '2007',  # include patents / to disable set '0,5'
+        'as_vis': '0',  # include citations / to disable set '1'
+    }
+    args.update(time_range_args(params))
 
-    query_url += time_range_url(params)
-    params['url'] = query_url
-
-    params['cookies']['CONSENT'] = "YES+"
-    params['headers'].update(lang_info['headers'])
-    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
-
-    # params['google_subdomain'] = subdomain
+    params['url'] = 'https://' + google_info['subdomain'] + '/scholar?' + urlencode(args)
+    params['cookies'] = google_info['cookies']
+    params['headers'].update(google_info['headers'])
     return params
 
 
@@ -138,19 +148,15 @@ def parse_gs_a(text: Optional[str]):
 
 
 def response(resp):  # pylint: disable=too-many-locals
-    """Get response from google's search request"""
+    """Parse response from Google Scholar"""
     results = []
 
-    detect_google_sorry(resp)
-
-    # which subdomain ?
-    # subdomain = resp.search_params.get('google_subdomain')
-
     # convert the text to dom
     dom = html.fromstring(resp.text)
+    detect_google_captcha(dom)
 
     # parse results
-    for result in eval_xpath_list(dom, '//div[@data-cid]'):
+    for result in eval_xpath_list(dom, '//div[@data-rp]'):
 
         title = extract_text(eval_xpath(result, './/h3[1]//a'))
 
@@ -158,7 +164,7 @@ def response(resp):  # pylint: disable=too-many-locals
             # this is a [ZITATION] block
             continue
 
-        pub_type = extract_text(eval_xpath(result, './/span[@class="gs_ct1"]'))
+        pub_type = extract_text(eval_xpath(result, './/span[@class="gs_ctg2"]'))
         if pub_type:
             pub_type = pub_type[1:-1].lower()
 

searx/engines/google_videos.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the google videos engine.
+"""This is the implementation of the Google Videos engine.
 
 .. admonition:: Content-Security-Policy (CSP)
 
@@ -14,9 +14,8 @@
 
 """
 
-# pylint: disable=invalid-name
+from typing import TYPE_CHECKING
 
-import re
 from urllib.parse import urlencode
 from lxml import html
 
@@ -27,20 +26,22 @@ from searx.utils import (
     extract_text,
 )
 
+from searx.engines.google import fetch_traits  # pylint: disable=unused-import
 from searx.engines.google import (
-    get_lang_info,
+    get_google_info,
     time_range_dict,
     filter_mapping,
-    g_section_with_header,
-    title_xpath,
     suggestion_xpath,
     detect_google_sorry,
 )
+from searx.enginelib.traits import EngineTraits
 
-# pylint: disable=unused-import
-from searx.engines.google import supported_languages_url, _fetch_supported_languages
+if TYPE_CHECKING:
+    import logging
 
-# pylint: enable=unused-import
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
@@ -55,70 +56,32 @@ about = {
 # engine dependent config
 
 categories = ['videos', 'web']
-paging = False
+paging = True
 language_support = True
-use_locale_domain = True
 time_range_support = True
 safesearch = True
-send_accept_language_header = True
-
-RE_CACHE = {}
-
-
-def _re(regexpr):
-    """returns compiled regular expression"""
-    RE_CACHE[regexpr] = RE_CACHE.get(regexpr, re.compile(regexpr))
-    return RE_CACHE[regexpr]
-
-
-def scrap_out_thumbs_src(dom):
-    ret_val = {}
-    thumb_name = 'dimg_'
-    for script in eval_xpath_list(dom, '//script[contains(., "google.ldi={")]'):
-        _script = script.text
-        # "dimg_35":"https://i.ytimg.c....",
-        _dimurl = _re("s='([^']*)").findall(_script)
-        for k, v in _re('(' + thumb_name + '[0-9]*)":"(http[^"]*)').findall(_script):
-            v = v.replace(r'\u003d', '=')
-            v = v.replace(r'\u0026', '&')
-            ret_val[k] = v
-    logger.debug("found %s imgdata for: %s", thumb_name, ret_val.keys())
-    return ret_val
-
-
-def scrap_out_thumbs(dom):
-    """Scrap out thumbnail data from <script> tags."""
-    ret_val = {}
-    thumb_name = 'dimg_'
-
-    for script in eval_xpath_list(dom, '//script[contains(., "_setImagesSrc")]'):
-        _script = script.text
-
-        # var s='data:image/jpeg;base64, ...'
-        _imgdata = _re("s='([^']*)").findall(_script)
-        if not _imgdata:
-            continue
-
-        # var ii=['dimg_17']
-        for _vidthumb in _re(r"(%s\d+)" % thumb_name).findall(_script):
-            # At least the equal sign in the URL needs to be decoded
-            ret_val[_vidthumb] = _imgdata[0].replace(r"\x3d", "=")
-
-    logger.debug("found %s imgdata for: %s", thumb_name, ret_val.keys())
-    return ret_val
 
 
 def request(query, params):
     """Google-Video search request"""
 
-    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
+    google_info = get_google_info(params, traits)
 
     query_url = (
         'https://'
-        + lang_info['subdomain']
+        + google_info['subdomain']
         + '/search'
         + "?"
-        + urlencode({'q': query, 'tbm': "vid", **lang_info['params'], 'ie': "utf8", 'oe': "utf8"})
+        + urlencode(
+            {
+                'q': query,
+                'tbm': "vid",
+                'start': 10 * params['pageno'],
+                **google_info['params'],
+                'asearch': 'arc',
+                'async': 'use_ac:true,_fmt:html',
+            }
+        )
     )
 
     if params['time_range'] in time_range_dict:
@@ -127,9 +90,8 @@ def request(query, params):
         query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
     params['url'] = query_url
 
-    params['cookies']['CONSENT'] = "YES+"
-    params['headers'].update(lang_info['headers'])
-    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+    params['cookies'] = google_info['cookies']
+    params['headers'].update(google_info['headers'])
     return params
 
 
@@ -141,43 +103,30 @@ def response(resp):
 
     # convert the text to dom
     dom = html.fromstring(resp.text)
-    vidthumb_imgdata = scrap_out_thumbs(dom)
-    thumbs_src = scrap_out_thumbs_src(dom)
-    logger.debug(str(thumbs_src))
 
     # parse results
     for result in eval_xpath_list(dom, '//div[contains(@class, "g ")]'):
 
-        # ignore google *sections*
-        if extract_text(eval_xpath(result, g_section_with_header)):
-            logger.debug("ignoring <g-section-with-header>")
+        img_src = eval_xpath_getindex(result, './/img/@src', 0, None)
+        if img_src is None:
             continue
 
-        # ingnore articles without an image id / e.g. news articles
-        img_id = eval_xpath_getindex(result, './/g-img/img/@id', 0, default=None)
-        if img_id is None:
-            logger.error("no img_id found in item %s (news article?)", len(results) + 1)
-            continue
+        title = extract_text(eval_xpath_getindex(result, './/a/h3[1]', 0))
+        url = eval_xpath_getindex(result, './/a/h3[1]/../@href', 0)
 
-        img_src = vidthumb_imgdata.get(img_id, None)
-        if not img_src:
-            img_src = thumbs_src.get(img_id, "")
-
-        title = extract_text(eval_xpath_getindex(result, title_xpath, 0))
-        url = eval_xpath_getindex(result, './/div[@class="dXiKIc"]//a/@href', 0)
-        length = extract_text(eval_xpath(result, './/div[contains(@class, "P7xzyf")]/span/span'))
         c_node = eval_xpath_getindex(result, './/div[@class="Uroaid"]', 0)
         content = extract_text(c_node)
-        pub_info = extract_text(eval_xpath(result, './/div[@class="Zg1NU"]'))
+        pub_info = extract_text(eval_xpath(result, './/div[@class="P7xzyf"]'))
+        length = extract_text(eval_xpath(result, './/div[@class="J1mWY"]'))
 
         results.append(
             {
                 'url': url,
                 'title': title,
                 'content': content,
-                'length': length,
                 'author': pub_info,
                 'thumbnail': img_src,
+                'length': length,
                 'template': 'videos.html',
             }
         )

searx/engines/peertube.py

@@ -1,18 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- peertube (Videos)
+# lint: pylint
+"""Peertube and :py:obj:`SepiaSearch <searx.engines.sepiasearch>` do share
+(more or less) the same REST API and the schema of the JSON result is identical.
+
 """
 
-from json import loads
-from datetime import datetime
+import re
 from urllib.parse import urlencode
-from searx.utils import html_to_text
+from datetime import datetime
+from dateutil.parser import parse
+from dateutil.relativedelta import relativedelta
+
+import babel
+
+from searx import network
+from searx.locales import language_tag
+from searx.utils import html_to_text
+from searx.enginelib.traits import EngineTraits
+
+traits: EngineTraits
 
-# about
 about = {
+    # pylint: disable=line-too-long
     "website": 'https://joinpeertube.org',
     "wikidata_id": 'Q50938515',
-    "official_api_documentation": 'https://docs.joinpeertube.org/api-rest-reference.html',
+    "official_api_documentation": 'https://docs.joinpeertube.org/api-rest-reference.html#tag/Search/operation/searchVideos',
     "use_official_api": True,
     "require_api_key": False,
     "results": 'JSON',
@@ -22,66 +34,155 @@ about = {
 categories = ["videos"]
 paging = True
 base_url = "https://peer.tube"
-supported_languages_url = 'https://peer.tube/api/v1/videos/languages'
+"""Base URL of the Peertube instance.  A list of instances is available at:
+
+- https://instances.joinpeertube.org/instances
+"""
+
+time_range_support = True
+time_range_table = {
+    'day': relativedelta(),
+    'week': relativedelta(weeks=-1),
+    'month': relativedelta(months=-1),
+    'year': relativedelta(years=-1),
+}
+
+safesearch = True
+safesearch_table = {0: 'both', 1: 'false', 2: 'false'}
+
+
+def minute_to_hm(minute):
+    if isinstance(minute, int):
+        return "%d:%02d" % (divmod(minute, 60))
+    return None
 
 
-# do search-request
 def request(query, params):
-    sanitized_url = base_url.rstrip("/")
-    pageno = (params["pageno"] - 1) * 15
-    search_url = sanitized_url + "/api/v1/search/videos/?pageno={pageno}&{query}"
-    query_dict = {"search": query}
-    language = params["language"].split("-")[0]
-    if "all" != language and language in supported_languages:
-        query_dict["languageOneOf"] = language
-    params["url"] = search_url.format(query=urlencode(query_dict), pageno=pageno)
+    """Assemble request for the Peertube API"""
+
+    if not query:
+        return False
+
+    # eng_region = traits.get_region(params['searxng_locale'], 'en_US')
+    eng_lang = traits.get_language(params['searxng_locale'], None)
+
+    params['url'] = (
+        base_url.rstrip("/")
+        + "/api/v1/search/videos?"
+        + urlencode(
+            {
+                'search': query,
+                'searchTarget': 'search-index',  # Vidiversum
+                'resultType': 'videos',
+                'start': (params['pageno'] - 1) * 10,
+                'count': 10,
+                # -createdAt: sort by date ascending / createdAt: date descending
+                'sort': '-match',  # sort by *match descending*
+                'nsfw': safesearch_table[params['safesearch']],
+            }
+        )
+    )
+
+    if eng_lang is not None:
+        params['url'] += '&languageOneOf[]=' + eng_lang
+        params['url'] += '&boostLanguages[]=' + eng_lang
+
+    if params['time_range'] in time_range_table:
+        time = datetime.now().date() + time_range_table[params['time_range']]
+        params['url'] += '&startDate=' + time.isoformat()
+
     return params
 
 
-def _get_offset_from_pageno(pageno):
-    return (pageno - 1) * 15 + 1
-
-
-# get response from search-request
 def response(resp):
-    sanitized_url = base_url.rstrip("/")
+    return video_response(resp)
+
+
+def video_response(resp):
+    """Parse video response from SepiaSearch and Peertube instances."""
     results = []
 
-    search_res = loads(resp.text)
+    json_data = resp.json()
 
-    # return empty array if there are no results
-    if "data" not in search_res:
+    if 'data' not in json_data:
         return []
 
-    # parse results
-    for res in search_res["data"]:
-        title = res["name"]
-        url = sanitized_url + "/videos/watch/" + res["uuid"]
-        description = res["description"]
-        if description:
-            content = html_to_text(res["description"])
-        else:
-            content = ""
-        thumbnail = sanitized_url + res["thumbnailPath"]
-        publishedDate = datetime.strptime(res["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
+    for result in json_data['data']:
+        metadata = [
+            x
+            for x in [
+                result.get('channel', {}).get('displayName'),
+                result.get('channel', {}).get('name') + '@' + result.get('channel', {}).get('host'),
+                ', '.join(result.get('tags', [])),
+            ]
+            if x
+        ]
 
         results.append(
             {
-                "template": "videos.html",
-                "url": url,
-                "title": title,
-                "content": content,
-                "publishedDate": publishedDate,
-                "iframe_src": sanitized_url + res["embedPath"],
-                "thumbnail": thumbnail,
+                'url': result['url'],
+                'title': result['name'],
+                'content': html_to_text(result.get('description') or ''),
+                'author': result.get('account', {}).get('displayName'),
+                'length': minute_to_hm(result.get('duration')),
+                'template': 'videos.html',
+                'publishedDate': parse(result['publishedAt']),
+                'iframe_src': result.get('embedUrl'),
+                'thumbnail': result.get('thumbnailUrl') or result.get('previewUrl'),
+                'metadata': ' | '.join(metadata),
             }
         )
 
-    # return results
     return results
 
 
-def _fetch_supported_languages(resp):
-    videolanguages = resp.json()
-    peertube_languages = list(videolanguages.keys())
-    return peertube_languages
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages from peertube's search-index source code.
+
+    See videoLanguages_ in commit `8ed5c729 - Refactor and redesign client`_
+
+    .. _8ed5c729 - Refactor and redesign client:
+       https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729
+    .. _videoLanguages:
+       https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729#3d8747f9a60695c367c70bb64efba8f403721fad_0_291
+    """
+
+    resp = network.get(
+        'https://framagit.org/framasoft/peertube/search-index/-/raw/master/client/src/components/Filters.vue',
+        # the response from search-index repository is very slow
+        timeout=60,
+    )
+
+    if not resp.ok:
+        print("ERROR: response from peertube is not OK.")
+        return
+
+    js_lang = re.search(r"videoLanguages \(\)[^\n]+(.*?)\]", resp.text, re.DOTALL)
+    if not js_lang:
+        print("ERROR: can't determine languages from peertube")
+        return
+
+    for lang in re.finditer(r"\{ id: '([a-z]+)', label:", js_lang.group(1)):
+        try:
+            eng_tag = lang.group(1)
+            if eng_tag == 'oc':
+                # Occitanis not known by babel, its closest relative is Catalan
+                # but 'ca' is already in the list of engine_traits.languages -->
+                # 'oc' will be ignored.
+                continue
+
+            sxng_tag = language_tag(babel.Locale.parse(eng_tag))
+
+        except babel.UnknownLocaleError:
+            print("ERROR: %s is unknown by babel" % eng_tag)
+            continue
+
+        conflict = engine_traits.languages.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+        engine_traits.languages[sxng_tag] = eng_tag
+
+    engine_traits.languages['zh_Hans'] = 'zh'
+    engine_traits.languages['zh_Hant'] = 'zh'

searx/engines/qwant.py

@@ -34,7 +34,9 @@ import babel
 
 from searx.exceptions import SearxEngineAPIException
 from searx.network import raise_for_httperror
-from searx.locales import get_engine_locale
+from searx.enginelib.traits import EngineTraits
+
+traits: EngineTraits
 
 # about
 about = {
@@ -49,7 +51,6 @@ about = {
 # engine dependent config
 categories = []
 paging = True
-supported_languages_url = about['website']
 qwant_categ = None  # web|news|inages|videos
 
 safesearch = True
@@ -95,7 +96,7 @@ def request(query, params):
     )
 
     # add quant's locale
-    q_locale = get_engine_locale(params['language'], supported_languages, default='en_US')
+    q_locale = traits.get_region(params["searxng_locale"], default='en_US')
     params['url'] += '&locale=' + q_locale
 
     # add safesearch option
@@ -243,15 +244,20 @@ def response(resp):
     return results
 
 
-def _fetch_supported_languages(resp):
+def fetch_traits(engine_traits: EngineTraits):
 
+    # pylint: disable=import-outside-toplevel
+    from searx import network
+    from searx.locales import region_tag
+
+    resp = network.get(about['website'])
     text = resp.text
     text = text[text.find('INITIAL_PROPS') :]
     text = text[text.find('{') : text.find('</script>')]
 
     q_initial_props = loads(text)
     q_locales = q_initial_props.get('locales')
-    q_valid_locales = []
+    eng_tag_list = set()
 
     for country, v in q_locales.items():
         for lang in v['langs']:
@@ -261,25 +267,18 @@ def _fetch_supported_languages(resp):
                 # qwant-news does not support all locales from qwant-web:
                 continue
 
-            q_valid_locales.append(_locale)
+            eng_tag_list.add(_locale)
 
-    supported_languages = {}
-
-    for q_locale in q_valid_locales:
+    for eng_tag in eng_tag_list:
         try:
-            locale = babel.Locale.parse(q_locale, sep='_')
-        except babel.core.UnknownLocaleError:
-            print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
+            sxng_tag = region_tag(babel.Locale.parse(eng_tag, sep='_'))
+        except babel.UnknownLocaleError:
+            print("ERROR: can't determine babel locale of quant's locale %s" % eng_tag)
             continue
 
-        # note: supported_languages (dict)
-        #
-        #   dict's key is a string build up from a babel.Locale object / the
-        #   notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
-        #   language) notation and dict's values are the locale strings used by
-        #   the engine.
-
-        searxng_locale = locale.language + '-' + locale.territory  # --> params['language']
-        supported_languages[searxng_locale] = q_locale
-
-    return supported_languages
+        conflict = engine_traits.regions.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+        engine_traits.regions[sxng_tag] = eng_tag

searx/engines/sepiasearch.py

@@ -1,70 +1,80 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- SepiaSearch (Videos)
+# lint: pylint
+"""SepiaSearch uses the same languages as :py:obj:`Peertube
+<searx.engines.peertube>` and the response is identical to the response from the
+peertube engines.
+
 """
 
-from json import loads
-from dateutil import parser, relativedelta
+from typing import TYPE_CHECKING
+
 from urllib.parse import urlencode
 from datetime import datetime
 
-# about
+from searx.engines.peertube import fetch_traits  # pylint: disable=unused-import
+from searx.engines.peertube import (
+    # pylint: disable=unused-import
+    video_response,
+    safesearch_table,
+    time_range_table,
+)
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
+
 about = {
+    # pylint: disable=line-too-long
     "website": 'https://sepiasearch.org',
     "wikidata_id": None,
-    "official_api_documentation": "https://framagit.org/framasoft/peertube/search-index/-/tree/master/server/controllers/api",  # NOQA
+    "official_api_documentation": 'https://docs.joinpeertube.org/api-rest-reference.html#tag/Search/operation/searchVideos',
     "use_official_api": True,
     "require_api_key": False,
     "results": 'JSON',
 }
 
+# engine dependent config
 categories = ['videos']
 paging = True
+
+base_url = 'https://sepiasearch.org'
+
 time_range_support = True
 safesearch = True
-supported_languages = [
-    # fmt: off
-    'en', 'fr', 'ja', 'eu', 'ca', 'cs', 'eo', 'el',
-    'de', 'it', 'nl', 'es', 'oc', 'gd', 'zh', 'pt',
-    'sv', 'pl', 'fi', 'ru'
-    # fmt: on
-]
-base_url = 'https://sepiasearch.org/api/v1/search/videos'
-
-safesearch_table = {0: 'both', 1: 'false', 2: 'false'}
-
-time_range_table = {
-    'day': relativedelta.relativedelta(),
-    'week': relativedelta.relativedelta(weeks=-1),
-    'month': relativedelta.relativedelta(months=-1),
-    'year': relativedelta.relativedelta(years=-1),
-}
-
-
-def minute_to_hm(minute):
-    if isinstance(minute, int):
-        return "%d:%02d" % (divmod(minute, 60))
-    return None
 
 
 def request(query, params):
+    """Assemble request for the SepiaSearch API"""
+
+    if not query:
+        return False
+
+    # eng_region = traits.get_region(params['searxng_locale'], 'en_US')
+    eng_lang = traits.get_language(params['searxng_locale'], None)
+
     params['url'] = (
-        base_url
-        + '?'
+        base_url.rstrip("/")
+        + "/api/v1/search/videos?"
         + urlencode(
             {
                 'search': query,
                 'start': (params['pageno'] - 1) * 10,
                 'count': 10,
-                'sort': '-match',
+                # -createdAt: sort by date ascending / createdAt: date descending
+                'sort': '-match',  # sort by *match descending*
                 'nsfw': safesearch_table[params['safesearch']],
             }
         )
     )
 
-    language = params['language'].split('-')[0]
-    if language in supported_languages:
-        params['url'] += '&languageOneOf[]=' + language
+    if eng_lang is not None:
+        params['url'] += '&languageOneOf[]=' + eng_lang
+        params['url'] += '&boostLanguages[]=' + eng_lang
+
     if params['time_range'] in time_range_table:
         time = datetime.now().date() + time_range_table[params['time_range']]
         params['url'] += '&startDate=' + time.isoformat()
@@ -73,34 +83,4 @@ def request(query, params):
 
 
 def response(resp):
-    results = []
-
-    search_results = loads(resp.text)
-
-    if 'data' not in search_results:
-        return []
-
-    for result in search_results['data']:
-        title = result['name']
-        content = result['description']
-        thumbnail = result['thumbnailUrl']
-        publishedDate = parser.parse(result['publishedAt'])
-        author = result.get('account', {}).get('displayName')
-        length = minute_to_hm(result.get('duration'))
-        url = result['url']
-
-        results.append(
-            {
-                'url': url,
-                'title': title,
-                'content': content,
-                'author': author,
-                'length': length,
-                'template': 'videos.html',
-                'publishedDate': publishedDate,
-                'iframe_src': result.get('embedUrl'),
-                'thumbnail': thumbnail,
-            }
-        )
-
-    return results
+    return video_response(resp)

searx/engines/seznam.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- Seznam
+# lint: pylint
+"""Seznam
+
 """
 
 from urllib.parse import urlencode
@@ -11,7 +12,6 @@ from searx.utils import (
     extract_text,
     eval_xpath_list,
     eval_xpath_getindex,
-    eval_xpath,
 )
 
 # about
@@ -54,8 +54,12 @@ def response(resp):
     results = []
 
     dom = html.fromstring(resp.content.decode())
-    for result_element in eval_xpath_list(dom, '//div[@data-dot="results"]/div'):
-        result_data = eval_xpath_getindex(result_element, './/div[contains(@class, "bec586")]', 0, default=None)
+    for result_element in eval_xpath_list(
+        dom, '//div[@id="searchpage-root"]//div[@class="Layout--left"]/div[@class="f2c528"]'
+    ):
+        result_data = eval_xpath_getindex(
+            result_element, './/div[@class="c8774a" or @class="e69e8d a11657"]', 0, default=None
+        )
         if result_data is None:
             continue
         title_element = eval_xpath_getindex(result_element, './/h3/a', 0)
@@ -63,7 +67,7 @@ def response(resp):
             {
                 'url': title_element.get('href'),
                 'title': extract_text(title_element),
-                'content': extract_text(eval_xpath(result_data, './/div[@class="_3eded7"]')),
+                'content': extract_text(result_data),
             }
         )
 

searx/engines/startpage.py

@@ -1,28 +1,108 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""Startpage (Web)
+"""Startpage's language & region selectors are a mess ..
+
+.. _startpage regions:
+
+Startpage regions
+=================
+
+In the list of regions there are tags we need to map to common region tags::
+
+  pt-BR_BR --> pt_BR
+  zh-CN_CN --> zh_Hans_CN
+  zh-TW_TW --> zh_Hant_TW
+  zh-TW_HK --> zh_Hant_HK
+  en-GB_GB --> en_GB
+
+and there is at least one tag with a three letter language tag (ISO 639-2)::
+
+  fil_PH --> fil_PH
+
+The locale code ``no_NO`` from Startpage does not exists and is mapped to
+``nb-NO``::
+
+    babel.core.UnknownLocaleError: unknown locale 'no_NO'
+
+For reference see languages-subtag at iana; ``no`` is the macrolanguage [1]_ and
+W3C recommends subtag over macrolanguage [2]_.
+
+.. [1] `iana: language-subtag-registry
+   <https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry>`_ ::
+
+      type: language
+      Subtag: nb
+      Description: Norwegian Bokmål
+      Added: 2005-10-16
+      Suppress-Script: Latn
+      Macrolanguage: no
+
+.. [2]
+   Use macrolanguages with care.  Some language subtags have a Scope field set to
+   macrolanguage, i.e. this primary language subtag encompasses a number of more
+   specific primary language subtags in the registry.  ...  As we recommended for
+   the collection subtags mentioned above, in most cases you should try to use
+   the more specific subtags ... `W3: The primary language subtag
+   <https://www.w3.org/International/questions/qa-choosing-language-tags#langsubtag>`_
+
+.. _startpage languages:
+
+Startpage languages
+===================
+
+:py:obj:`send_accept_language_header`:
+  The displayed name in Startpage's settings page depend on the location of the
+  IP when ``Accept-Language`` HTTP header is unset.  In :py:obj:`fetch_traits`
+  we use::
+
+    'Accept-Language': "en-US,en;q=0.5",
+    ..
+
+  to get uniform names independent from the IP).
+
+.. _startpage categories:
+
+Startpage categories
+====================
+
+Startpage's category (for Web-search, News, Videos, ..) is set by
+:py:obj:`startpage_categ` in  settings.yml::
+
+  - name: startpage
+    engine: startpage
+    startpage_categ: web
+    ...
+
+.. hint::
+
+   The default category is ``web`` .. and other categories than ``web`` are not
+   yet implemented.
 
 """
 
+from typing import TYPE_CHECKING
+from collections import OrderedDict
 import re
-from time import time
-
-from urllib.parse import urlencode
 from unicodedata import normalize, combining
+from time import time
 from datetime import datetime, timedelta
 
-from dateutil import parser
-from lxml import html
-from babel import Locale
-from babel.localedata import locale_identifiers
+import dateutil.parser
+import lxml.html
+import babel
 
-from searx.network import get
-from searx.utils import extract_text, eval_xpath, match_language
-from searx.exceptions import (
-    SearxEngineResponseException,
-    SearxEngineCaptchaException,
-)
+from searx import network
+from searx.utils import extract_text, eval_xpath, gen_useragent
+from searx.exceptions import SearxEngineCaptchaException
+from searx.locales import region_tag
+from searx.enginelib.traits import EngineTraits
 
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
@@ -34,18 +114,28 @@ about = {
     "results": 'HTML',
 }
 
+startpage_categ = 'web'
+"""Startpage's category, visit :ref:`startpage categories`.
+"""
+
+send_accept_language_header = True
+"""Startpage tries to guess user's language and territory from the HTTP
+``Accept-Language``.  Optional the user can select a search-language (can be
+different to the UI language) and a region filter.
+"""
+
 # engine dependent config
 categories = ['general', 'web']
-# there is a mechanism to block "bot" search
-# (probably the parameter qid), require
-# storing of qid's between mulitble search-calls
-
 paging = True
-supported_languages_url = 'https://www.startpage.com/do/settings'
+time_range_support = True
+safesearch = True
+
+time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
+safesearch_dict = {0: '0', 1: '1', 2: '1'}
 
 # search-url
-base_url = 'https://startpage.com/'
-search_url = base_url + 'sp/search?'
+base_url = 'https://www.startpage.com'
+search_url = base_url + '/sp/search'
 
 # specific xpath variables
 # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
@@ -53,92 +143,193 @@ search_url = base_url + 'sp/search?'
 results_xpath = '//div[@class="w-gl__result__main"]'
 link_xpath = './/a[@class="w-gl__result-title result-link"]'
 content_xpath = './/p[@class="w-gl__description"]'
+search_form_xpath = '//form[@id="search"]'
+"""XPath of Startpage's origin search form
+
+.. code: html
+
+    <form action="/sp/search" method="post">
+      <input type="text" name="query"  value="" ..>
+      <input type="hidden" name="t" value="device">
+      <input type="hidden" name="lui" value="english">
+      <input type="hidden" name="sc" value="Q7Mt5TRqowKB00">
+      <input type="hidden" name="cat" value="web">
+      <input type="hidden" class="abp" id="abp-input" name="abp" value="1">
+    </form>
+"""
 
 # timestamp of the last fetch of 'sc' code
 sc_code_ts = 0
 sc_code = ''
+sc_code_cache_sec = 30
+"""Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
 
 
-def raise_captcha(resp):
+def get_sc_code(searxng_locale, params):
+    """Get an actual ``sc`` argument from Startpage's search form (HTML page).
 
-    if str(resp.url).startswith('https://www.startpage.com/sp/captcha'):
-        raise SearxEngineCaptchaException()
+    Startpage puts a ``sc`` argument on every HTML :py:obj:`search form
+    <search_form_xpath>`.  Without this argument Startpage considers the request
+    is from a bot.  We do not know what is encoded in the value of the ``sc``
+    argument, but it seems to be a kind of a *time-stamp*.
 
-
-def get_sc_code(headers):
-    """Get an actual `sc` argument from startpage's home page.
-
-    Startpage puts a `sc` argument on every link.  Without this argument
-    startpage considers the request is from a bot.  We do not know what is
-    encoded in the value of the `sc` argument, but it seems to be a kind of a
-    *time-stamp*.  This *time-stamp* is valid for a few hours.
-
-    This function scrap a new *time-stamp* from startpage's home page every hour
-    (3000 sec).
+    Startpage's search form generates a new sc-code on each request.  This
+    function scrap a new sc-code from Startpage's home page every
+    :py:obj:`sc_code_cache_sec` seconds.
 
     """
 
     global sc_code_ts, sc_code  # pylint: disable=global-statement
 
-    if time() > (sc_code_ts + 3000):
-        logger.debug("query new sc time-stamp ...")
+    if sc_code and (time() < (sc_code_ts + sc_code_cache_sec)):
+        logger.debug("get_sc_code: reuse '%s'", sc_code)
+        return sc_code
 
-        resp = get(base_url, headers=headers)
-        raise_captcha(resp)
-        dom = html.fromstring(resp.text)
+    headers = {**params['headers']}
+    headers['Origin'] = base_url
+    headers['Referer'] = base_url + '/'
+    # headers['Connection'] = 'keep-alive'
+    # headers['Accept-Encoding'] = 'gzip, deflate, br'
+    # headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'
+    # headers['User-Agent'] = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0'
+
+    # add Accept-Language header
+    if searxng_locale == 'all':
+        searxng_locale = 'en-US'
+    locale = babel.Locale.parse(searxng_locale, sep='-')
+
+    if send_accept_language_header:
+        ac_lang = locale.language
+        if locale.territory:
+            ac_lang = "%s-%s,%s;q=0.9,*;q=0.5" % (
+                locale.language,
+                locale.territory,
+                locale.language,
+            )
+        headers['Accept-Language'] = ac_lang
+
+    get_sc_url = base_url + '/?sc=%s' % (sc_code)
+    logger.debug("query new sc time-stamp ... %s", get_sc_url)
+    logger.debug("headers: %s", headers)
+    resp = network.get(get_sc_url, headers=headers)
+
+    # ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
+    # ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
+    # ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
+
+    if str(resp.url).startswith('https://www.startpage.com/sp/captcha'):
+        raise SearxEngineCaptchaException(
+            message="get_sc_code: got redirected to https://www.startpage.com/sp/captcha",
+        )
+
+    dom = lxml.html.fromstring(resp.text)
 
     try:
-            # <input type="hidden" name="sc" value="...">
-            sc_code = eval_xpath(dom, '//input[@name="sc"]/@value')[0]
+        sc_code = eval_xpath(dom, search_form_xpath + '//input[@name="sc"]/@value')[0]
     except IndexError as exc:
-            # suspend startpage API --> https://github.com/searxng/searxng/pull/695
-            raise SearxEngineResponseException(
-                suspended_time=7 * 24 * 3600, message="PR-695: query new sc time-stamp failed!"
+        logger.debug("suspend startpage API --> https://github.com/searxng/searxng/pull/695")
+        raise SearxEngineCaptchaException(
+            message="get_sc_code: [PR-695] query new sc time-stamp failed! (%s)" % resp.url,
         ) from exc
 
     sc_code_ts = time()
-        logger.debug("new value is: %s", sc_code)
-
+    logger.debug("get_sc_code: new value is: %s", sc_code)
     return sc_code
 
 
-# do search-request
 def request(query, params):
+    """Assemble a Startpage request.
 
-    # pylint: disable=line-too-long
-    # The format string from Startpage's FFox add-on [1]::
-    #
-    #     https://www.startpage.com/do/dsearch?query={searchTerms}&cat=web&pl=ext-ff&language=__MSG_extensionUrlLanguage__&extVersion=1.3.0
-    #
-    # [1] https://addons.mozilla.org/en-US/firefox/addon/startpage-private-search/
+    To avoid CAPTCHA we need to send a well formed HTTP POST request with a
+    cookie.  We need to form a request that is identical to the request build by
+    Startpage's search form:
 
+    - in the cookie the **region** is selected
+    - in the HTTP POST data the **language** is selected
+
+    Additionally the arguments form Startpage's search form needs to be set in
+    HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
+    """
+    if startpage_categ == 'web':
+        return _request_cat_web(query, params)
+
+    logger.error("Startpages's category '%' is not yet implemented.", startpage_categ)
+    return params
+
+
+def _request_cat_web(query, params):
+
+    engine_region = traits.get_region(params['searxng_locale'], 'en-US')
+    engine_language = traits.get_language(params['searxng_locale'], 'en')
+
+    # build arguments
     args = {
         'query': query,
-        'page': params['pageno'],
         'cat': 'web',
-        # 'pl': 'ext-ff',
-        # 'extVersion': '1.3.0',
-        # 'abp': "-1",
-        'sc': get_sc_code(params['headers']),
+        't': 'device',
+        'sc': get_sc_code(params['searxng_locale'], params),  # hint: this func needs HTTP headers,
+        'with_date': time_range_dict.get(params['time_range'], ''),
     }
 
-    # set language if specified
-    if params['language'] != 'all':
-        lang_code = match_language(params['language'], supported_languages, fallback=None)
-        if lang_code:
-            language_name = supported_languages[lang_code]['alias']
-            args['language'] = language_name
-            args['lui'] = language_name
+    if engine_language:
+        args['language'] = engine_language
+        args['lui'] = engine_language
+
+    args['abp'] = '1'
+    if params['pageno'] > 1:
+        args['page'] = params['pageno']
+
+    # build cookie
+    lang_homepage = 'en'
+    cookie = OrderedDict()
+    cookie['date_time'] = 'world'
+    cookie['disable_family_filter'] = safesearch_dict[params['safesearch']]
+    cookie['disable_open_in_new_window'] = '0'
+    cookie['enable_post_method'] = '1'  # hint: POST
+    cookie['enable_proxy_safety_suggest'] = '1'
+    cookie['enable_stay_control'] = '1'
+    cookie['instant_answers'] = '1'
+    cookie['lang_homepage'] = 's/device/%s/' % lang_homepage
+    cookie['num_of_results'] = '10'
+    cookie['suggestions'] = '1'
+    cookie['wt_unit'] = 'celsius'
+
+    if engine_language:
+        cookie['language'] = engine_language
+        cookie['language_ui'] = engine_language
+
+    if engine_region:
+        cookie['search_results_region'] = engine_region
+
+    params['cookies']['preferences'] = 'N1N'.join(["%sEEE%s" % x for x in cookie.items()])
+    logger.debug('cookie preferences: %s', params['cookies']['preferences'])
+
+    # POST request
+    logger.debug("data: %s", args)
+    params['data'] = args
+    params['method'] = 'POST'
+    params['url'] = search_url
+    params['headers']['Origin'] = base_url
+    params['headers']['Referer'] = base_url + '/'
+    # is the Accept header needed?
+    # params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
 
-    params['url'] = search_url + urlencode(args)
     return params
 
 
 # get response from search-request
 def response(resp):
-    results = []
+    dom = lxml.html.fromstring(resp.text)
 
-    dom = html.fromstring(resp.text)
+    if startpage_categ == 'web':
+        return _response_cat_web(dom)
+
+    logger.error("Startpages's category '%' is not yet implemented.", startpage_categ)
+    return []
+
+
+def _response_cat_web(dom):
+    results = []
 
     # parse results
     for result in eval_xpath(dom, results_xpath):
@@ -173,7 +364,7 @@ def response(resp):
             content = content[date_pos:]
 
             try:
-                published_date = parser.parse(date_string, dayfirst=True)
+                published_date = dateutil.parser.parse(date_string, dayfirst=True)
             except ValueError:
                 pass
 
@@ -199,62 +390,103 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
-    # startpage's language selector is a mess each option has a displayed name
-    # and a value, either of which may represent the language name in the native
-    # script, the language name in English, an English transliteration of the
-    # native name, the English name of the writing script used by the language,
-    # or occasionally something else entirely.
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch :ref:`languages <startpage languages>` and :ref:`regions <startpage
+    regions>` from Startpage."""
+    # pylint: disable=too-many-branches
 
-    # this cases are so special they need to be hardcoded, a couple of them are misspellings
-    language_names = {
-        'english_uk': 'en-GB',
-        'fantizhengwen': ['zh-TW', 'zh-HK'],
-        'hangul': 'ko',
-        'malayam': 'ml',
-        'norsk': 'nb',
-        'sinhalese': 'si',
-        'sudanese': 'su',
+    headers = {
+        'User-Agent': gen_useragent(),
+        'Accept-Language': "en-US,en;q=0.5",  # bing needs to set the English language
     }
+    resp = network.get('https://www.startpage.com/do/settings', headers=headers)
 
-    # get the English name of every language known by babel
-    language_names.update(
-        {
-            # fmt: off
-            name.lower(): lang_code
-            # pylint: disable=protected-access
-            for lang_code, name in Locale('en')._data['languages'].items()
-            # fmt: on
-        }
-    )
+    if not resp.ok:
+        print("ERROR: response from Startpage is not OK.")
+
+    dom = lxml.html.fromstring(resp.text)
+
+    # regions
+
+    sp_region_names = []
+    for option in dom.xpath('//form[@name="settings"]//select[@name="search_results_region"]/option'):
+        sp_region_names.append(option.get('value'))
+
+    for eng_tag in sp_region_names:
+        if eng_tag == 'all':
+            continue
+        babel_region_tag = {'no_NO': 'nb_NO'}.get(eng_tag, eng_tag)  # norway
+
+        if '-' in babel_region_tag:
+            l, r = babel_region_tag.split('-')
+            r = r.split('_')[-1]
+            sxng_tag = region_tag(babel.Locale.parse(l + '_' + r, sep='_'))
+
+        else:
+            try:
+                sxng_tag = region_tag(babel.Locale.parse(babel_region_tag, sep='_'))
+
+            except babel.UnknownLocaleError:
+                print("ERROR: can't determine babel locale of startpage's locale %s" % eng_tag)
+                continue
+
+        conflict = engine_traits.regions.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+        engine_traits.regions[sxng_tag] = eng_tag
+
+    # languages
+
+    catalog_engine2code = {name.lower(): lang_code for lang_code, name in babel.Locale('en').languages.items()}
 
     # get the native name of every language known by babel
-    for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, locale_identifiers()):
-        native_name = Locale(lang_code).get_language_name().lower()
+
+    for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, babel.localedata.locale_identifiers()):
+        native_name = babel.Locale(lang_code).get_language_name().lower()
         # add native name exactly as it is
-        language_names[native_name] = lang_code
+        catalog_engine2code[native_name] = lang_code
 
         # add "normalized" language name (i.e. français becomes francais and español becomes espanol)
         unaccented_name = ''.join(filter(lambda c: not combining(c), normalize('NFKD', native_name)))
         if len(unaccented_name) == len(unaccented_name.encode()):
             # add only if result is ascii (otherwise "normalization" didn't work)
-            language_names[unaccented_name] = lang_code
+            catalog_engine2code[unaccented_name] = lang_code
+
+    # values that can't be determined by babel's languages names
+
+    catalog_engine2code.update(
+        {
+            # traditional chinese used in ..
+            'fantizhengwen': 'zh_Hant',
+            # Korean alphabet
+            'hangul': 'ko',
+            # Malayalam is one of 22 scheduled languages of India.
+            'malayam': 'ml',
+            'norsk': 'nb',
+            'sinhalese': 'si',
+        }
+    )
+
+    skip_eng_tags = {
+        'english_uk',  # SearXNG lang 'en' already maps to 'english'
+    }
 
-    dom = html.fromstring(resp.text)
-    sp_lang_names = []
     for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'):
-        sp_lang_names.append((option.get('value'), extract_text(option).lower()))
 
-    supported_languages = {}
-    for sp_option_value, sp_option_text in sp_lang_names:
-        lang_code = language_names.get(sp_option_value) or language_names.get(sp_option_text)
-        if isinstance(lang_code, str):
-            supported_languages[lang_code] = {'alias': sp_option_value}
-        elif isinstance(lang_code, list):
-            for _lc in lang_code:
-                supported_languages[_lc] = {'alias': sp_option_value}
-        else:
-            print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text))
+        eng_tag = option.get('value')
+        if eng_tag in skip_eng_tags:
+            continue
+        name = extract_text(option).lower()
 
-    return supported_languages
+        sxng_tag = catalog_engine2code.get(eng_tag)
+        if sxng_tag is None:
+            sxng_tag = catalog_engine2code[name]
+
+        conflict = engine_traits.languages.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+        engine_traits.languages[sxng_tag] = eng_tag

searx/engines/wikidata.py

@@ -1,9 +1,12 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""Wikidata
+"""This module implements the Wikidata engine.  Some implementations are shared
+from :ref:`wikipedia engine`.
+
 """
 # pylint: disable=missing-class-docstring
 
+from typing import TYPE_CHECKING
 from hashlib import md5
 from urllib.parse import urlencode, unquote
 from json import loads
@@ -13,12 +16,17 @@ from babel.dates import format_datetime, format_date, format_time, get_datetime_
 
 from searx.data import WIKIDATA_UNITS
 from searx.network import post, get
-from searx.utils import match_language, searx_useragent, get_string_replaces_function
+from searx.utils import searx_useragent, get_string_replaces_function
 from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
-from searx.engines.wikipedia import (  # pylint: disable=unused-import
-    _fetch_supported_languages,
-    supported_languages_url,
-)
+from searx.engines.wikipedia import fetch_traits as _fetch_traits
+from searx.enginelib.traits import EngineTraits
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
+traits: EngineTraits
 
 # about
 about = {
@@ -154,33 +162,35 @@ def send_wikidata_query(query, method='GET'):
 
 
 def request(query, params):
-    language = params['language'].split('-')[0]
-    if language == 'all':
-        language = 'en'
-    else:
-        language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
+
+    # wikidata does not support zh-classical (zh_Hans) / zh-TW, zh-HK and zh-CN
+    # mapped to zh
+    sxng_lang = params['searxng_locale'].split('-')[0]
+    language = traits.get_language(sxng_lang, 'en')
 
     query, attributes = get_query(query, language)
+    logger.debug("request --> language %s // len(attributes): %s", language, len(attributes))
 
     params['method'] = 'POST'
     params['url'] = SPARQL_ENDPOINT_URL
     params['data'] = {'query': query}
     params['headers'] = get_headers()
-
     params['language'] = language
     params['attributes'] = attributes
+
     return params
 
 
 def response(resp):
+
     results = []
     jsonresponse = loads(resp.content.decode())
 
-    language = resp.search_params['language'].lower()
+    language = resp.search_params['language']
     attributes = resp.search_params['attributes']
+    logger.debug("request --> language %s // len(attributes): %s", language, len(attributes))
 
     seen_entities = set()
-
     for result in jsonresponse.get('results', {}).get('bindings', []):
         attribute_result = {key: value['value'] for key, value in result.items()}
         entity_url = attribute_result['item']
@@ -260,7 +270,7 @@ def get_results(attribute_result, attributes, language):
                     infobox_urls.append({'title': attribute.get_label(language), 'url': url, **attribute.kwargs})
                     # "normal" results (not infobox) include official website and Wikipedia links.
                     if attribute.kwargs.get('official') or attribute_type == WDArticle:
-                        results.append({'title': infobox_title, 'url': url})
+                        results.append({'title': infobox_title, 'url': url, "content": infobox_content})
                     # update the infobox_id with the wikipedia URL
                     # first the local wikipedia URL, and as fallback the english wikipedia URL
                     if attribute_type == WDArticle and (
@@ -756,3 +766,15 @@ def init(engine_settings=None):  # pylint: disable=unused-argument
         lang = result['name']['xml:lang']
         entity_id = result['item']['value'].replace('http://www.wikidata.org/entity/', '')
         WIKIDATA_PROPERTIES[(entity_id, lang)] = name.capitalize()
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Use languages evaluated from :py:obj:`wikipedia.fetch_traits
+    <searx.engines.wikipedia.fetch_traits>` except zh-classical (zh_Hans) what
+    is not supported by wikidata."""
+
+    _fetch_traits(engine_traits)
+    # wikidata does not support zh-classical (zh_Hans)
+    engine_traits.languages.pop('zh_Hans')
+    # wikidata does not have net-locations for the languages
+    engine_traits.custom['wiki_netloc'] = {}

searx/engines/wikipedia.py

@@ -1,13 +1,26 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- Wikipedia (Web)
+# lint: pylint
+"""This module implements the Wikipedia engine.  Some of this implementations
+are shared by other engines:
+
+- :ref:`wikidata engine`
+
+The list of supported languages is fetched from the article linked by
+:py:obj:`wikipedia_article_depth`.  Unlike traditional search engines, wikipedia
+does not support one Wikipedia for all the languages, but there is one Wikipedia
+for every language (:py:obj:`fetch_traits`).
 """
 
-from urllib.parse import quote
-from json import loads
-from lxml.html import fromstring
-from searx.utils import match_language, searx_useragent
-from searx.network import raise_for_httperror
+import urllib.parse
+import babel
+
+from lxml import html
+
+from searx import network
+from searx.locales import language_tag
+from searx.enginelib.traits import EngineTraits
+
+traits: EngineTraits
 
 # about
 about = {
@@ -19,32 +32,40 @@ about = {
     "results": 'JSON',
 }
 
-
 send_accept_language_header = True
 
-# search-url
-search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'
-supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
-language_variants = {"zh": ("zh-cn", "zh-hk", "zh-mo", "zh-my", "zh-sg", "zh-tw")}
+wikipedia_article_depth = 'https://meta.wikimedia.org/wiki/Wikipedia_article_depth'
+"""The *editing depth* of Wikipedia is one of several possible rough indicators
+of the encyclopedia's collaborative quality, showing how frequently its articles
+are updated.  The measurement of depth was introduced after some limitations of
+the classic measurement of article count were realized.
+"""
+
+# example: https://zh-classical.wikipedia.org/api/rest_v1/page/summary/日
+rest_v1_summary_url = 'https://{wiki_netloc}/api/rest_v1/page/summary/{title}'
+"""`wikipedia rest_v1 summary API`_: The summary response includes an extract of
+the first paragraph of the page in plain text and HTML as well as the type of
+page. This is useful for page previews (fka. Hovercards, aka. Popups) on the web
+and link previews in the apps.
+
+.. _wikipedia rest_v1 summary API: https://en.wikipedia.org/api/rest_v1/#/Page%20content/get_page_summary__title_
+
+"""
 
 
-# set language in base_url
-def url_lang(lang):
-    lang_pre = lang.split('-')[0]
-    if lang_pre == 'all' or lang_pre not in supported_languages and lang_pre not in language_aliases:
-        return 'en'
-    return match_language(lang, supported_languages, language_aliases).split('-')[0]
-
-
-# do search-request
 def request(query, params):
+    """Assemble a request (`wikipedia rest_v1 summary API`_)."""
     if query.islower():
         query = query.title()
 
-    language = url_lang(params['language'])
-    params['url'] = search_url.format(title=quote(query), language=language)
+    engine_language = traits.get_language(params['searxng_locale'], 'en')
+    wiki_netloc = traits.custom['wiki_netloc'].get(engine_language, 'https://en.wikipedia.org/wiki/')
+    title = urllib.parse.quote(query)
+
+    # '!wikipedia 日 :zh-TW' --> https://zh-classical.wikipedia.org/
+    # '!wikipedia 日 :zh' --> https://zh.wikipedia.org/
+    params['url'] = rest_v1_summary_url.format(wiki_netloc=wiki_netloc, title=title)
 
-    params['headers']['User-Agent'] = searx_useragent()
     params['raise_for_httperror'] = False
     params['soft_max_redirects'] = 2
 
@@ -53,13 +74,14 @@ def request(query, params):
 
 # get response from search-request
 def response(resp):
+
+    results = []
     if resp.status_code == 404:
         return []
-
     if resp.status_code == 400:
         try:
-            api_result = loads(resp.text)
-        except:
+            api_result = resp.json()
+        except Exception:  # pylint: disable=broad-except
             pass
         else:
             if (
@@ -68,20 +90,14 @@ def response(resp):
             ):
                 return []
 
-    raise_for_httperror(resp)
-
-    results = []
-    api_result = loads(resp.text)
-
-    # skip disambiguation pages
-    if api_result.get('type') != 'standard':
-        return []
+    network.raise_for_httperror(resp)
 
+    api_result = resp.json()
     title = api_result['title']
     wikipedia_link = api_result['content_urls']['desktop']['page']
+    results.append({'url': wikipedia_link, 'title': title, 'content': api_result.get('description', '')})
 
-    results.append({'url': wikipedia_link, 'title': title})
-
+    if api_result.get('type') == 'standard':
         results.append(
             {
                 'infobox': title,
@@ -95,22 +111,114 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
-    supported_languages = {}
-    dom = fromstring(resp.text)
-    tables = dom.xpath('//table[contains(@class,"sortable")]')
-    for table in tables:
-        # exclude header row
-        trs = table.xpath('.//tr')[1:]
-        for tr in trs:
-            td = tr.xpath('./td')
-            code = td[3].xpath('./a')[0].text
-            name = td[1].xpath('./a')[0].text
-            english_name = td[1].xpath('./a')[0].text
-            articles = int(td[4].xpath('./a')[0].text.replace(',', ''))
-            # exclude languages with too few articles
-            if articles >= 100:
-                supported_languages[code] = {"name": name, "english_name": english_name}
+# Nonstandard language codes
+#
+# These Wikipedias use language codes that do not conform to the ISO 639
+# standard (which is how wiki subdomains are chosen nowadays).
 
-    return supported_languages
+lang_map = {
+    'be-tarask': 'bel',
+    'ak': 'aka',
+    'als': 'gsw',
+    'bat-smg': 'sgs',
+    'cbk-zam': 'cbk',
+    'fiu-vro': 'vro',
+    'map-bms': 'map',
+    'nrm': 'nrf',
+    'roa-rup': 'rup',
+    'nds-nl': 'nds',
+    #'simple: – invented code used for the Simple English Wikipedia (not the official IETF code en-simple)
+    'zh-min-nan': 'nan',
+    'zh-yue': 'yue',
+    'an': 'arg',
+    'zh-classical': 'zh-Hant',  # babel maps classical to zh-Hans (for whatever reason)
+}
+
+unknown_langs = [
+    'an',  # Aragonese
+    'ba',  # Bashkir
+    'bar',  # Bavarian
+    'bcl',  # Central Bicolano
+    'be-tarask',  # Belarusian variant / Belarusian is already covered by 'be'
+    'bpy',  # Bishnupriya Manipuri is unknown by babel
+    'hif',  # Fiji Hindi
+    'ilo',  # Ilokano
+    'li',  # Limburgish
+    'sco',  # Scots (sco) is not known by babel, Scottish Gaelic (gd) is known by babel
+    'sh',  # Serbo-Croatian
+    'simple',  # simple english is not know as a natural language different to english (babel)
+    'vo',  # Volapük
+    'wa',  # Walloon
+]
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages from Wikipedia.
+
+    The location of the Wikipedia address of a language is mapped in a
+    :py:obj:`custom field <searx.enginelib.traits.EngineTraits.custom>`
+    (``wiki_netloc``).  Here is a reduced example:
+
+    .. code:: python
+
+       traits.custom['wiki_netloc'] = {
+           "en": "en.wikipedia.org",
+           ..
+           "gsw": "als.wikipedia.org",
+           ..
+           "zh": "zh.wikipedia.org",
+           "zh-classical": "zh-classical.wikipedia.org"
+       }
+
+    """
+
+    engine_traits.custom['wiki_netloc'] = {}
+
+    # insert alias to map from a region like zh-CN to a language zh_Hans
+    engine_traits.languages['zh_Hans'] = 'zh'
+
+    resp = network.get(wikipedia_article_depth)
+    if not resp.ok:
+        print("ERROR: response from Wikipedia is not OK.")
+
+    dom = html.fromstring(resp.text)
+    for row in dom.xpath('//table[contains(@class,"sortable")]//tbody/tr'):
+
+        cols = row.xpath('./td')
+        if not cols:
+            continue
+        cols = [c.text_content().strip() for c in cols]
+
+        depth = float(cols[3].replace('-', '0').replace(',', ''))
+        articles = int(cols[4].replace(',', '').replace(',', ''))
+
+        if articles < 10000:
+            # exclude languages with too few articles
+            continue
+
+        if int(depth) < 20:
+            # Rough indicator of a Wikipedia’s quality, showing how frequently
+            # its articles are updated.
+            continue
+
+        eng_tag = cols[2]
+        wiki_url = row.xpath('./td[3]/a/@href')[0]
+        wiki_url = urllib.parse.urlparse(wiki_url)
+
+        if eng_tag in unknown_langs:
+            continue
+
+        try:
+            sxng_tag = language_tag(babel.Locale.parse(lang_map.get(eng_tag, eng_tag), sep='-'))
+        except babel.UnknownLocaleError:
+            print("ERROR: %s [%s] is unknown by babel" % (cols[0], eng_tag))
+            continue
+
+        conflict = engine_traits.languages.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+
+        engine_traits.languages[sxng_tag] = eng_tag
+        engine_traits.custom['wiki_netloc'][eng_tag] = wiki_url.netloc

searx/engines/yahoo.py

@@ -17,8 +17,10 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
-    match_language,
 )
+from searx.enginelib.traits import EngineTraits
+
+traits: EngineTraits
 
 # about
 about = {
@@ -34,8 +36,7 @@ about = {
 categories = ['general', 'web']
 paging = True
 time_range_support = True
-supported_languages_url = 'https://search.yahoo.com/preferences/languages'
-"""Supported languages are read from Yahoo preference page."""
+# send_accept_language_header = True
 
 time_range_dict = {
     'day': ('1d', 'd'),
@@ -43,15 +44,10 @@ time_range_dict = {
     'month': ('1m', 'm'),
 }
 
-language_aliases = {
-    'zh-HK': 'zh_chs',
-    'zh-CN': 'zh_chs',  # dead since 2015 / routed to hk.search.yahoo.com
-    'zh-TW': 'zh_cht',
-}
-
 lang2domain = {
     'zh_chs': 'hk.search.yahoo.com',
     'zh_cht': 'tw.search.yahoo.com',
+    'any': 'search.yahoo.com',
     'en': 'search.yahoo.com',
     'bg': 'search.yahoo.com',
     'cs': 'search.yahoo.com',
@@ -67,21 +63,23 @@ lang2domain = {
 }
 """Map language to domain"""
 
-
-def _get_language(params):
-
-    lang = language_aliases.get(params['language'])
-    if lang is None:
-        lang = match_language(params['language'], supported_languages, language_aliases)
-    lang = lang.split('-')[0]
-    logger.debug("params['language']: %s --> %s", params['language'], lang)
-    return lang
+locale_aliases = {
+    'zh': 'zh_Hans',
+    'zh-HK': 'zh_Hans',
+    'zh-CN': 'zh_Hans',  # dead since 2015 / routed to hk.search.yahoo.com
+    'zh-TW': 'zh_Hant',
+}
 
 
 def request(query, params):
     """build request"""
+
+    lang = locale_aliases.get(params['language'], None)
+    if not lang:
+        lang = params['language'].split('-')[0]
+    lang = traits.get_language(lang, traits.all_locale)
+
     offset = (params['pageno'] - 1) * 7 + 1
-    lang = _get_language(params)
     age, btf = time_range_dict.get(params['time_range'], ('', ''))
 
     args = urlencode(
@@ -154,13 +152,37 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
-    supported_languages = []
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages from yahoo"""
+
+    # pylint: disable=import-outside-toplevel
+    import babel
+    from searx import network
+    from searx.locales import language_tag
+
+    engine_traits.all_locale = 'any'
+
+    resp = network.get('https://search.yahoo.com/preferences/languages')
+    if not resp.ok:
+        print("ERROR: response from peertube is not OK.")
+
     dom = html.fromstring(resp.text)
     offset = len('lang_')
 
-    for val in eval_xpath_list(dom, '//div[contains(@class, "lang-item")]/input/@value'):
-        supported_languages.append(val[offset:])
+    eng2sxng = {'zh_chs': 'zh_Hans', 'zh_cht': 'zh_Hant'}
 
-    return supported_languages
+    for val in eval_xpath_list(dom, '//div[contains(@class, "lang-item")]/input/@value'):
+        eng_tag = val[offset:]
+
+        try:
+            sxng_tag = language_tag(babel.Locale.parse(eng2sxng.get(eng_tag, eng_tag)))
+        except babel.UnknownLocaleError:
+            print('ERROR: unknown language --> %s' % eng_tag)
+            continue
+
+        conflict = engine_traits.languages.get(sxng_tag)
+        if conflict:
+            if conflict != eng_tag:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
+            continue
+        engine_traits.languages[sxng_tag] = eng_tag

searx/locales.py

@@ -4,11 +4,11 @@
 """Initialize :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`.
 """
 
-from typing import Set
+from typing import Set, Optional, List
 import os
 import pathlib
 
-from babel import Locale
+import babel
 from babel.support import Translations
 import babel.languages
 import babel.core
@@ -134,7 +134,7 @@ def locales_initialize(directory=None):
     flask_babel.get_translations = get_translations
 
     for tag, descr in ADDITIONAL_TRANSLATIONS.items():
-        locale = Locale.parse(LOCALE_BEST_MATCH[tag], sep='-')
+        locale = babel.Locale.parse(LOCALE_BEST_MATCH[tag], sep='-')
         LOCALE_NAMES[tag] = descr
         if locale.text_direction == 'rtl':
             RTL_LOCALES.add(tag)
@@ -142,7 +142,7 @@ def locales_initialize(directory=None):
     for tag in LOCALE_BEST_MATCH:
         descr = LOCALE_NAMES.get(tag)
         if not descr:
-            locale = Locale.parse(tag, sep='-')
+            locale = babel.Locale.parse(tag, sep='-')
             LOCALE_NAMES[tag] = get_locale_descr(locale, tag.replace('-', '_'))
             if locale.text_direction == 'rtl':
                 RTL_LOCALES.add(tag)
@@ -154,12 +154,77 @@ def locales_initialize(directory=None):
         tag = dirname.replace('_', '-')
         descr = LOCALE_NAMES.get(tag)
         if not descr:
-            locale = Locale.parse(dirname)
+            locale = babel.Locale.parse(dirname)
             LOCALE_NAMES[tag] = get_locale_descr(locale, dirname)
             if locale.text_direction == 'rtl':
                 RTL_LOCALES.add(tag)
 
 
+def region_tag(locale: babel.Locale) -> str:
+    """Returns SearXNG's region tag from the locale (e.g. zh-TW , en-US)."""
+    if not locale.territory:
+        raise ValueError('%s missed a territory')
+    return locale.language + '-' + locale.territory
+
+
+def language_tag(locale: babel.Locale) -> str:
+    """Returns SearXNG's language tag from the locale and if exits, the tag
+    includes the script name (e.g. en, zh_Hant).
+    """
+    sxng_lang = locale.language
+    if locale.script:
+        sxng_lang += '_' + locale.script
+    return sxng_lang
+
+
+def get_locale(locale_tag: str) -> Optional[babel.Locale]:
+    """Returns a :py:obj:`babel.Locale` object parsed from argument
+    ``locale_tag``"""
+    try:
+        locale = babel.Locale.parse(locale_tag, sep='-')
+        return locale
+
+    except babel.core.UnknownLocaleError:
+        return None
+
+
+def get_offical_locales(
+    territory: str, languages=None, regional: bool = False, de_facto: bool = True
+) -> Set[babel.Locale]:
+    """Returns a list of :py:obj:`babel.Locale` with languages from
+    :py:obj:`babel.languages.get_official_languages`.
+
+    :param territory: The territory (country or region) code.
+
+    :param languages: A list of language codes the languages from
+      :py:obj:`babel.languages.get_official_languages` should be in
+      (intersection).  If this argument is ``None``, all official languages in
+      this territory are used.
+
+    :param regional: If the regional flag is set, then languages which are
+      regionally official are also returned.
+
+    :param de_facto: If the de_facto flag is set to `False`, then languages
+      which are “de facto” official are not returned.
+
+    """
+    ret_val = set()
+    o_languages = babel.languages.get_official_languages(territory, regional=regional, de_facto=de_facto)
+
+    if languages:
+        languages = [l.lower() for l in languages]
+        o_languages = set(l for l in o_languages if l.lower() in languages)
+
+    for lang in o_languages:
+        try:
+            locale = babel.Locale.parse(lang + '_' + territory)
+            ret_val.add(locale)
+        except babel.UnknownLocaleError:
+            continue
+
+    return ret_val
+
+
 def get_engine_locale(searxng_locale, engine_locales, default=None):
     """Return engine's language (aka locale) string that best fits to argument
     ``searxng_locale``.
@@ -177,6 +242,10 @@ def get_engine_locale(searxng_locale, engine_locales, default=None):
           ...
           'pl-PL'          : 'pl_PL',
           'pt-PT'          : 'pt_PT'
+          ..
+          'zh'             : 'zh'
+          'zh_Hans'        : 'zh'
+          'zh_Hant'        : 'zh-classical'
       }
 
     .. hint::
@@ -210,13 +279,13 @@ def get_engine_locale(searxng_locale, engine_locales, default=None):
       engine.
 
     """
-    # pylint: disable=too-many-branches
+    # pylint: disable=too-many-branches, too-many-return-statements
 
     engine_locale = engine_locales.get(searxng_locale)
 
     if engine_locale is not None:
-        # There was a 1:1 mapping (e.g. "fr-BE --> fr_BE" or "fr --> fr_FR"), no
-        # need to narrow language nor territory.
+        # There was a 1:1 mapping (e.g. a region "fr-BE --> fr_BE" or a language
+        # "zh --> zh"), no need to narrow language-script nor territory.
         return engine_locale
 
     try:
@@ -227,6 +296,12 @@ def get_engine_locale(searxng_locale, engine_locales, default=None):
         except babel.core.UnknownLocaleError:
             return default
 
+    searxng_lang = language_tag(locale)
+    engine_locale = engine_locales.get(searxng_lang)
+    if engine_locale is not None:
+        # There was a 1:1 mapping (e.g. "zh-HK --> zh_Hant" or "zh-CN --> zh_Hans")
+        return engine_locale
+
     # SearXNG's selected locale is not supported by the engine ..
 
     if locale.territory:
@@ -247,10 +322,6 @@ def get_engine_locale(searxng_locale, engine_locales, default=None):
 
     if locale.language:
 
-        searxng_lang = locale.language
-        if locale.script:
-            searxng_lang += '_' + locale.script
-
         terr_lang_dict = {}
         for territory, langs in babel.core.get_global("territory_languages").items():
             if not langs.get(searxng_lang, {}).get('official_status'):
@@ -303,3 +374,98 @@ def get_engine_locale(searxng_locale, engine_locales, default=None):
         engine_locale = default
 
     return default
+
+
+def match_locale(searxng_locale: str, locale_tag_list: List[str], fallback: Optional[str] = None) -> Optional[str]:
+    """Return tag from ``locale_tag_list`` that best fits to ``searxng_locale``.
+
+    :param str searxng_locale: SearXNG's internal representation of locale (de,
+        de-DE, fr-BE, zh, zh-CN, zh-TW ..).
+
+    :param list locale_tag_list: The list of locale tags to select from
+
+    :param str fallback: fallback locale tag (if unset --> ``None``)
+
+    The rules to find a match are implemented in :py:obj:`get_engine_locale`,
+    the ``engine_locales`` is build up by :py:obj:`build_engine_locales`.
+
+    .. hint::
+
+       The *SearXNG locale* string and the members of ``locale_tag_list`` has to
+       be known by babel!  The :py:obj:`ADDITIONAL_TRANSLATIONS` are used in the
+       UI and are not known by babel --> will be ignored.
+    """
+
+    # searxng_locale = 'es'
+    # locale_tag_list = ['es-AR', 'es-ES', 'es-MX']
+
+    if not searxng_locale:
+        return fallback
+
+    locale = get_locale(searxng_locale)
+    if locale is None:
+        return fallback
+
+    # normalize to a SearXNG locale that can be passed to get_engine_locale
+
+    searxng_locale = language_tag(locale)
+    if locale.territory:
+        searxng_locale = region_tag(locale)
+
+    # clean up locale_tag_list
+
+    tag_list = []
+    for tag in locale_tag_list:
+        if tag in ('all', 'auto') or tag in ADDITIONAL_TRANSLATIONS:
+            continue
+        tag_list.append(tag)
+
+    # emulate fetch_traits
+    engine_locales = build_engine_locales(tag_list)
+    return get_engine_locale(searxng_locale, engine_locales, default=fallback)
+
+
+def build_engine_locales(tag_list: List[str]):
+    """From a list of locale tags a dictionary is build that can be passed by
+    argument ``engine_locales`` to :py:obj:`get_engine_locale`.  This function
+    is mainly used by :py:obj:`match_locale` and is similar to what the
+    ``fetch_traits(..)`` function of engines do.
+
+    If there are territory codes in the ``tag_list`` that have a *script code*
+    additional keys are added to the returned dictionary.
+
+    .. code:: python
+
+       >>> import locales
+       >>> engine_locales = locales.build_engine_locales(['en', 'en-US', 'zh', 'zh-CN', 'zh-TW'])
+       >>> engine_locales
+       {
+           'en': 'en', 'en-US': 'en-US',
+           'zh': 'zh', 'zh-CN': 'zh-CN', 'zh_Hans': 'zh-CN',
+           'zh-TW': 'zh-TW', 'zh_Hant': 'zh-TW'
+       }
+       >>> get_engine_locale('zh-Hans', engine_locales)
+       'zh-CN'
+
+    This function is a good example to understand the language/region model
+    of SearXNG:
+
+      SearXNG only distinguishes between **search languages** and **search
+      regions**, by adding the *script-tags*, languages with *script-tags* can
+      be assigned to the **regions** that SearXNG supports.
+
+    """
+    engine_locales = {}
+
+    for tag in tag_list:
+        locale = get_locale(tag)
+        if locale is None:
+            logger.warn("build_engine_locales: skip locale tag %s / unknown by babel", tag)
+            continue
+        if locale.territory:
+            engine_locales[region_tag(locale)] = tag
+            if locale.script:
+                engine_locales[language_tag(locale)] = tag
+        else:
+            engine_locales[language_tag(locale)] = tag
+    return engine_locales

searx/plugins/limiter.py

@@ -17,21 +17,26 @@ import re
 from flask import request
 
 from searx import redisdb
+from searx.plugins import logger
 from searx.redislib import incr_sliding_window
 
 name = "Request limiter"
 description = "Limit the number of request"
 default_on = False
 preference_section = 'service'
+logger = logger.getChild('limiter')
 
-
-re_bot = re.compile(
+block_user_agent = re.compile(
     r'('
-    + r'[Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp'
+    + r'unknown'
+    + r'|[Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp'
     + r'|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser|Googlebot|GoogleImageProxy'
     + r'|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot'
     + r'|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot'
     + r'|ZmEu|BLEXBot|bitlybot'
+    # when you block requests from Farside instances, your instance will
+    # disappear from https://farside.link/
+    # + r'|Farside'
     + r')'
 )
 
@@ -39,47 +44,59 @@ re_bot = re.compile(
 def is_accepted_request() -> bool:
     # pylint: disable=too-many-return-statements
     redis_client = redisdb.client()
-    user_agent = request.headers.get('User-Agent', '')
+    user_agent = request.headers.get('User-Agent', 'unknown')
     x_forwarded_for = request.headers.get('X-Forwarded-For', '')
 
-    if request.path == '/image_proxy':
-        if re_bot.match(user_agent):
-            return False
+    if request.path == '/healthz':
         return True
 
+    if block_user_agent.match(user_agent):
+        logger.debug("BLOCK %s: %s --> detected User-Agent: %s" % (x_forwarded_for, request.path, user_agent))
+        return False
+
     if request.path == '/search':
+
         c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20)
         c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600)
         if c_burst > 15 or c_10min > 150:
-            logger.debug("to many request")  # pylint: disable=undefined-variable
-            return False
-
-        if re_bot.match(user_agent):
-            logger.debug("detected bot")  # pylint: disable=undefined-variable
+            logger.debug("BLOCK %s: to many request", x_forwarded_for)
             return False
 
         if len(request.headers.get('Accept-Language', '').strip()) == '':
-            logger.debug("missing Accept-Language")  # pylint: disable=undefined-variable
+            logger.debug("BLOCK %s: missing Accept-Language", x_forwarded_for)
             return False
 
         if request.headers.get('Connection') == 'close':
-            logger.debug("got Connection=close")  # pylint: disable=undefined-variable
+            logger.debug("BLOCK %s: got Connection=close", x_forwarded_for)
             return False
 
         accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')]
         if 'gzip' not in accept_encoding_list and 'deflate' not in accept_encoding_list:
-            logger.debug("suspicious Accept-Encoding")  # pylint: disable=undefined-variable
+            logger.debug("BLOCK %s: suspicious Accept-Encoding", x_forwarded_for)
             return False
 
         if 'text/html' not in request.accept_mimetypes:
-            logger.debug("Accept-Encoding misses text/html")  # pylint: disable=undefined-variable
+            logger.debug("BLOCK %s: Accept-Encoding misses text/html", x_forwarded_for)
             return False
 
         if request.args.get('format', 'html') != 'html':
             c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600)
             if c > 4:
-                logger.debug("API limit exceeded")  # pylint: disable=undefined-variable
+                logger.debug("BLOCK %s: API limit exceeded", x_forwarded_for)
                 return False
+
+    logger.debug(
+        "OK %s: '%s'" % (x_forwarded_for, request.path)
+        + " || form: %s" % request.form
+        + " || Accept: %s" % request.headers.get('Accept', '')
+        + " || Accept-Language: %s" % request.headers.get('Accept-Language', '')
+        + " || Accept-Encoding: %s" % request.headers.get('Accept-Encoding', '')
+        + " || Content-Type: %s" % request.headers.get('Content-Type', '')
+        + " || Content-Length: %s" % request.headers.get('Content-Length', '')
+        + " || Connection: %s" % request.headers.get('Connection', '')
+        + " || User-Agent: %s" % user_agent
+    )
+
     return True
 
 

searx/preferences.py

@@ -13,7 +13,7 @@ from typing import Iterable, Dict, List
 import flask
 
 from searx import settings, autocomplete
-from searx.engines import Engine
+from searx.enginelib import Engine
 from searx.plugins import Plugin
 from searx.locales import LOCALE_NAMES
 from searx.webutils import VALID_LANGUAGE_CODE

searx/query.py

@@ -4,7 +4,7 @@ from abc import abstractmethod, ABC
 import re
 
 from searx import settings
-from searx.languages import language_codes
+from searx.sxng_locales import sxng_locales
 from searx.engines import categories, engines, engine_shortcuts
 from searx.external_bang import get_bang_definition_and_autocomplete
 from searx.search import EngineRef
@@ -84,7 +84,7 @@ class LanguageParser(QueryPartParser):
         found = False
         # check if any language-code is equal with
         # declared language-codes
-        for lc in language_codes:
+        for lc in sxng_locales:
             lang_id, lang_name, country, english_name, _flag = map(str.lower, lc)
 
             # if correct language-code is found
@@ -125,7 +125,7 @@ class LanguageParser(QueryPartParser):
                     self.raw_text_query.autocomplete_list.append(lang)
             return
 
-        for lc in language_codes:
+        for lc in sxng_locales:
             if lc[0] not in settings['search']['languages']:
                 continue
             lang_id, lang_name, country, english_name, _flag = map(str.lower, lc)

searx/search/processors/__init__.py

@@ -30,7 +30,10 @@ from .abstract import EngineProcessor
 
 logger = logger.getChild('search.processors')
 PROCESSORS: Dict[str, EngineProcessor] = {}
-"""Cache request processores, stored by *engine-name* (:py:func:`initialize`)"""
+"""Cache request processores, stored by *engine-name* (:py:func:`initialize`)
+
+:meta hide-value:
+"""
 
 
 def get_processor_class(engine_type):

searx/search/processors/abstract.py

@@ -138,7 +138,8 @@ class EngineProcessor(ABC):
         return False
 
     def get_params(self, search_query, engine_category):
-        """Returns a set of *request params* or ``None`` if request is not supported.
+        """Returns a set of (see :ref:`request params <engine request arguments>`) or
+        ``None`` if request is not supported.
 
         Not supported conditions (``None`` is returned):
 
@@ -159,11 +160,20 @@ class EngineProcessor(ABC):
         params['safesearch'] = search_query.safesearch
         params['time_range'] = search_query.time_range
         params['engine_data'] = search_query.engine_data.get(self.engine_name, {})
+        params['searxng_locale'] = search_query.lang
+
+        # deprecated / vintage --> use params['searxng_locale']
+        #
+        # Conditions related to engine's traits are implemented in engine.traits
+        # module. Don't do 'locale' decissions here in the abstract layer of the
+        # search processor, just pass the value from user's choice unchanged to
+        # the engine request.
 
         if hasattr(self.engine, 'language') and self.engine.language:
             params['language'] = self.engine.language
         else:
             params['language'] = search_query.lang
+
         return params
 
     @abstractmethod

searx/search/processors/online.py

@@ -51,6 +51,9 @@ class OnlineProcessor(EngineProcessor):
         super().initialize()
 
     def get_params(self, search_query, engine_category):
+        """Returns a set of :ref:`request params <engine request online>` or ``None``
+        if request is not supported.
+        """
         params = super().get_params(search_query, engine_category)
         if params is None:
             return None
@@ -184,11 +187,6 @@ class OnlineProcessor(EngineProcessor):
             self.handle_exception(result_container, e, suspend=True)
             self.logger.exception('CAPTCHA')
         except SearxEngineTooManyRequestsException as e:
-            if "google" in self.engine_name:
-                self.logger.warn(
-                    "Set to 'true' the use_mobile_ui parameter in the 'engines:'"
-                    " section of your settings.yml file if google is blocked for you."
-                )
             self.handle_exception(result_container, e, suspend=True)
             self.logger.exception('Too many requests')
         except SearxEngineAccessDeniedException as e:
@@ -223,7 +221,7 @@ class OnlineProcessor(EngineProcessor):
                 'test': ['unique_results'],
             }
 
-        if getattr(self.engine, 'supported_languages', []):
+        if getattr(self.engine, 'traits', False):
             tests['lang_fr'] = {
                 'matrix': {'query': 'paris', 'lang': 'fr'},
                 'result_container': ['not_empty', ('has_language', 'fr')],

searx/search/processors/online_currency.py

@@ -38,8 +38,8 @@ class OnlineCurrencyProcessor(OnlineProcessor):
     engine_type = 'online_currency'
 
     def get_params(self, search_query, engine_category):
-        """Returns a set of *request params* or ``None`` if search query does not match
-        to :py:obj:`parser_re`."""
+        """Returns a set of :ref:`request params <engine request online_currency>`
+        or ``None`` if search query does not match to :py:obj:`parser_re`."""
 
         params = super().get_params(search_query, engine_category)
         if params is None:

searx/search/processors/online_dictionary.py

@@ -18,8 +18,9 @@ class OnlineDictionaryProcessor(OnlineProcessor):
     engine_type = 'online_dictionary'
 
     def get_params(self, search_query, engine_category):
-        """Returns a set of *request params* or ``None`` if search query does not match
-        to :py:obj:`parser_re`."""
+        """Returns a set of :ref:`request params <engine request online_dictionary>` or
+        ``None`` if search query does not match to :py:obj:`parser_re`.
+        """
         params = super().get_params(search_query, engine_category)
         if params is None:
             return None

searx/search/processors/online_url_search.py

@@ -20,9 +20,10 @@ class OnlineUrlSearchProcessor(OnlineProcessor):
     engine_type = 'online_url_search'
 
     def get_params(self, search_query, engine_category):
-        """Returns a set of *request params* or ``None`` if search query does not match
-        to at least one of :py:obj:`re_search_urls`.
+        """Returns a set of :ref:`request params <engine request online>` or ``None`` if
+        search query does not match to :py:obj:`re_search_urls`.
         """
+
         params = super().get_params(search_query, engine_category)
         if params is None:
             return None

searx/settings.yml

@@ -674,6 +674,9 @@ engines:
     engine: gigablast
     shortcut: gb
     timeout: 4.0
+    # API key required, see https://gigablast.com/searchfeed.html
+    # gb_userid: unset
+    # gb_code: unknown
     disabled: true
     additional_tests:
       rosebud: *test_rosebud
@@ -731,22 +734,9 @@ engines:
   - name: google
     engine: google
     shortcut: go
-    # see https://docs.searxng.org/src/searx.engines.google.html#module-searx.engines.google
-    use_mobile_ui: false
     # additional_tests:
     #   android: *test_android
 
-  # - name: google italian
-  #   engine: google
-  #   shortcut: goit
-  #   use_mobile_ui: false
-  #   language: it
-
-  # - name: google mobile ui
-  #   engine: google
-  #   shortcut: gomui
-  #   use_mobile_ui: true
-
   - name: google images
     engine: google_images
     shortcut: goi
@@ -1758,9 +1748,8 @@ engines:
     engine: peertube
     shortcut: ptb
     paging: true
-    # https://instances.joinpeertube.org/instances
-    base_url: https://peertube.biz/
-    # base_url: https://tube.tardis.world/
+    # alternatives see: https://instances.joinpeertube.org/instances
+    # base_url: https://tube.4aem.com
     categories: videos
     disabled: true
     timeout: 6.0

searx/settings_defaults.py

@@ -12,13 +12,13 @@ import logging
 from base64 import b64decode
 from os.path import dirname, abspath
 
-from searx.languages import language_codes as languages
+from .sxng_locales import sxng_locales
 
 searx_dir = abspath(dirname(__file__))
 
 logger = logging.getLogger('searx')
 OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
-LANGUAGE_CODES = ['all', 'auto'] + list(l[0] for l in languages)
+SXNG_LOCALE_TAGS = ['all', 'auto'] + list(l[0] for l in sxng_locales)
 SIMPLE_STYLE = ('auto', 'light', 'dark')
 CATEGORIES_AS_TABS = {
     'general': {},
@@ -156,8 +156,8 @@ SCHEMA = {
         'safe_search': SettingsValue((0, 1, 2), 0),
         'autocomplete': SettingsValue(str, ''),
         'autocomplete_min': SettingsValue(int, 4),
-        'default_lang': SettingsValue(tuple(LANGUAGE_CODES + ['']), ''),
-        'languages': SettingSublistValue(LANGUAGE_CODES, LANGUAGE_CODES),
+        'default_lang': SettingsValue(tuple(SXNG_LOCALE_TAGS + ['']), ''),
+        'languages': SettingSublistValue(SXNG_LOCALE_TAGS, SXNG_LOCALE_TAGS),
         'ban_time_on_fail': SettingsValue(numbers.Real, 5),
         'max_ban_time_on_fail': SettingsValue(numbers.Real, 120),
         'suspended_times': {

searx/settings_loader.py

@@ -74,6 +74,10 @@ def update_settings(default_settings, user_settings):
             else:
                 default_settings[k] = v
 
+    categories_as_tabs = user_settings.get('categories_as_tabs')
+    if categories_as_tabs:
+        default_settings['categories_as_tabs'] = categories_as_tabs
+
     # parse the engines
     remove_engines = None
     keep_only_engines = None

searx/static/themes/simple/package.json

@@ -11,10 +11,10 @@
     "grunt-eslint": "^24.0.0",
     "grunt-stylelint": "^0.16.0",
     "grunt-image": "^6.4.0",
-    "ionicons": "^6.0.2",
+    "ionicons": "^7.1.0",
     "less": "^4.1.3",
     "less-plugin-clean-css": "^1.5.1",
-    "sharp": "^0.31.0",
+    "sharp": "^0.32.0",
     "stylelint": "^13.13.1",
     "stylelint-config-standard": "^22.0.0",
     "ejs": "^3.1.8",

searx/sxng_locales.py

@@ -1,73 +1,120 @@
 # -*- coding: utf-8 -*-
-# list of language codes
-# this file is generated automatically by utils/fetch_languages.py
-language_codes = (
-    ('af-ZA', 'Afrikaans', 'Suid-Afrika', 'Afrikaans', '\U0001f1ff\U0001f1e6'),
-    ('ar-EG', 'العربية', 'مصر', 'Arabic', '\U0001f1ea\U0001f1ec'),
-    ('be-BY', 'Беларуская', 'Беларусь', 'Belarusian', '\U0001f1e7\U0001f1fe'),
+'''List of SearXNG's locale codes.
+
+This file is generated automatically by::
+
+   ./manage pyenv.cmd searxng_extra/update/update_engine_traits.py
+'''
+
+sxng_locales = (
+    ('ar', 'العربية', '', 'Arabic', '\U0001f310'),
+    ('bg', 'Български', '', 'Bulgarian', '\U0001f310'),
     ('bg-BG', 'Български', 'България', 'Bulgarian', '\U0001f1e7\U0001f1ec'),
+    ('ca', 'Català', '', 'Catalan', '\U0001f310'),
     ('ca-ES', 'Català', 'Espanya', 'Catalan', '\U0001f1ea\U0001f1f8'),
+    ('cs', 'Čeština', '', 'Czech', '\U0001f310'),
     ('cs-CZ', 'Čeština', 'Česko', 'Czech', '\U0001f1e8\U0001f1ff'),
+    ('da', 'Dansk', '', 'Danish', '\U0001f310'),
     ('da-DK', 'Dansk', 'Danmark', 'Danish', '\U0001f1e9\U0001f1f0'),
     ('de', 'Deutsch', '', 'German', '\U0001f310'),
     ('de-AT', 'Deutsch', 'Österreich', 'German', '\U0001f1e6\U0001f1f9'),
     ('de-CH', 'Deutsch', 'Schweiz', 'German', '\U0001f1e8\U0001f1ed'),
     ('de-DE', 'Deutsch', 'Deutschland', 'German', '\U0001f1e9\U0001f1ea'),
+    ('el', 'Ελληνικά', '', 'Greek', '\U0001f310'),
     ('el-GR', 'Ελληνικά', 'Ελλάδα', 'Greek', '\U0001f1ec\U0001f1f7'),
     ('en', 'English', '', 'English', '\U0001f310'),
     ('en-AU', 'English', 'Australia', 'English', '\U0001f1e6\U0001f1fa'),
     ('en-CA', 'English', 'Canada', 'English', '\U0001f1e8\U0001f1e6'),
     ('en-GB', 'English', 'United Kingdom', 'English', '\U0001f1ec\U0001f1e7'),
     ('en-IE', 'English', 'Ireland', 'English', '\U0001f1ee\U0001f1ea'),
+    ('en-IN', 'English', 'India', 'English', '\U0001f1ee\U0001f1f3'),
     ('en-MY', 'English', 'Malaysia', 'English', '\U0001f1f2\U0001f1fe'),
     ('en-NZ', 'English', 'New Zealand', 'English', '\U0001f1f3\U0001f1ff'),
+    ('en-PH', 'English', 'Philippines', 'English', '\U0001f1f5\U0001f1ed'),
     ('en-US', 'English', 'United States', 'English', '\U0001f1fa\U0001f1f8'),
+    ('en-ZA', 'English', 'South Africa', 'English', '\U0001f1ff\U0001f1e6'),
     ('es', 'Español', '', 'Spanish', '\U0001f310'),
     ('es-AR', 'Español', 'Argentina', 'Spanish', '\U0001f1e6\U0001f1f7'),
     ('es-CL', 'Español', 'Chile', 'Spanish', '\U0001f1e8\U0001f1f1'),
     ('es-ES', 'Español', 'España', 'Spanish', '\U0001f1ea\U0001f1f8'),
     ('es-MX', 'Español', 'México', 'Spanish', '\U0001f1f2\U0001f1fd'),
+    ('es-US', 'Español', 'Estados Unidos', 'Spanish', '\U0001f1fa\U0001f1f8'),
+    ('et', 'Eesti', '', 'Estonian', '\U0001f310'),
     ('et-EE', 'Eesti', 'Eesti', 'Estonian', '\U0001f1ea\U0001f1ea'),
-    ('fa-IR', 'فارسی', 'ایران', 'Persian', '\U0001f1ee\U0001f1f7'),
+    ('fi', 'Suomi', '', 'Finnish', '\U0001f310'),
     ('fi-FI', 'Suomi', 'Suomi', 'Finnish', '\U0001f1eb\U0001f1ee'),
-    ('fil-PH', 'Filipino', 'Pilipinas', 'Filipino', '\U0001f1f5\U0001f1ed'),
     ('fr', 'Français', '', 'French', '\U0001f310'),
     ('fr-BE', 'Français', 'Belgique', 'French', '\U0001f1e7\U0001f1ea'),
     ('fr-CA', 'Français', 'Canada', 'French', '\U0001f1e8\U0001f1e6'),
     ('fr-CH', 'Français', 'Suisse', 'French', '\U0001f1e8\U0001f1ed'),
     ('fr-FR', 'Français', 'France', 'French', '\U0001f1eb\U0001f1f7'),
-    ('he-IL', 'עברית', 'ישראל', 'Hebrew', '\U0001f1ee\U0001f1f1'),
-    ('hi-IN', 'हिन्दी', 'भारत', 'Hindi', '\U0001f1ee\U0001f1f3'),
-    ('hr-HR', 'Hrvatski', 'Hrvatska', 'Croatian', '\U0001f1ed\U0001f1f7'),
+    ('he', 'עברית', '', 'Hebrew', '\U0001f1ee\U0001f1f7'),
+    ('hi', 'हिन्दी', '', 'Hindi', '\U0001f310'),
+    ('hr', 'Hrvatski', '', 'Croatian', '\U0001f310'),
+    ('hu', 'Magyar', '', 'Hungarian', '\U0001f310'),
     ('hu-HU', 'Magyar', 'Magyarország', 'Hungarian', '\U0001f1ed\U0001f1fa'),
+    ('id', 'Indonesia', '', 'Indonesian', '\U0001f310'),
     ('id-ID', 'Indonesia', 'Indonesia', 'Indonesian', '\U0001f1ee\U0001f1e9'),
-    ('is-IS', 'Íslenska', 'Ísland', 'Icelandic', '\U0001f1ee\U0001f1f8'),
+    ('is', 'Íslenska', '', 'Icelandic', '\U0001f310'),
+    ('it', 'Italiano', '', 'Italian', '\U0001f310'),
+    ('it-CH', 'Italiano', 'Svizzera', 'Italian', '\U0001f1e8\U0001f1ed'),
     ('it-IT', 'Italiano', 'Italia', 'Italian', '\U0001f1ee\U0001f1f9'),
+    ('ja', '日本語', '', 'Japanese', '\U0001f310'),
     ('ja-JP', '日本語', '日本', 'Japanese', '\U0001f1ef\U0001f1f5'),
+    ('ko', '한국어', '', 'Korean', '\U0001f310'),
     ('ko-KR', '한국어', '대한민국', 'Korean', '\U0001f1f0\U0001f1f7'),
-    ('lt-LT', 'Lietuvių', 'Lietuva', 'Lithuanian', '\U0001f1f1\U0001f1f9'),
-    ('lv-LV', 'Latviešu', 'Latvija', 'Latvian', '\U0001f1f1\U0001f1fb'),
+    ('lt', 'Lietuvių', '', 'Lithuanian', '\U0001f310'),
+    ('lv', 'Latviešu', '', 'Latvian', '\U0001f310'),
+    ('nb', 'Norsk Bokmål', '', 'Norwegian Bokmål', '\U0001f310'),
+    ('nb-NO', 'Norsk Bokmål', 'Norge', 'Norwegian Bokmål', '\U0001f1f3\U0001f1f4'),
     ('nl', 'Nederlands', '', 'Dutch', '\U0001f310'),
     ('nl-BE', 'Nederlands', 'België', 'Dutch', '\U0001f1e7\U0001f1ea'),
     ('nl-NL', 'Nederlands', 'Nederland', 'Dutch', '\U0001f1f3\U0001f1f1'),
-    ('no-NO', 'Norsk', '', 'Norwegian (Bokmål)', '\U0001f1f3\U0001f1f4'),
+    ('pl', 'Polski', '', 'Polish', '\U0001f310'),
     ('pl-PL', 'Polski', 'Polska', 'Polish', '\U0001f1f5\U0001f1f1'),
     ('pt', 'Português', '', 'Portuguese', '\U0001f310'),
     ('pt-BR', 'Português', 'Brasil', 'Portuguese', '\U0001f1e7\U0001f1f7'),
     ('pt-PT', 'Português', 'Portugal', 'Portuguese', '\U0001f1f5\U0001f1f9'),
+    ('ro', 'Română', '', 'Romanian', '\U0001f310'),
     ('ro-RO', 'Română', 'România', 'Romanian', '\U0001f1f7\U0001f1f4'),
+    ('ru', 'Русский', '', 'Russian', '\U0001f310'),
     ('ru-RU', 'Русский', 'Россия', 'Russian', '\U0001f1f7\U0001f1fa'),
-    ('sk-SK', 'Slovenčina', 'Slovensko', 'Slovak', '\U0001f1f8\U0001f1f0'),
-    ('sl-SI', 'Slovenščina', 'Slovenija', 'Slovenian', '\U0001f1f8\U0001f1ee'),
-    ('sr-RS', 'Српски', 'Србија', 'Serbian', '\U0001f1f7\U0001f1f8'),
+    ('sk', 'Slovenčina', '', 'Slovak', '\U0001f310'),
+    ('sl', 'Slovenščina', '', 'Slovenian', '\U0001f310'),
+    ('sr', 'Српски', '', 'Serbian', '\U0001f310'),
+    ('sv', 'Svenska', '', 'Swedish', '\U0001f310'),
     ('sv-SE', 'Svenska', 'Sverige', 'Swedish', '\U0001f1f8\U0001f1ea'),
-    ('sw-TZ', 'Kiswahili', 'Tanzania', 'Swahili', '\U0001f1f9\U0001f1ff'),
+    ('th', 'ไทย', '', 'Thai', '\U0001f310'),
     ('th-TH', 'ไทย', 'ไทย', 'Thai', '\U0001f1f9\U0001f1ed'),
+    ('tr', 'Türkçe', '', 'Turkish', '\U0001f310'),
     ('tr-TR', 'Türkçe', 'Türkiye', 'Turkish', '\U0001f1f9\U0001f1f7'),
-    ('uk-UA', 'Українська', 'Україна', 'Ukrainian', '\U0001f1fa\U0001f1e6'),
-    ('vi-VN', 'Tiếng Việt', 'Việt Nam', 'Vietnamese', '\U0001f1fb\U0001f1f3'),
+    ('uk', 'Українська', '', 'Ukrainian', '\U0001f310'),
+    ('vi', 'Tiếng Việt', '', 'Vietnamese', '\U0001f310'),
     ('zh', '中文', '', 'Chinese', '\U0001f310'),
     ('zh-CN', '中文', '中国', 'Chinese', '\U0001f1e8\U0001f1f3'),
-    ('zh-HK', '中文', '中國香港', 'Chinese', '\U0001f1ed\U0001f1f0'),
+    ('zh-HK', '中文', '中國香港特別行政區', 'Chinese', '\U0001f1ed\U0001f1f0'),
     ('zh-TW', '中文', '台灣', 'Chinese', '\U0001f1f9\U0001f1fc'),
 )
+'''
+A list of five-digit tuples:
+
+0. SearXNG's internal locale tag (a language or region tag)
+1. Name of the language (:py:obj:`babel.core.Locale.get_language_name`)
+2. For region tags the name of the region (:py:obj:`babel.core.Locale.get_territory_name`).
+   Empty string for language tags.
+3. English language name (from :py:obj:`babel.core.Locale.english_name`)
+4. Unicode flag (emoji) that fits to SearXNG's internal region tag. Languages
+   are represented by a globe (🌐)
+
+.. code:: python
+
+   ('en',    'English', '',              'English', '🌐'),
+   ('en-CA', 'English', 'Canada',        'English', '🇨🇦'),
+   ('en-US', 'English', 'United States', 'English', '🇺🇸'),
+   ..
+   ('fr',    'Français', '',             'French',  '🌐'),
+   ('fr-BE', 'Français', 'Belgique',     'French',  '🇧🇪'),
+   ('fr-CA', 'Français', 'Canada',       'French',  '🇨🇦'),
+
+:meta hide-value:
+'''

searx/templates/simple/filters/languages.html

@@ -1,12 +1,12 @@
 <select class="language" id="language" name="language" aria-label="{{ _('Search language') }}">{{- '' -}}
-	<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
+	<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }} [all]</option>
 	<option value="auto" {% if current_language == 'auto' %}selected="selected"{% endif %}>
 		{{- _('Auto-detect') -}}
 		{%- if current_language == 'auto' %} ({{ search_language }}){%- endif -%}
 	</option>
-	{%- for lang_id,lang_name,country_name,english_name,flag in language_codes | sort(attribute=1) -%}
-	<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
-		{% if flag %}{{ flag }} {% endif%} {{- lang_name }} {% if country_name %}({{ country_name }}) {% endif %}
+	{%- for sxng_tag,lang_name,country_name,english_name,flag in sxng_locales | sort(attribute=1) -%}
+	<option value="{{ sxng_tag }}" {% if sxng_tag == current_language %}selected="selected"{% endif %}>
+		{% if flag %}{{ flag }} {% endif%} {{- lang_name }} {% if country_name %} - {{ country_name }} {% endif %} [{{sxng_tag}}]
 	</option>
 	{%- endfor -%}
 </select>

searx/templates/simple/preferences.html

@@ -115,10 +115,10 @@
       <legend id="pref_language">{{ _('Search language') }}</legend>
       <p class="value">{{- '' -}}
         <select name='language' aria-labelledby="pref_language" aria-describedby="desc_language">{{- '' -}}
-          <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
-          <option value="auto" {% if current_language == 'auto' %}selected="selected"{% endif %}>{{ _('Auto-detect') }}</option>
-          {%- for lang_id,lang_name,country_name,english_name,flag in language_codes | sort(attribute=1) -%}
-          <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{% if flag %}{{ flag }} {% endif%} {{- lang_name }} {% if country_name %}({{ country_name }}) {% endif %}</option>
+          <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }} [all]</option>
+          <option value="auto" {% if current_language == 'auto' %}selected="selected"{% endif %}>{{ _('Auto-detect') }} [auto]</option>
+          {%- for sxng_tag,lang_name,country_name,english_name,flag in sxng_locales | sort(attribute=1) -%}
+          <option value="{{ sxng_tag }}" {% if sxng_tag == current_language %}selected="selected"{% endif %}>{% if flag %}{{ flag }} {% endif%} {{- lang_name }} {% if country_name %} - {{ country_name }} {% endif %} [{{sxng_tag}}]</option>
           {%- endfor -%}
         </select>{{- '' -}}
       </p>

searx/translations/da/LC_MESSAGES/messages.po

@@ -12,15 +12,16 @@ msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2023-02-20 11:22+0000\n"
-"PO-Revision-Date: 2023-02-19 11:39+0000\n"
+"PO-Revision-Date: 2023-03-21 17:37+0000\n"
 "Last-Translator: return42 <markus.heiser@darmarit.de>\n"
+"Language-Team: Danish <https://translate.codeberg.org/projects/searxng/"
+"searxng/da/>\n"
 "Language: da\n"
-"Language-Team: Danish "
-"<https://translate.codeberg.org/projects/searxng/searxng/da/>\n"
-"Plural-Forms: nplurals=2; plural=n != 1;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+"X-Generator: Weblate 4.16.4\n"
 "Generated-By: Babel 2.11.0\n"
 
 #. CONSTANT_NAMES['DEFAULT_GROUP_NAME']
@@ -390,11 +391,11 @@ msgstr ""
 msgid ""
 "You are using Tor and it looks like you have this external IP address: "
 "{ip_address}"
-msgstr ""
+msgstr "Du bruger ikke Tor og du har denne eksterne IP adresse: {ip_address}"
 
 #: searx/plugins/tor_check.py:86
 msgid "You are not using Tor and you have this external IP address: {ip_address}"
-msgstr ""
+msgstr "Du bruger ikke Tor og du har denne eksterne IP adresse: {ip_address}"
 
 #: searx/plugins/tracker_url_remover.py:29
 msgid "Tracker URL remover"
@@ -1548,4 +1549,3 @@ msgstr "skjul video"
 
 #~ msgid "Automatically detect the query search language and switch to it."
 #~ msgstr "Registrer automatisk søgesproget og skift til det."
-

searx/translations/el_GR/LC_MESSAGES/messages.po

@@ -8,20 +8,22 @@
 # Markus Heiser <markus.heiser@darmarit.de>, 2022, 2023.
 # Constantine Giannopoulos <K.Giannopoulos@acg.edu>, 2022.
 # Alexandre Flament <alex@al-f.net>, 2022.
+# return42 <markus.heiser@darmarit.de>, 2023.
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2023-02-20 11:22+0000\n"
-"PO-Revision-Date: 2023-01-06 07:14+0000\n"
-"Last-Translator: Markus Heiser <markus.heiser@darmarit.de>\n"
+"PO-Revision-Date: 2023-03-24 07:07+0000\n"
+"Last-Translator: return42 <markus.heiser@darmarit.de>\n"
+"Language-Team: Greek <https://translate.codeberg.org/projects/searxng/"
+"searxng/el/>\n"
 "Language: el_GR\n"
-"Language-Team: Greek "
-"<https://weblate.bubu1.eu/projects/searxng/searxng/el/>\n"
-"Plural-Forms: nplurals=2; plural=n != 1;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+"X-Generator: Weblate 4.16.4\n"
 "Generated-By: Babel 2.11.0\n"
 
 #. CONSTANT_NAMES['DEFAULT_GROUP_NAME']
@@ -380,22 +382,31 @@ msgid ""
 "This plugin checks if the address of the request is a Tor exit-node, and "
 "informs the user if it is; like check.torproject.org, but from SearXNG."
 msgstr ""
+"Αυτό το πρόσθετο ελέγχει εάν η διεύθυνση του χρήστη είναι διεύθυνση εξόδου "
+"του δικτύου Tor και ενημερώνει τον χρήστη εάν είναι έτσι. Όπως στο "
+"check.torproject.org, αλλά από το SearXNG."
 
 #: searx/plugins/tor_check.py:62
 msgid ""
 "Could not download the list of Tor exit-nodes from: "
 "https://check.torproject.org/exit-addresses"
 msgstr ""
+"Δεν ήταν δυνατή η λήψη της λίστας διευθύνσεων εξόδου του δικτύου Tor από το: "
+"https://check.torproject.org/exit-addresses"
 
 #: searx/plugins/tor_check.py:78
 msgid ""
 "You are using Tor and it looks like you have this external IP address: "
 "{ip_address}"
 msgstr ""
+"Χρησιμοποιείτε το δίκτυο Tor και φαίνεται πως η εξωτερική σας διεύθυνση "
+"είναι η: {ip_address}"
 
 #: searx/plugins/tor_check.py:86
 msgid "You are not using Tor and you have this external IP address: {ip_address}"
 msgstr ""
+"Δεν χρησιμοποιείτε το δίκτυο Tor. Η εξωτερική σας διεύθυνση είναι: "
+"{ip_address}"
 
 #: searx/plugins/tracker_url_remover.py:29
 msgid "Tracker URL remover"
@@ -580,7 +591,7 @@ msgstr "Προεπιλεγμένη γλώσσα"
 #: searx/templates/simple/filters/languages.html:4
 #: searx/templates/simple/preferences.html:119
 msgid "Auto-detect"
-msgstr ""
+msgstr "Αυτόματη αναγνώριση της γλώσσας"
 
 #: searx/templates/simple/preferences.html:126
 msgid "What language do you prefer for search?"
@@ -589,6 +600,8 @@ msgstr "Τι γλώσσα προτιμάτε για αναζήτηση;"
 #: searx/templates/simple/preferences.html:126
 msgid "Choose Auto-detect to let SearXNG detect the language of your query."
 msgstr ""
+"Επιλέξτε αυτόματη αναγνώριση για να αφήσετε το SearXNG να αναγνωρίσει την "
+"γλώσσα του ερωτήματος σας αυτόματα."
 
 #: searx/templates/simple/preferences.html:132
 msgid "Autocomplete"
@@ -987,7 +1000,7 @@ msgstr "αναζήτηση"
 
 #: searx/templates/simple/stats.html:21
 msgid "There is currently no data available. "
-msgstr "Δεν υπάρχουν διαθέσιμα δεδομένα."
+msgstr "Δεν υπάρχουν διαθέσιμα δεδομένα. "
 
 #: searx/templates/simple/stats.html:26
 msgid "Scores"
@@ -1150,7 +1163,7 @@ msgstr "Ημερομηνία δημοσίευσης"
 
 #: searx/templates/simple/result_templates/paper.html:9
 msgid "Journal"
-msgstr ""
+msgstr "Περιοδικό"
 
 #: searx/templates/simple/result_templates/paper.html:22
 msgid "Editor"
@@ -1170,23 +1183,23 @@ msgstr "Σημάνσεις"
 
 #: searx/templates/simple/result_templates/paper.html:26
 msgid "DOI"
-msgstr ""
+msgstr "DOI"
 
 #: searx/templates/simple/result_templates/paper.html:27
 msgid "ISSN"
-msgstr ""
+msgstr "ISSN"
 
 #: searx/templates/simple/result_templates/paper.html:28
 msgid "ISBN"
-msgstr ""
+msgstr "ISBN"
 
 #: searx/templates/simple/result_templates/paper.html:33
 msgid "PDF"
-msgstr ""
+msgstr "PDF"
 
 #: searx/templates/simple/result_templates/paper.html:34
 msgid "HTML"
-msgstr ""
+msgstr "HTML"
 
 #: searx/templates/simple/result_templates/torrent.html:6
 msgid "magnet link"
@@ -1551,4 +1564,3 @@ msgstr "απόκρυψη βίντεο"
 
 #~ msgid "Automatically detect the query search language and switch to it."
 #~ msgstr ""
-