o

searx/autocomplete.py

@@ -5,20 +5,20 @@
 """
 # pylint: disable=use-dict-literal
 
-import json
+from json import loads
 from urllib.parse import urlencode
 
-import lxml
+from lxml import etree
 from httpx import HTTPError
 
 from searx import settings
-from searx.engines import (
+from searx.data import ENGINES_LANGUAGES
-    engines,
-    google,
-)
 from searx.network import get as http_get
 from searx.exceptions import SearxEngineResponseException
 
+# a fetch_supported_languages() for XPath engines isn't available right now
+# _brave = ENGINES_LANGUAGES['brave'].keys()
+
 
 def get(*args, **kwargs):
     if 'timeout' not in kwargs:
@@ -55,58 +55,34 @@ def dbpedia(query, _lang):
     results = []
 
     if response.ok:
-        dom = lxml.etree.fromstring(response.content)
+        dom = etree.fromstring(response.content)
         results = dom.xpath('//Result/Label//text()')
 
     return results
 
 
-def duckduckgo(query, sxng_locale):
+def duckduckgo(query, _lang):
-    """Autocomplete from DuckDuckGo. Supports DuckDuckGo's languages"""
+    # duckduckgo autocompleter
+    url = 'https://ac.duckduckgo.com/ac/?{0}&type=list'
 
-    traits = engines['duckduckgo'].traits
+    resp = loads(get(url.format(urlencode(dict(q=query)))).text)
-    args = {
+    if len(resp) > 1:
-        'q': query,
+        return resp[1]
-        'kl': traits.get_region(sxng_locale, traits.all_locale),
+    return []
-    }
-
-    url = 'https://duckduckgo.com/ac/?type=list&' + urlencode(args)
-    resp = get(url)
-
-    ret_val = []
-    if resp.ok:
-        j = resp.json()
-        if len(j) > 1:
-            ret_val = j[1]
-    return ret_val
 
 
-def google_complete(query, sxng_locale):
+def google(query, lang):
-    """Autocomplete from Google.  Supports Google's languages and subdomains
+    # google autocompleter
-    (:py:obj:`searx.engines.google.get_google_info`) by using the async REST
+    autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&'
-    API::
 
-        https://{subdomain}/complete/search?{args}
+    response = get(autocomplete_url + urlencode(dict(hl=lang, q=query)))
 
-    """
-
-    google_info = google.get_google_info({'searxng_locale': sxng_locale}, engines['google'].traits)
-
-    url = 'https://{subdomain}/complete/search?{args}'
-    args = urlencode(
-        {
-            'q': query,
-            'client': 'gws-wiz',
-            'hl': google_info['params']['hl'],
-        }
-    )
     results = []
-    resp = get(url.format(subdomain=google_info['subdomain'], args=args))
+
-    if resp.ok:
+    if response.ok:
-        json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
+        dom = etree.fromstring(response.text)
-        data = json.loads(json_txt)
+        results = dom.xpath('//suggestion/@data')
-        for item in data[0]:
+
-            results.append(lxml.html.fromstring(item[0]).text_content())
     return results
 
 
@@ -133,9 +109,9 @@ def seznam(query, _lang):
     ]
 
 
-def startpage(query, sxng_locale):
+def startpage(query, lang):
-    """Autocomplete from Startpage. Supports Startpage's languages"""
+    # startpage autocompleter
-    lui = engines['startpage'].traits.get_language(sxng_locale, 'english')
+    lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')
     url = 'https://startpage.com/suggestions?{query}'
     resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))
     data = resp.json()
@@ -146,20 +122,20 @@ def swisscows(query, _lang):
     # swisscows autocompleter
     url = 'https://swisscows.ch/api/suggest?{query}&itemsCount=5'
 
-    resp = json.loads(get(url.format(query=urlencode({'query': query}))).text)
+    resp = loads(get(url.format(query=urlencode({'query': query}))).text)
     return resp
 
 
-def qwant(query, sxng_locale):
+def qwant(query, lang):
-    """Autocomplete from Qwant. Supports Qwant's regions."""
+    # qwant autocompleter (additional parameter : lang=en_en&count=xxx )
+    url = 'https://api.qwant.com/api/suggest?{query}'
+
+    resp = get(url.format(query=urlencode({'q': query, 'lang': lang})))
+
     results = []
 
-    locale = engines['startpage'].traits.get_region(sxng_locale, 'en_US')
-    url = 'https://api.qwant.com/v3/suggest?{query}'
-    resp = get(url.format(query=urlencode({'q': query, 'locale': locale, 'version': '2'})))
-
     if resp.ok:
-        data = resp.json()
+        data = loads(resp.text)
         if data['status'] == 'success':
             for item in data['data']['items']:
                 results.append(item['value'])
@@ -167,38 +143,21 @@ def qwant(query, sxng_locale):
     return results
 
 
-def wikipedia(query, sxng_locale):
+def wikipedia(query, lang):
-    """Autocomplete from Wikipedia. Supports Wikipedia's languages (aka netloc)."""
+    # wikipedia autocompleter
-    results = []
+    url = 'https://' + lang + '.wikipedia.org/w/api.php?action=opensearch&{0}&limit=10&namespace=0&format=json'
-    eng_traits = engines['wikipedia'].traits
-    wiki_lang = eng_traits.get_language(sxng_locale, 'en')
-    wiki_netloc = eng_traits.custom['wiki_netloc'].get(wiki_lang, 'en.wikipedia.org')
 
-    url = 'https://{wiki_netloc}/w/api.php?{args}'
+    resp = loads(get(url.format(urlencode(dict(search=query)))).text)
-    args = urlencode(
+    if len(resp) > 1:
-        {
+        return resp[1]
-            'action': 'opensearch',
+    return []
-            'format': 'json',
-            'formatversion': '2',
-            'search': query,
-            'namespace': '0',
-            'limit': '10',
-        }
-    )
-    resp = get(url.format(args=args, wiki_netloc=wiki_netloc))
-    if resp.ok:
-        data = resp.json()
-        if len(data) > 1:
-            results = data[1]
-
-    return results
 
 
 def yandex(query, _lang):
     # yandex autocompleter
     url = "https://suggest.yandex.com/suggest-ff.cgi?{0}"
 
-    resp = json.loads(get(url.format(urlencode(dict(part=query)))).text)
+    resp = loads(get(url.format(urlencode(dict(part=query)))).text)
     if len(resp) > 1:
         return resp[1]
     return []
@@ -207,7 +166,7 @@ def yandex(query, _lang):
 backends = {
     'dbpedia': dbpedia,
     'duckduckgo': duckduckgo,
-    'google': google_complete,
+    'google': google,
     'seznam': seznam,
     'startpage': startpage,
     'swisscows': swisscows,
@@ -218,11 +177,12 @@ backends = {
 }
 
 
-def search_autocomplete(backend_name, query, sxng_locale):
+def search_autocomplete(backend_name, query, lang):
     backend = backends.get(backend_name)
     if backend is None:
         return []
+
     try:
-        return backend(query, sxng_locale)
+        return backend(query, lang)
     except (HTTPError, SearxEngineResponseException):
         return []

searx/baidu.py

@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Bing (Web)
+
+- https://github.com/searx/searx/issues/2019#issuecomment-648227442
+"""
+
+import re
+from urllib.parse import urlencode
+from lxml import html
+from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_getindex
+from searx.network import raise_for_httperror, multi_requests, get, Request
+from searx.exceptions import SearxEngineCaptchaException
+
+about = {
+    "website": 'https://www.baidu.com',
+    "wikidata_id": 'Q14772',
+    "official_api_documentation": 'https://apis.baidu.com/',
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+    "language": 'zn',
+}
+
+# engine dependent config
+categories = ['general', 'web']
+paging = False
+time_range_support = False
+safesearch = False
+
+base_url = 'https://www.baidu.com/'
+search_string = 's?{query}'
+
+skip_tpls = ('img_normal', 'short_video', 'yl_music_song', 'dict3', 'recommend_list')
+
+desc_xpath_per_tpl = {
+    'se_com_default': './/span[contains(@class, "content-right_8Zs40")]',
+    'kaifa_pc_open_source_software': './/p[contains(@class, "c-color-text")]',
+    'bk_polysemy': './/div/@aria-label',
+    'se_st_single_video_zhanzhang': './/span[contains(@class, "c-span-last")]//p[2]',
+}
+
+
+def get_initial_parameters(params):
+    resp_index = get(base_url, headers=params['headers'], raise_for_httperror=True)
+    dom = html.fromstring(resp_index.text)
+    query_params = {}
+    for ielement in eval_xpath_list(dom, '//form[@id="form"]//input[@name]'):
+        name = ielement.attrib.get('name')
+        value = ielement.attrib.get('value')
+        query_params[name] = value
+    return query_params, resp_index.cookies
+
+
+def request(query, params):
+    params['headers'].update(
+        {
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'same-origin',
+            'Sec-Fetch-User': '?1',
+            'Sec-GPC': '1',
+            'Upgrade-Insecure-Requests': '1',
+            'TE': 'trailers',
+        }
+    )
+
+    query_params, cookies = get_initial_parameters(params)
+    query_params['wd'] = query
+
+    params['url'] = base_url + search_string.format(query=urlencode(query_params))
+    params['cookies'] = cookies
+    params['raise_for_httperror'] = False
+    return params
+
+
+def response(resp):
+    results = []
+
+    if resp.url.host == 'wappass.baidu.com' or resp.url.path.startswith('/static/captcha'):
+        raise SearxEngineCaptchaException()
+    raise_for_httperror(resp)
+
+    dom = html.fromstring(resp.text)
+
+    # follow redirect but don't use the result page to reduce the CAPTCHA issue
+    redirect_element = eval_xpath_getindex(dom, '//noscript/meta[@http-equiv="refresh"]/@content', 0, default=None)
+    if redirect_element and redirect_element.startswith('0; url='):
+        get(
+            base_url + redirect_element[8:],
+            headers=resp.search_params['headers'],
+            cookies=resp.search_params['cookies'],
+        )
+
+    for result in eval_xpath_list(dom, '//div[contains(@id,"content_left")]/div[contains(@class, "c-container")]'):
+        tpl = result.attrib.get('tpl')
+        if tpl in skip_tpls:
+            continue
+
+        if tpl == 'kaifa_pc_blog_weak':
+            # skip the result to kaifa.baidu.com (search engine for IT)
+            # but includes results from kaifa
+            for r2 in eval_xpath_list(result, './/div[contains(@class, "c-gap-bottom-small")]'):
+                title = extract_text(eval_xpath(r2, './/div[@class="c-row"]//a'))
+                url = extract_text(eval_xpath(r2, './/div[@class="c-row"]//a/@href'))
+                content = extract_text(eval_xpath(r2, '//span[@class="c-line-clamp2"]'))
+                results.append(
+                    {
+                        'url': url,
+                        'title': title,
+                        'content': content,
+                    }
+                )
+            continue
+
+        # normal results
+        title = extract_text(eval_xpath(result, './/h3/a'))
+        url = extract_text(eval_xpath(result, './/h3/a/@href'))
+
+        if not title or not url:
+            continue
+
+        content = None
+        if tpl in desc_xpath_per_tpl:
+            # try the XPath for the Baidu template
+            content = extract_text(eval_xpath(result, desc_xpath_per_tpl[tpl]))
+        if not content:
+            # no content was found: try all the XPath from the Baidu templates
+            for xp in desc_xpath_per_tpl.values():
+                content = extract_text(eval_xpath(result, xp))
+                if content:
+                    break
+        results.append(
+            {
+                'url': url,
+                'title': title,
+                'content': content,
+            }
+        )
+
+    # resolve the Baidu redirections
+    # note: Baidu does not support HTTP/2
+    request_list = [
+        Request.get(
+            u['url'].replace('http://www.baidu.com/link?url=', 'https://www.baidu.com/link?url='),
+            allow_redirects=False,
+            headers=resp.search_params['headers'],
+        )
+        for u in results
+    ]
+    response_list = multi_requests(request_list)
+    for i, redirect_response in enumerate(response_list):
+        if not isinstance(redirect_response, Exception):
+            results[i]['url'] = redirect_response.headers['location']
+
+    return results
+
+
+def debug_write_content_to_file(text):
+    RE_STYLE_ELEMENT = re.compile(r'<style[^>]*>[^<]+</style>')
+    RE_SCRIPT_ELEMENT = re.compile(r'<script[^>]*>[^<]+</script>')
+    RE_COMMENT_ELEMENT = re.compile(r'\<\!\-\-[^-]+\-\-\>')
+    with open('baidu.html', 'wt', encoding='utf-8') as f:
+        text = RE_STYLE_ELEMENT.sub("", text)
+        text = RE_SCRIPT_ELEMENT.sub("", text)
+        text = RE_COMMENT_ELEMENT.sub("", text)
+        text = "\n".join([ll.rstrip() for ll in text.splitlines() if ll.strip()])
+        f.write(text)

searx/engines/google.py

@@ -1,39 +1,34 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the Google WEB engine.  Some of this
+"""This is the implementation of the google WEB engine.  Some of this
-implementations (manly the :py:obj:`get_google_info`) are shared by other
+implementations are shared by other engines:
-engines:
 
 - :ref:`google images engine`
 - :ref:`google news engine`
 - :ref:`google videos engine`
-- :ref:`google scholar engine`
+
-- :ref:`google autocomplete`
+The google WEB engine itself has a special setup option:
+
+.. code:: yaml
+
+  - name: google
+    ...
+    use_mobile_ui: false
+
+``use_mobile_ui``: (default: ``false``)
+  Enables to use *mobile endpoint* to bypass the google blocking (see
+  :issue:`159`).  On the mobile UI of Google Search, the button :guilabel:`More
+  results` is not affected by Google rate limiting and we can still do requests
+  while actively blocked by the original Google search.  By activate
+  ``use_mobile_ui`` this behavior is simulated by adding the parameter
+  ``async=use_ac:true,_fmt:pc`` to the :py:func:`request`.
 
 """
 
-from typing import TYPE_CHECKING
-
-import re
 from urllib.parse import urlencode
 from lxml import html
-import babel
+from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
-import babel.core
-import babel.languages
-
-from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
-from searx.locales import language_tag, region_tag, get_offical_locales
-from searx import network
 from searx.exceptions import SearxEngineCaptchaException
-from searx.enginelib.traits import EngineTraits
-
-if TYPE_CHECKING:
-    import logging
-
-    logger: logging.Logger
-
-traits: EngineTraits
-
 
 # about
 about = {
@@ -50,6 +45,64 @@ categories = ['general', 'web']
 paging = True
 time_range_support = True
 safesearch = True
+send_accept_language_header = True
+use_mobile_ui = False
+supported_languages_url = 'https://www.google.com/preferences?#languages'
+
+# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
+google_domains = {
+    'BG': 'google.bg',  # Bulgaria
+    'CZ': 'google.cz',  # Czech Republic
+    'DE': 'google.de',  # Germany
+    'DK': 'google.dk',  # Denmark
+    'AT': 'google.at',  # Austria
+    'CH': 'google.ch',  # Switzerland
+    'GR': 'google.gr',  # Greece
+    'AU': 'google.com.au',  # Australia
+    'CA': 'google.ca',  # Canada
+    'GB': 'google.co.uk',  # United Kingdom
+    'ID': 'google.co.id',  # Indonesia
+    'IE': 'google.ie',  # Ireland
+    'IN': 'google.co.in',  # India
+    'MY': 'google.com.my',  # Malaysia
+    'NZ': 'google.co.nz',  # New Zealand
+    'PH': 'google.com.ph',  # Philippines
+    'SG': 'google.com.sg',  # Singapore
+    'US': 'google.com',  # United States (google.us) redirects to .com
+    'ZA': 'google.co.za',  # South Africa
+    'AR': 'google.com.ar',  # Argentina
+    'CL': 'google.cl',  # Chile
+    'ES': 'google.es',  # Spain
+    'MX': 'google.com.mx',  # Mexico
+    'EE': 'google.ee',  # Estonia
+    'FI': 'google.fi',  # Finland
+    'BE': 'google.be',  # Belgium
+    'FR': 'google.fr',  # France
+    'IL': 'google.co.il',  # Israel
+    'HR': 'google.hr',  # Croatia
+    'HU': 'google.hu',  # Hungary
+    'IT': 'google.it',  # Italy
+    'JP': 'google.co.jp',  # Japan
+    'KR': 'google.co.kr',  # South Korea
+    'LT': 'google.lt',  # Lithuania
+    'LV': 'google.lv',  # Latvia
+    'NO': 'google.no',  # Norway
+    'NL': 'google.nl',  # Netherlands
+    'PL': 'google.pl',  # Poland
+    'BR': 'google.com.br',  # Brazil
+    'PT': 'google.pt',  # Portugal
+    'RO': 'google.ro',  # Romania
+    'RU': 'google.ru',  # Russia
+    'SK': 'google.sk',  # Slovakia
+    'SI': 'google.si',  # Slovenia
+    'SE': 'google.se',  # Sweden
+    'TH': 'google.co.th',  # Thailand
+    'TR': 'google.com.tr',  # Turkey
+    'UA': 'google.com.ua',  # Ukraine
+    'CN': 'google.com.hk',  # There is no google.cn, we use .com.hk for zh-CN
+    'HK': 'google.com.hk',  # Hong Kong
+    'TW': 'google.com.tw',  # Taiwan
+}
 
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
 
@@ -62,50 +115,47 @@ filter_mapping = {0: 'off', 1: 'medium', 2: 'high'}
 results_xpath = './/div[@data-sokoban-container]'
 title_xpath = './/a/h3[1]'
 href_xpath = './/a[h3]/@href'
-content_xpath = './/div[@data-content-feature]'
+content_xpath = './/div[@data-content-feature=1]'
 
 # google *sections* are no usual *results*, we ignore them
 g_section_with_header = './g-section-with-header'
 
+
 # Suggestions are links placed in a *card-section*, we extract only the text
 # from the links not the links itself.
 suggestion_xpath = '//div[contains(@class, "EIaa9b")]//a'
 
-# UI_ASYNC = 'use_ac:true,_fmt:html' # returns a HTTP 500 when user search for
-#                                    # celebrities like '!google natasha allegri'
-#                                    # or '!google chris evans'
-UI_ASYNC = 'use_ac:true,_fmt:prog'
-"""Format of the response from UI's async request."""
 
-
+def get_lang_info(params, lang_list, custom_aliases, supported_any_language):
-def get_google_info(params, eng_traits):
+    """Composing various language properties for the google engines.
-    """Composing various (language) properties for the google engines (:ref:`google
-    API`).
 
     This function is called by the various google engines (:ref:`google web
     engine`, :ref:`google images engine`, :ref:`google news engine` and
     :ref:`google videos engine`).
 
-    :param dict param: Request parameters of the engine.  At least
+    :param dict param: request parameters of the engine
-        a ``searxng_locale`` key should be in the dictionary.
 
-    :param eng_traits: Engine's traits fetched from google preferences
+    :param list lang_list: list of supported languages of the engine
-        (:py:obj:`searx.enginelib.traits.EngineTraits`)
+        :py:obj:`ENGINES_LANGUAGES[engine-name] <searx.data.ENGINES_LANGUAGES>`
+
+    :param dict lang_list: custom aliases for non standard language codes
+        (used when calling :py:func:`searx.utils.match_language`)
+
+    :param bool supported_any_language: When a language is not specified, the
+        language interpretation is left up to Google to decide how the search
+        results should be delivered.  This argument is ``True`` for the google
+        engine and ``False`` for the other engines (google-images, -news,
+        -scholar, -videos).
 
     :rtype: dict
     :returns:
         Py-Dictionary with the key/value pairs:
 
         language:
-            The language code that is used by google (e.g. ``lang_en`` or
+            Return value from :py:func:`searx.utils.match_language`
-            ``lang_zh-TW``)
 
         country:
-            The country code that is used by google (e.g. ``US`` or ``TW``)
+            The country code (e.g. US, AT, CA, FR, DE ..)
-
-        locale:
-            A instance of :py:obj:`babel.core.Locale` build from the
-            ``searxng_locale`` value.
 
         subdomain:
             Google subdomain :py:obj:`google_domains` that fits to the country
@@ -115,67 +165,52 @@ def get_google_info(params, eng_traits):
             Py-Dictionary with additional request arguments (can be passed to
             :py:func:`urllib.parse.urlencode`).
 
-            - ``hl`` parameter: specifies the interface language of user interface.
-            - ``lr`` parameter: restricts search results to documents written in
-              a particular language.
-            - ``cr`` parameter: restricts search results to documents
-              originating in a particular country.
-            - ``ie`` parameter: sets the character encoding scheme that should
-              be used to interpret the query string ('utf8').
-            - ``oe`` parameter: sets the character encoding scheme that should
-              be used to decode the XML result ('utf8').
-
         headers:
             Py-Dictionary with additional HTTP headers (can be passed to
             request's headers)
-
-            - ``Accept: '*/*``
-
     """
-
     ret_val = {
         'language': None,
         'country': None,
         'subdomain': None,
         'params': {},
         'headers': {},
-        'cookies': {},
-        'locale': None,
     }
 
-    sxng_locale = params.get('searxng_locale', 'all')
+    # language ...
-    try:
-        locale = babel.Locale.parse(sxng_locale, sep='-')
-    except babel.core.UnknownLocaleError:
-        locale = None
 
-    eng_lang = eng_traits.get_language(sxng_locale, 'lang_en')
+    _lang = params['language']
-    lang_code = eng_lang.split('_')[-1]  # lang_zh-TW --> zh-TW / lang_en --> en
+    _any_language = _lang.lower() == 'all'
-    country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
+    if _any_language:
+        _lang = 'en-US'
+    language = match_language(_lang, lang_list, custom_aliases)
+    ret_val['language'] = language
 
-    # Test zh_hans & zh_hant --> in the topmost links in the result list of list
+    # country ...
-    # TW and HK you should a find wiktionary.org zh_hant link.  In the result
-    # list of zh-CN should not be no hant link instead you should find
-    # zh.m.wikipedia.org/zh somewhere in the top.
 
-    # '!go 日 :zh-TW' --> https://zh.m.wiktionary.org/zh-hant/%E6%97%A5
+    _l = _lang.split('-')
-    # '!go 日 :zh-CN' --> https://zh.m.wikipedia.org/zh/%E6%97%A5
+    if len(_l) == 2:
-
+        country = _l[1]
-    ret_val['language'] = eng_lang
+    else:
+        country = _l[0].upper()
+        if country == 'EN':
+            country = 'US'
     ret_val['country'] = country
-    ret_val['locale'] = locale
+
-    ret_val['subdomain'] = eng_traits.custom['supported_domains'].get(country.upper(), 'www.google.com')
+    # subdomain ...
+
+    ret_val['subdomain'] = 'www.' + google_domains.get(country.upper(), 'google.com')
+
+    # params & headers
+
+    lang_country = '%s-%s' % (language, country)  # (en-US, en-EN, de-DE, de-AU, fr-FR ..)
 
     # hl parameter:
-    #   The hl parameter specifies the interface language (host language) of
+    #   https://developers.google.com/custom-search/docs/xml_results#hlsp The
-    #   your user interface. To improve the performance and the quality of your
+    # Interface Language:
-    #   search results, you are strongly encouraged to set this parameter
-    #   explicitly.
-    #   https://developers.google.com/custom-search/docs/xml_results#hlsp
-    # The Interface Language:
     #   https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
 
-    ret_val['params']['hl'] = lang_code
+    ret_val['params']['hl'] = lang_list.get(lang_country, language)
 
     # lr parameter:
     #   The lr (language restrict) parameter restricts search results to
@@ -183,72 +218,22 @@ def get_google_info(params, eng_traits):
     #   https://developers.google.com/custom-search/docs/xml_results#lrsp
     #   Language Collection Values:
     #   https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections
+
+    if _any_language and supported_any_language:
+
+        # interpretation is left up to Google (based on whoogle)
         #
-    # To select 'all' languages an empty 'lr' value is used.
+        # - add parameter ``source=lnt``
-    #
+        # - don't use parameter ``lr``
-    # Different to other google services, Google Schloar supports to select more
+        # - don't add a ``Accept-Language`` HTTP header.
-    # than one language. The languages are seperated by a pipe '|' (logical OR).
-    # By example: &lr=lang_zh-TW%7Clang_de selects articles written in
-    # traditional chinese OR german language.
 
-    ret_val['params']['lr'] = eng_lang
+        ret_val['params']['source'] = 'lnt'
-    if sxng_locale == 'all':
-        ret_val['params']['lr'] = ''
 
-    # cr parameter:
+    else:
-    #   The cr parameter restricts search results to documents originating in a
-    #   particular country.
-    #   https://developers.google.com/custom-search/docs/xml_results#crsp
 
-    ret_val['params']['cr'] = 'country' + country
+        # restricts search results to documents written in a particular
-    if sxng_locale == 'all':
+        # language.
-        ret_val['params']['cr'] = ''
+        ret_val['params']['lr'] = "lang_" + lang_list.get(lang_country, language)
-
-    # gl parameter: (mandatory by Geeogle News)
-    #   The gl parameter value is a two-letter country code. For WebSearch
-    #   results, the gl parameter boosts search results whose country of origin
-    #   matches the parameter value. See the Country Codes section for a list of
-    #   valid values.
-    #   Specifying a gl parameter value in WebSearch requests should improve the
-    #   relevance of results. This is particularly true for international
-    #   customers and, even more specifically, for customers in English-speaking
-    #   countries other than the United States.
-    #   https://developers.google.com/custom-search/docs/xml_results#glsp
-
-    ret_val['params']['gl'] = country
-
-    # ie parameter:
-    #   The ie parameter sets the character encoding scheme that should be used
-    #   to interpret the query string. The default ie value is latin1.
-    #   https://developers.google.com/custom-search/docs/xml_results#iesp
-
-    ret_val['params']['ie'] = 'utf8'
-
-    # oe parameter:
-    #   The oe parameter sets the character encoding scheme that should be used
-    #   to decode the XML result. The default oe value is latin1.
-    #   https://developers.google.com/custom-search/docs/xml_results#oesp
-
-    ret_val['params']['oe'] = 'utf8'
-
-    # num parameter:
-    #   The num parameter identifies the number of search results to return.
-    #   The default num value is 10, and the maximum value is 20. If you request
-    #   more than 20 results, only 20 results will be returned.
-    #   https://developers.google.com/custom-search/docs/xml_results#numsp
-
-    # HINT: seems to have no effect (tested in google WEB & Images)
-    # ret_val['params']['num'] = 20
-
-    # HTTP headers
-
-    ret_val['headers']['Accept'] = '*/*'
-
-    # Cookies
-
-    # - https://github.com/searxng/searxng/pull/1679#issuecomment-1235432746
-    # - https://github.com/searxng/searxng/issues/1555
-    ret_val['cookies']['CONSENT'] = "YES+"
 
     return ret_val
 
@@ -260,34 +245,33 @@ def detect_google_sorry(resp):
 
 def request(query, params):
     """Google search request"""
-    # pylint: disable=line-too-long
+
     offset = (params['pageno'] - 1) * 10
-    google_info = get_google_info(params, traits)
+
+    lang_info = get_lang_info(params, supported_languages, language_aliases, True)
+
+    additional_parameters = {}
+    if use_mobile_ui:
+        additional_parameters = {
+            'asearch': 'arc',
+            'async': 'use_ac:true,_fmt:prog',
+        }
 
     # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
     query_url = (
         'https://'
-        + google_info['subdomain']
+        + lang_info['subdomain']
         + '/search'
         + "?"
         + urlencode(
             {
                 'q': query,
-                **google_info['params'],
+                **lang_info['params'],
-                'filter': '0',
+                'ie': "utf8",
+                'oe': "utf8",
                 'start': offset,
-                # 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
+                'filter': '0',
-                # 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
+                **additional_parameters,
-                # 'cs' : 1,
-                # 'sa': 'N',
-                # 'yv': 3,
-                # 'prmd': 'vin',
-                # 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
-                # 'sa': 'N',
-                # 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
-                # formally known as use_mobile_ui
-                'asearch': 'arc',
-                'async': UI_ASYNC,
             }
         )
     )
@@ -298,45 +282,25 @@ def request(query, params):
         query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
     params['url'] = query_url
 
-    params['cookies'] = google_info['cookies']
+    params['cookies']['CONSENT'] = "YES+"
-    params['headers'].update(google_info['headers'])
+    params['headers'].update(lang_info['headers'])
+    if use_mobile_ui:
+        params['headers']['Accept'] = '*/*'
+    else:
+        params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+
     return params
 
 
-# (function(){var s='data:image/jpeg;base64,/9j/4AAQ ...
-# ... DX/Ff5XSpSgdU32xSlKDJ//9k\x3d';var ii=['dimg_21'];_setImagesSrc(ii,s);})();
-RE_DATA_IMAGE = re.compile(r"'(data:image[^']*)'[^']*ii=\['([^']*)'\];_setImagesSrc")
-
-
-def _parse_data_images(dom):
-    data_image_map = {}
-    for _script in eval_xpath_list(dom, "//script[@nonce]"):
-        script = _script.text
-        if not script:
-            continue
-        script = RE_DATA_IMAGE.search(script)
-        if not script:
-            continue
-        data_image_map[script.group(2)] = script.group(1).replace(r'\x3d', '=')
-    logger.debug('data:image objects --> %s', list(data_image_map.keys()))
-    return data_image_map
-
-
 def response(resp):
     """Get response from google's search request"""
-    # pylint: disable=too-many-branches, too-many-statements
+
     detect_google_sorry(resp)
 
     results = []
 
     # convert the text to dom
     dom = html.fromstring(resp.text)
-
-    data_image_map = []
-    if '_fmt:html' in UI_ASYNC:
-        # in this format images are embedded by a bse64 encoded 'data:image'
-        data_image_map = _parse_data_images(dom)
-
     # results --> answer
     answer_list = eval_xpath(dom, '//div[contains(@class, "LGOjhe")]')
     if answer_list:
@@ -345,9 +309,20 @@ def response(resp):
     else:
         logger.debug("did not find 'answer'")
 
+        # results --> number_of_results
+        if not use_mobile_ui:
+            try:
+                _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0)
+                _digit = ''.join([n for n in _txt if n.isdigit()])
+                number_of_results = int(_digit)
+                results.append({'number_of_results': number_of_results})
+            except Exception as e:  # pylint: disable=broad-except
+                logger.debug("did not 'number_of_results'")
+                logger.error(e, exc_info=True)
+
     # parse results
 
-    for result in eval_xpath_list(dom, results_xpath):  # pylint: disable=too-many-nested-blocks
+    for result in eval_xpath_list(dom, results_xpath):
 
         # google *sections*
         if extract_text(eval_xpath(result, g_section_with_header)):
@@ -364,31 +339,13 @@ def response(resp):
             url = eval_xpath_getindex(result, href_xpath, 0, None)
             if url is None:
                 continue
-
+            content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True)
-            content = []
+            if content is None:
-            img_list = []
-            for content_feature in eval_xpath(result, content_xpath):
-                val = content_feature.attrib['data-content-feature']
-                if val in ['1', '2']:
-                    txt = extract_text(content_feature, allow_none=True)
-                    if txt:
-                        content.append(txt)
-                elif '0' in val:
-                    img = content_feature.xpath('.//img/@src')
-                    if img:
-                        img = img[0]
-                        if img.startswith('data:image'):
-                            img_id = content_feature.xpath('.//img/@id')
-                            if img_id:
-                                img = data_image_map.get(img_id[0])
-                        img_list.append(img)
-
-            if not content:
                 logger.debug('ignoring item from the result_xpath list: missing content of title "%s"', title)
                 continue
-            content = ' / '.join(content)
+
-            img_src = img_list[0] if img_list else None
+            logger.debug('add link to results: %s', title)
-            results.append({'url': url, 'title': title, 'content': content, 'img_src': img_src})
+            results.append({'url': url, 'title': title, 'content': content})
 
         except Exception as e:  # pylint: disable=broad-except
             logger.error(e, exc_info=True)
@@ -404,107 +361,15 @@ def response(resp):
 
 
 # get supported languages from their site
-
+def _fetch_supported_languages(resp):
-
+    ret_val = {}
-skip_countries = [
-    # official language of google-country not in google-languages
-    'AL',  # Albanien (sq)
-    'AZ',  # Aserbaidschan  (az)
-    'BD',  # Bangladesch (bn)
-    'BN',  # Brunei Darussalam (ms)
-    'BT',  # Bhutan (dz)
-    'ET',  # Äthiopien (am)
-    'GE',  # Georgien (ka, os)
-    'GL',  # Grönland (kl)
-    'KH',  # Kambodscha (km)
-    'LA',  # Laos (lo)
-    'LK',  # Sri Lanka (si, ta)
-    'ME',  # Montenegro (sr)
-    'MK',  # Nordmazedonien (mk, sq)
-    'MM',  # Myanmar (my)
-    'MN',  # Mongolei (mn)
-    'MV',  # Malediven (dv) // dv_MV is unknown by babel
-    'MY',  # Malaysia (ms)
-    'NP',  # Nepal (ne)
-    'TJ',  # Tadschikistan (tg)
-    'TM',  # Turkmenistan (tk)
-    'UZ',  # Usbekistan (uz)
-]
-
-
-def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
-    """Fetch languages from Google."""
-    # pylint: disable=import-outside-toplevel, too-many-branches
-
-    engine_traits.custom['supported_domains'] = {}
-
-    resp = network.get('https://www.google.com/preferences')
-    if not resp.ok:
-        raise RuntimeError("Response from Google's preferences is not OK.")
-
     dom = html.fromstring(resp.text)
 
-    # supported language codes
+    radio_buttons = eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]')
 
-    lang_map = {'no': 'nb'}
+    for x in radio_buttons:
-    for x in eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]'):
+        name = x.get("data-name")
+        code = x.get("value").split('_')[-1]
+        ret_val[code] = {"name": name}
 
-        eng_lang = x.get("value").split('_')[-1]
+    return ret_val
-        try:
-            locale = babel.Locale.parse(lang_map.get(eng_lang, eng_lang), sep='-')
-        except babel.UnknownLocaleError:
-            print("ERROR: %s -> %s is unknown by babel" % (x.get("data-name"), eng_lang))
-            continue
-        sxng_lang = language_tag(locale)
-
-        conflict = engine_traits.languages.get(sxng_lang)
-        if conflict:
-            if conflict != eng_lang:
-                print("CONFLICT: babel %s --> %s, %s" % (sxng_lang, conflict, eng_lang))
-            continue
-        engine_traits.languages[sxng_lang] = 'lang_' + eng_lang
-
-    # alias languages
-    engine_traits.languages['zh'] = 'lang_zh-CN'
-
-    # supported region codes
-
-    for x in eval_xpath_list(dom, '//*[@name="region"]/..//input[@name="region"]'):
-        eng_country = x.get("value")
-
-        if eng_country in skip_countries:
-            continue
-        if eng_country == 'ZZ':
-            engine_traits.all_locale = 'ZZ'
-            continue
-
-        sxng_locales = get_offical_locales(eng_country, engine_traits.languages.keys(), regional=True)
-
-        if not sxng_locales:
-            print("ERROR: can't map from google country %s (%s) to a babel region." % (x.get('data-name'), eng_country))
-            continue
-
-        for sxng_locale in sxng_locales:
-            engine_traits.regions[region_tag(sxng_locale)] = eng_country
-
-    # alias regions
-    engine_traits.regions['zh-CN'] = 'HK'
-
-    # supported domains
-
-    if add_domains:
-        resp = network.get('https://www.google.com/supported_domains')
-        if not resp.ok:
-            raise RuntimeError("Response from https://www.google.com/supported_domains is not OK.")
-
-        for domain in resp.text.split():
-            domain = domain.strip()
-            if not domain or domain in [
-                '.google.com',
-            ]:
-                continue
-            region = domain.split('.')[-1].upper()
-            engine_traits.custom['supported_domains'][region] = 'www' + domain
-            if region == 'HK':
-                # There is no google.cn, we use .com.hk for zh-CN
-                engine_traits.custom['supported_domains']['CN'] = 'www' + domain

searx/engines/google_images.py

@@ -1,38 +1,31 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the Google Images engine using the internal
+"""This is the implementation of the google images engine using the google
-Google API used by the Google Go Android app.
+internal API used the Google Go Android app.
 
 This internal API offer results in
 
-- JSON (``_fmt:json``)
+- JSON (_fmt:json)
-- Protobuf_ (``_fmt:pb``)
+- Protobuf (_fmt:pb)
-- Protobuf_ compressed? (``_fmt:pc``)
+- Protobuf compressed? (_fmt:pc)
-- HTML (``_fmt:html``)
+- HTML (_fmt:html)
-- Protobuf_ encoded in JSON (``_fmt:jspb``).
+- Protobuf encoded in JSON (_fmt:jspb).
 
-.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
 """
 
-from typing import TYPE_CHECKING
-
 from urllib.parse import urlencode
 from json import loads
 
-from searx.engines.google import fetch_traits  # pylint: disable=unused-import
 from searx.engines.google import (
-    get_google_info,
+    get_lang_info,
     time_range_dict,
     detect_google_sorry,
 )
 
-if TYPE_CHECKING:
+# pylint: disable=unused-import
-    import logging
+from searx.engines.google import supported_languages_url, _fetch_supported_languages
-    from searx.enginelib.traits import EngineTraits
-
-    logger: logging.Logger
-    traits: EngineTraits
 
+# pylint: enable=unused-import
 
 # about
 about = {
@@ -47,6 +40,7 @@ about = {
 # engine dependent config
 categories = ['images', 'web']
 paging = True
+use_locale_domain = True
 time_range_support = True
 safesearch = True
 send_accept_language_header = True
@@ -57,18 +51,20 @@ filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
 def request(query, params):
     """Google-Image search request"""
 
-    google_info = get_google_info(params, traits)
+    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
 
     query_url = (
         'https://'
-        + google_info['subdomain']
+        + lang_info['subdomain']
         + '/search'
         + "?"
         + urlencode(
             {
                 'q': query,
                 'tbm': "isch",
-                **google_info['params'],
+                **lang_info['params'],
+                'ie': "utf8",
+                'oe': "utf8",
                 'asearch': 'isch',
                 'async': '_fmt:json,p:1,ijn:' + str(params['pageno']),
             }
@@ -81,8 +77,9 @@ def request(query, params):
         query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
     params['url'] = query_url
 
-    params['cookies'] = google_info['cookies']
+    params['headers'].update(lang_info['headers'])
-    params['headers'].update(google_info['headers'])
+    params['headers']['User-Agent'] = 'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12; US) gzip'
+    params['headers']['Accept'] = '*/*'
     return params
 
 
@@ -114,11 +111,7 @@ def response(resp):
 
         copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
         if copyright_notice:
-            result_item['source'] += ' | ' + copyright_notice
+            result_item['source'] += ' / ' + copyright_notice
-
-        freshness_date = item["result"].get("freshness_date")
-        if freshness_date:
-            result_item['source'] += ' | ' + freshness_date
 
         file_size = item.get('gsa', {}).get('file_size')
         if file_size:

searx/engines/google_news.py

@@ -1,40 +1,24 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the Google News engine.
+"""This is the implementation of the google news engine.  The google news API
+ignores some parameters from the common :ref:`google API`:
 
-Google News has a different region handling compared to Google WEB.
+- num_ : the number of search results is ignored
-
-- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
-- the hl_ argument has to be set correctly (and different to Google WEB)
-- the gl_ argument is mandatory
-
-If one of this argument is not set correctly, the request is redirected to
-CONSENT dialog::
-
-  https://consent.google.com/m?continue=
-
-The google news API ignores some parameters from the common :ref:`google API`:
-
-- num_ : the number of search results is ignored / there is no paging all
-  results for a query term are in the first response.
 - save_ : is ignored / Google-News results are always *SafeSearch*
 
-.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
-.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
 .. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
 .. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
+
 """
 
-from typing import TYPE_CHECKING
+# pylint: disable=invalid-name
 
 import binascii
 import re
 from urllib.parse import urlencode
 from base64 import b64decode
 from lxml import html
-import babel
 
-from searx import locales
 from searx.utils import (
     eval_xpath,
     eval_xpath_list,
@@ -42,19 +26,18 @@ from searx.utils import (
     extract_text,
 )
 
-from searx.engines.google import fetch_traits as _fetch_traits  # pylint: disable=unused-import
+# pylint: disable=unused-import
 from searx.engines.google import (
-    get_google_info,
+    supported_languages_url,
+    _fetch_supported_languages,
+)
+
+# pylint: enable=unused-import
+
+from searx.engines.google import (
+    get_lang_info,
     detect_google_sorry,
 )
-from searx.enginelib.traits import EngineTraits
-
-if TYPE_CHECKING:
-    import logging
-
-    logger: logging.Logger
-
-traits: EngineTraits
 
 # about
 about = {
@@ -66,77 +49,70 @@ about = {
     "results": 'HTML',
 }
 
+# compared to other google engines google-news has a different time range
+# support.  The time range is included in the search term.
+time_range_dict = {
+    'day': 'when:1d',
+    'week': 'when:7d',
+    'month': 'when:1m',
+    'year': 'when:1y',
+}
+
 # engine dependent config
+
 categories = ['news']
 paging = False
-time_range_support = False
+use_locale_domain = True
+time_range_support = True
 
 # Google-News results are always *SafeSearch*. Option 'safesearch' is set to
 # False here, otherwise checker will report safesearch-errors::
 #
 #  safesearch : results are identitical for safesearch=0 and safesearch=2
-safesearch = True
+safesearch = False
-# send_accept_language_header = True
+send_accept_language_header = True
 
 
 def request(query, params):
     """Google-News search request"""
 
-    sxng_locale = params.get('searxng_locale', 'en-US')
+    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
-    ceid = locales.get_engine_locale(sxng_locale, traits.custom['ceid'], default='US:en')
-    google_info = get_google_info(params, traits)
-    google_info['subdomain'] = 'news.google.com'  # google news has only one domain
 
-    ceid_region, ceid_lang = ceid.split(':')
+    # google news has only one domain
-    ceid_lang, ceid_suffix = (
+    lang_info['subdomain'] = 'news.google.com'
-        ceid_lang.split('-')
-        + [
-            None,
-        ]
-    )[:2]
 
-    google_info['params']['hl'] = ceid_lang
+    ceid = "%s:%s" % (lang_info['country'], lang_info['language'])
 
-    if ceid_suffix and ceid_suffix not in ['Hans', 'Hant']:
+    # google news redirects en to en-US
+    if lang_info['params']['hl'] == 'en':
+        lang_info['params']['hl'] = 'en-US'
 
-        if ceid_region.lower() == ceid_lang:
+    # Very special to google-news compared to other google engines, the time
-            google_info['params']['hl'] = ceid_lang + '-' + ceid_region
+    # range is included in the search term.
-        else:
+    if params['time_range']:
-            google_info['params']['hl'] = ceid_lang + '-' + ceid_suffix
+        query += ' ' + time_range_dict[params['time_range']]
-
-    elif ceid_region.lower() != ceid_lang:
-
-        if ceid_region in ['AT', 'BE', 'CH', 'IL', 'SA', 'IN', 'BD', 'PT']:
-            google_info['params']['hl'] = ceid_lang
-        else:
-            google_info['params']['hl'] = ceid_lang + '-' + ceid_region
-
-    google_info['params']['lr'] = 'lang_' + ceid_lang.split('-')[0]
-    google_info['params']['gl'] = ceid_region
 
     query_url = (
         'https://'
-        + google_info['subdomain']
+        + lang_info['subdomain']
-        + "/search?"
+        + '/search'
-        + urlencode(
+        + "?"
-            {
+        + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'gl': lang_info['country']})
-                'q': query,
-                **google_info['params'],
-            }
-        )
-        # ceid includes a ':' character which must not be urlencoded
         + ('&ceid=%s' % ceid)
-    )
+    )  # ceid includes a ':' character which must not be urlencoded
-
     params['url'] = query_url
-    params['cookies'] = google_info['cookies']
+
-    params['headers'].update(google_info['headers'])
+    params['cookies']['CONSENT'] = "YES+"
+    params['headers'].update(lang_info['headers'])
+    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+
     return params
 
 
 def response(resp):
     """Get response from google's search request"""
     results = []
+
     detect_google_sorry(resp)
 
     # convert the text to dom
@@ -176,8 +152,8 @@ def response(resp):
 
         # The pub_date is mostly a string like 'yesertday', not a real
         # timezone date or time.  Therefore we can't use publishedDate.
-        pub_date = extract_text(eval_xpath(result, './article//time'))
+        pub_date = extract_text(eval_xpath(result, './article/div[1]/div[1]/time'))
-        pub_origin = extract_text(eval_xpath(result, './article//a[@data-n-tid]'))
+        pub_origin = extract_text(eval_xpath(result, './article/div[1]/div[1]/a'))
 
         content = ' / '.join([x for x in [pub_origin, pub_date] if x])
 
@@ -198,127 +174,3 @@ def response(resp):
 
     # return results
     return results
-
-
-ceid_list = [
-    'AE:ar',
-    'AR:es-419',
-    'AT:de',
-    'AU:en',
-    'BD:bn',
-    'BE:fr',
-    'BE:nl',
-    'BG:bg',
-    'BR:pt-419',
-    'BW:en',
-    'CA:en',
-    'CA:fr',
-    'CH:de',
-    'CH:fr',
-    'CL:es-419',
-    'CN:zh-Hans',
-    'CO:es-419',
-    'CU:es-419',
-    'CZ:cs',
-    'DE:de',
-    'EG:ar',
-    'ES:es',
-    'ET:en',
-    'FR:fr',
-    'GB:en',
-    'GH:en',
-    'GR:el',
-    'HK:zh-Hant',
-    'HU:hu',
-    'ID:en',
-    'ID:id',
-    'IE:en',
-    'IL:en',
-    'IL:he',
-    'IN:bn',
-    'IN:en',
-    'IN:hi',
-    'IN:ml',
-    'IN:mr',
-    'IN:ta',
-    'IN:te',
-    'IT:it',
-    'JP:ja',
-    'KE:en',
-    'KR:ko',
-    'LB:ar',
-    'LT:lt',
-    'LV:en',
-    'LV:lv',
-    'MA:fr',
-    'MX:es-419',
-    'MY:en',
-    'NA:en',
-    'NG:en',
-    'NL:nl',
-    'NO:no',
-    'NZ:en',
-    'PE:es-419',
-    'PH:en',
-    'PK:en',
-    'PL:pl',
-    'PT:pt-150',
-    'RO:ro',
-    'RS:sr',
-    'RU:ru',
-    'SA:ar',
-    'SE:sv',
-    'SG:en',
-    'SI:sl',
-    'SK:sk',
-    'SN:fr',
-    'TH:th',
-    'TR:tr',
-    'TW:zh-Hant',
-    'TZ:en',
-    'UA:ru',
-    'UA:uk',
-    'UG:en',
-    'US:en',
-    'US:es-419',
-    'VE:es-419',
-    'VN:vi',
-    'ZA:en',
-    'ZW:en',
-]
-"""List of region/language combinations supported by Google News.  Values of the
-``ceid`` argument of the Google News REST API."""
-
-
-_skip_values = [
-    'ET:en',  # english (ethiopia)
-    'ID:en',  # english (indonesia)
-    'LV:en',  # english (latvia)
-]
-
-_ceid_locale_map = {'NO:no': 'nb-NO'}
-
-
-def fetch_traits(engine_traits: EngineTraits):
-    _fetch_traits(engine_traits, add_domains=False)
-
-    engine_traits.custom['ceid'] = {}
-
-    for ceid in ceid_list:
-        if ceid in _skip_values:
-            continue
-
-        region, lang = ceid.split(':')
-        x = lang.split('-')
-        if len(x) > 1:
-            if x[1] not in ['Hant', 'Hans']:
-                lang = x[0]
-
-        sxng_locale = _ceid_locale_map.get(ceid, lang + '-' + region)
-        try:
-            locale = babel.Locale.parse(sxng_locale, sep='-')
-        except babel.UnknownLocaleError:
-            print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
-            continue
-
-        engine_traits.custom['ceid'][locales.region_tag(locale)] = ceid

searx/engines/google_scholar.py

@@ -1,18 +1,19 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the Google Scholar engine.
+"""Google (Scholar)
 
-Compared to other Google services the Scholar engine has a simple GET REST-API
+For detailed description of the *REST-full* API see: `Query Parameter
-and there does not exists `async` API.  Even though the API slightly vintage we
+Definitions`_.
-can make use of the :ref:`google API` to assemble the arguments of the GET
+
-request.
+.. _Query Parameter Definitions:
+   https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
 """
 
-from typing import TYPE_CHECKING
+# pylint: disable=invalid-name
-from typing import Optional
 
 from urllib.parse import urlencode
 from datetime import datetime
+from typing import Optional
 from lxml import html
 
 from searx.utils import (
@@ -22,21 +23,19 @@ from searx.utils import (
     extract_text,
 )
 
-from searx.exceptions import SearxEngineCaptchaException
-
-from searx.engines.google import fetch_traits  # pylint: disable=unused-import
 from searx.engines.google import (
-    get_google_info,
+    get_lang_info,
     time_range_dict,
+    detect_google_sorry,
 )
-from searx.enginelib.traits import EngineTraits
 
-if TYPE_CHECKING:
+# pylint: disable=unused-import
-    import logging
+from searx.engines.google import (
+    supported_languages_url,
+    _fetch_supported_languages,
+)
 
-    logger: logging.Logger
+# pylint: enable=unused-import
-
-traits: EngineTraits
 
 # about
 about = {
@@ -52,62 +51,53 @@ about = {
 categories = ['science', 'scientific publications']
 paging = True
 language_support = True
+use_locale_domain = True
 time_range_support = True
 safesearch = False
 send_accept_language_header = True
 
 
-def time_range_args(params):
+def time_range_url(params):
-    """Returns a dictionary with a time range arguments based on
+    """Returns a URL query component for a google-Scholar time range based on
-    ``params['time_range']``.
+    ``params['time_range']``.  Google-Scholar does only support ranges in years.
-
+    To have any effect, all the Searx ranges (*day*, *week*, *month*, *year*)
-    Google Scholar supports a detailed search by year.  Searching by *last
+    are mapped to *year*.  If no range is set, an empty string is returned.
-    month* or *last week* (as offered by SearXNG) is uncommon for scientific
+    Example::
-    publications and is not supported by Google Scholar.
-
-    To limit the result list when the users selects a range, all the SearXNG
-    ranges (*day*, *week*, *month*, *year*) are mapped to *year*.  If no range
-    is set an empty dictionary of arguments is returned.  Example;  when
-    user selects a time range (current year minus one in 2022):
-
-    .. code:: python
-
-        { 'as_ylo' : 2021 }
 
+        &as_ylo=2019
     """
-    ret_val = {}
+    # as_ylo=2016&as_yhi=2019
+    ret_val = ''
     if params['time_range'] in time_range_dict:
-        ret_val['as_ylo'] = datetime.now().year - 1
+        ret_val = urlencode({'as_ylo': datetime.now().year - 1})
-    return ret_val
+    return '&' + ret_val
-
-
-def detect_google_captcha(dom):
-    """In case of CAPTCHA Google Scholar open its own *not a Robot* dialog and is
-    not redirected to ``sorry.google.com``.
-    """
-    if eval_xpath(dom, "//form[@id='gs_captcha_f']"):
-        raise SearxEngineCaptchaException()
 
 
 def request(query, params):
     """Google-Scholar search request"""
 
-    google_info = get_google_info(params, traits)
+    offset = (params['pageno'] - 1) * 10
+    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
+
     # subdomain is: scholar.google.xy
-    google_info['subdomain'] = google_info['subdomain'].replace("www.", "scholar.")
+    lang_info['subdomain'] = lang_info['subdomain'].replace("www.", "scholar.")
 
-    args = {
+    query_url = (
-        'q': query,
+        'https://'
-        **google_info['params'],
+        + lang_info['subdomain']
-        'start': (params['pageno'] - 1) * 10,
+        + '/scholar'
-        'as_sdt': '2007',  # include patents / to disable set '0,5'
+        + "?"
-        'as_vis': '0',  # include citations / to disable set '1'
+        + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'start': offset})
-    }
+    )
-    args.update(time_range_args(params))
 
-    params['url'] = 'https://' + google_info['subdomain'] + '/scholar?' + urlencode(args)
+    query_url += time_range_url(params)
-    params['cookies'] = google_info['cookies']
+    params['url'] = query_url
-    params['headers'].update(google_info['headers'])
+
+    params['cookies']['CONSENT'] = "YES+"
+    params['headers'].update(lang_info['headers'])
+    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+
+    # params['google_subdomain'] = subdomain
     return params
 
 
@@ -148,15 +138,19 @@ def parse_gs_a(text: Optional[str]):
 
 
 def response(resp):  # pylint: disable=too-many-locals
-    """Parse response from Google Scholar"""
+    """Get response from google's search request"""
     results = []
 
+    detect_google_sorry(resp)
+
+    # which subdomain ?
+    # subdomain = resp.search_params.get('google_subdomain')
+
     # convert the text to dom
     dom = html.fromstring(resp.text)
-    detect_google_captcha(dom)
 
     # parse results
-    for result in eval_xpath_list(dom, '//div[@data-rp]'):
+    for result in eval_xpath_list(dom, '//div[@data-cid]'):
 
         title = extract_text(eval_xpath(result, './/h3[1]//a'))
 
@@ -164,7 +158,7 @@ def response(resp):  # pylint: disable=too-many-locals
             # this is a [ZITATION] block
             continue
 
-        pub_type = extract_text(eval_xpath(result, './/span[@class="gs_ctg2"]'))
+        pub_type = extract_text(eval_xpath(result, './/span[@class="gs_ct1"]'))
         if pub_type:
             pub_type = pub_type[1:-1].lower()
 

searx/engines/google_videos.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""This is the implementation of the Google Videos engine.
+"""This is the implementation of the google videos engine.
 
 .. admonition:: Content-Security-Policy (CSP)
 
@@ -14,8 +14,9 @@
 
 """
 
-from typing import TYPE_CHECKING
+# pylint: disable=invalid-name
 
+import re
 from urllib.parse import urlencode
 from lxml import html
 
@@ -26,22 +27,20 @@ from searx.utils import (
     extract_text,
 )
 
-from searx.engines.google import fetch_traits  # pylint: disable=unused-import
 from searx.engines.google import (
-    get_google_info,
+    get_lang_info,
     time_range_dict,
     filter_mapping,
+    g_section_with_header,
+    title_xpath,
     suggestion_xpath,
     detect_google_sorry,
 )
-from searx.enginelib.traits import EngineTraits
 
-if TYPE_CHECKING:
+# pylint: disable=unused-import
-    import logging
+from searx.engines.google import supported_languages_url, _fetch_supported_languages
 
-    logger: logging.Logger
+# pylint: enable=unused-import
-
-traits: EngineTraits
 
 # about
 about = {
@@ -56,32 +55,70 @@ about = {
 # engine dependent config
 
 categories = ['videos', 'web']
-paging = True
+paging = False
 language_support = True
+use_locale_domain = True
 time_range_support = True
 safesearch = True
+send_accept_language_header = True
+
+RE_CACHE = {}
+
+
+def _re(regexpr):
+    """returns compiled regular expression"""
+    RE_CACHE[regexpr] = RE_CACHE.get(regexpr, re.compile(regexpr))
+    return RE_CACHE[regexpr]
+
+
+def scrap_out_thumbs_src(dom):
+    ret_val = {}
+    thumb_name = 'dimg_'
+    for script in eval_xpath_list(dom, '//script[contains(., "google.ldi={")]'):
+        _script = script.text
+        # "dimg_35":"https://i.ytimg.c....",
+        _dimurl = _re("s='([^']*)").findall(_script)
+        for k, v in _re('(' + thumb_name + '[0-9]*)":"(http[^"]*)').findall(_script):
+            v = v.replace(r'\u003d', '=')
+            v = v.replace(r'\u0026', '&')
+            ret_val[k] = v
+    logger.debug("found %s imgdata for: %s", thumb_name, ret_val.keys())
+    return ret_val
+
+
+def scrap_out_thumbs(dom):
+    """Scrap out thumbnail data from <script> tags."""
+    ret_val = {}
+    thumb_name = 'dimg_'
+
+    for script in eval_xpath_list(dom, '//script[contains(., "_setImagesSrc")]'):
+        _script = script.text
+
+        # var s='data:image/jpeg;base64, ...'
+        _imgdata = _re("s='([^']*)").findall(_script)
+        if not _imgdata:
+            continue
+
+        # var ii=['dimg_17']
+        for _vidthumb in _re(r"(%s\d+)" % thumb_name).findall(_script):
+            # At least the equal sign in the URL needs to be decoded
+            ret_val[_vidthumb] = _imgdata[0].replace(r"\x3d", "=")
+
+    logger.debug("found %s imgdata for: %s", thumb_name, ret_val.keys())
+    return ret_val
 
 
 def request(query, params):
     """Google-Video search request"""
 
-    google_info = get_google_info(params, traits)
+    lang_info = get_lang_info(params, supported_languages, language_aliases, False)
 
     query_url = (
         'https://'
-        + google_info['subdomain']
+        + lang_info['subdomain']
         + '/search'
         + "?"
-        + urlencode(
+        + urlencode({'q': query, 'tbm': "vid", **lang_info['params'], 'ie': "utf8", 'oe': "utf8"})
-            {
-                'q': query,
-                'tbm': "vid",
-                'start': 10 * params['pageno'],
-                **google_info['params'],
-                'asearch': 'arc',
-                'async': 'use_ac:true,_fmt:html',
-            }
-        )
     )
 
     if params['time_range'] in time_range_dict:
@@ -90,8 +127,9 @@ def request(query, params):
         query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
     params['url'] = query_url
 
-    params['cookies'] = google_info['cookies']
+    params['cookies']['CONSENT'] = "YES+"
-    params['headers'].update(google_info['headers'])
+    params['headers'].update(lang_info['headers'])
+    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
     return params
 
 
@@ -103,30 +141,43 @@ def response(resp):
 
     # convert the text to dom
     dom = html.fromstring(resp.text)
+    vidthumb_imgdata = scrap_out_thumbs(dom)
+    thumbs_src = scrap_out_thumbs_src(dom)
+    logger.debug(str(thumbs_src))
 
     # parse results
     for result in eval_xpath_list(dom, '//div[contains(@class, "g ")]'):
 
-        img_src = eval_xpath_getindex(result, './/img/@src', 0, None)
+        # ignore google *sections*
-        if img_src is None:
+        if extract_text(eval_xpath(result, g_section_with_header)):
+            logger.debug("ignoring <g-section-with-header>")
             continue
 
-        title = extract_text(eval_xpath_getindex(result, './/a/h3[1]', 0))
+        # ingnore articles without an image id / e.g. news articles
-        url = eval_xpath_getindex(result, './/a/h3[1]/../@href', 0)
+        img_id = eval_xpath_getindex(result, './/g-img/img/@id', 0, default=None)
+        if img_id is None:
+            logger.error("no img_id found in item %s (news article?)", len(results) + 1)
+            continue
 
+        img_src = vidthumb_imgdata.get(img_id, None)
+        if not img_src:
+            img_src = thumbs_src.get(img_id, "")
+
+        title = extract_text(eval_xpath_getindex(result, title_xpath, 0))
+        url = eval_xpath_getindex(result, './/div[@class="dXiKIc"]//a/@href', 0)
+        length = extract_text(eval_xpath(result, './/div[contains(@class, "P7xzyf")]/span/span'))
         c_node = eval_xpath_getindex(result, './/div[@class="Uroaid"]', 0)
         content = extract_text(c_node)
-        pub_info = extract_text(eval_xpath(result, './/div[@class="P7xzyf"]'))
+        pub_info = extract_text(eval_xpath(result, './/div[@class="Zg1NU"]'))
-        length = extract_text(eval_xpath(result, './/div[@class="J1mWY"]'))
 
         results.append(
             {
                 'url': url,
                 'title': title,
                 'content': content,
+                'length': length,
                 'author': pub_info,
                 'thumbnail': img_src,
-                'length': length,
                 'template': 'videos.html',
             }
         )