searxng/searx/engines/bing.py

# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""Bing (Web)

- https://github.com/searx/searx/issues/2019#issuecomment-648227442
"""

import re
from urllib.parse import urlencode, urlparse, parse_qs
from lxml import html
import babel
from searx.utils import eval_xpath, extract_text

about = {
    "website": 'https://www.bing.com',
    "wikidata_id": 'Q182496',
    "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-web-search-api',
    "use_official_api": False,
    "require_api_key": False,
    "results": 'HTML',
}

# engine dependent config
categories = ['general', 'web']
paging = True
supported_languages_url = 'https://www.bing.com/account/general'
language_aliases = {}

time_range_support = True
time_range_dict = {'day': 'ex1:"ez1"', 'week': 'ex1:"ez2"', 'month': 'ex1:"ez3"'}

safesearch = True
safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'}  # cookie: ADLT=STRICT

# search-url
base_url = 'https://www.bing.com/'

# initial query:     https://www.bing.com/search?q=foo&search=&form=QBLH
inital_query = 'search?{query}&search=&form=QBLH'

# following queries: https://www.bing.com/search?q=foo&search=&first=11&FORM=PERE
page_query = 'search?{query}&search=&first={offset}&FORM=PERE'


def _get_offset_from_pageno(pageno):
    return (pageno - 1) * 10 + 1


def request(query, params):
    """Assemble a bing request.

    Bing tries to guess user's language and territory from the HTTP
    Accept-Language.  Optional the user can select a language by adding a
    language tag to the search term.  By example if the user searches for the
    word 'foo' in articles written in english::

      language:en foo

    Bing supports only language tags, the user can't add a territory
    (e.g. 'en-US') or a script (e.g. 'zh- Hans') to the selected language.

    """
    language = params['language']
    if language == 'all':
        language = 'en-US'
    locale = babel.Locale.parse(language, sep='-')

    # query and paging

    bing_language = ''
    if locale.language in supported_languages:
        bing_language = 'language:%s ' % locale.language
    query_str = urlencode({'q': bing_language + query})

    if params['pageno'] == 1:
        search_path = inital_query.format(query=query_str)
    else:
        offset = _get_offset_from_pageno(params.get('pageno', 1))
        search_path = page_query.format(query=query_str, offset=offset)
        referer = base_url + inital_query.format(query=query_str)
        params['headers']['Referer'] = referer
        logger.debug("headers.Referer --> %s", referer)

    params['url'] = base_url + search_path

    # cookies

    # On bing users can disable SafeSearch but this does not have an effect in
    # the bing search results (except you switch to bing-videos or bing-images)

    # SRCHHPGUSR = [
    #     # 'SRCHLANG=%s' % locale.language,
    #     'ADLT=%s' % safesearch_types.get(params['safesearch'], 'OFF')
    # ]
    # params['cookies']['SRCHHPGUSR'] = '&'.join(SRCHHPGUSR) + ';'
    # logger.debug("cookies SRCHHPGUSR=%s", params['cookies']['SRCHHPGUSR'])

    # time range

    time_range = time_range_dict.get(params['time_range'])
    if time_range:
        params['url'] += "&filters=" + time_range

    # language & locale

    # language of the UI ...
    # if locale.language in supported_languages:
    #     params['url'] += '&setlang=%s' % locale.language

    ac_lang = locale.language
    if locale.territory:
        ac_lang = "%s-%s,%s;q=0.5" % (locale.language, locale.territory, locale.language)
    logger.debug("headers.Accept-Language --> %s", ac_lang)
    params['headers']['Accept-Language'] = ac_lang
    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'

    return params


def response(resp):

    results = []
    result_len = 0

    dom = html.fromstring(resp.text)

    for result in eval_xpath(dom, '//div[@class="sa_cc"]'):

        # IMO //div[@class="sa_cc"] does no longer match
        logger.debug('found //div[@class="sa_cc"] --> %s', result)

        link = eval_xpath(result, './/h3/a')[0]
        url = link.attrib.get('href')
        title = extract_text(link)
        content = extract_text(eval_xpath(result, './/p'))

        # append result
        results.append({'url': url, 'title': title, 'content': content})

    # parse results again if nothing is found yet
    for result in eval_xpath(dom, '//li[@class="b_algo"]'):

        link = eval_xpath(result, './/h2/a')[0]
        url = link.attrib.get('href')
        title = extract_text(link)
        content = extract_text(eval_xpath(result, './/p'))

        # append result
        results.append({'url': url, 'title': title, 'content': content})

    try:
        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
        if "-" in result_len_container:

            # Remove the part "from-to" for paginated request ...
            result_len_container = result_len_container[result_len_container.find("-") * 2 + 2 :]

        result_len_container = re.sub('[^0-9]', '', result_len_container)

        if len(result_len_container) > 0:
            result_len = int(result_len_container)

    except Exception as e:  # pylint: disable=broad-except
        logger.debug('result error :\n%s', e)

    if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
        return []

    results.append({'number_of_results': result_len})
    return results


# get supported languages from their site
def _fetch_supported_languages(resp):

    lang_tags = set()

    dom = html.fromstring(resp.text)

    # Selector to get items from "Display language"
    ui_lang_links = eval_xpath(dom, '//div[@id="language-section"]//li')

    for _li in ui_lang_links:

        href = eval_xpath(_li, './/@href')[0]
        (_scheme, _netloc, _path, _params, query, _fragment) = urlparse(href)
        query = parse_qs(query, keep_blank_values=True)

        # The 'language:xx' query string in the request function (above) does
        # only support the 2 letter language codes from the "Display languages"
        # list.  Examples of items from the "Display languages" not sopported in
        # the query string:
        #    'mn-Cyrl-MN', 'chr-cher', 'zh-Hans', ha-latn, 'ca-es-valencia'

        setlang = query.get('setlang', [None, ])[0]
        lang = setlang.split('-')[0]
        lang_tags.add(lang)

    return list(lang_tags)