From b428ccc5a0710a8b9c958e9b8687a2fac3435ebc Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 26 Sep 2023 12:59:56 +0200 Subject: [PATCH] [fix] engine brave - fetch traits (modified settings menu) Brave has changed it settings menu fundamental. Region codes are no longer in the HTML page, we have to read the regional codes from a JS: https://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.js Signed-off-by: Markus Heiser --- searx/engines/brave.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/searx/engines/brave.py b/searx/engines/brave.py index c713ffe50..5e5579e74 100644 --- a/searx/engines/brave.py +++ b/searx/engines/brave.py @@ -97,7 +97,6 @@ Implementations from typing import TYPE_CHECKING -import re from urllib.parse import ( urlencode, urlparse, @@ -354,7 +353,7 @@ def fetch_traits(engine_traits: EngineTraits): """Fetch :ref:`languages ` and :ref:`regions ` from Brave.""" - # pylint: disable=import-outside-toplevel + # pylint: disable=import-outside-toplevel, too-many-branches import babel.languages from searx.locales import region_tag, language_tag @@ -397,19 +396,26 @@ def fetch_traits(engine_traits: EngineTraits): # search regions of brave - engine_traits.all_locale = 'all' + resp = get('https://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.js', headers=headers) - for country in dom.xpath('//div[@id="sidebar"]//ul/li/div[contains(@class, "country")]'): + if not resp.ok: # type: ignore + print("ERROR: response from Brave is not OK.") - flag = country.xpath('./span[contains(@class, "flag")]')[0] - # country_name = extract_text(flag.xpath('./following-sibling::*')[0]) - country_tag = re.search(r'flag-([^\s]*)\s', flag.xpath('./@class')[0]).group(1) # type: ignore + country_js = resp.text[resp.text.index("options:{all") + len('options:') :] + country_js = country_js[: country_js.index("},k={default")] + country_tags = js_variable_to_python(country_js) + + for k, v in country_tags.items(): + if k == 'all': + engine_traits.all_locale = 'all' + continue + country_tag = v['value'] # add official languages of the country .. for lang_tag in babel.languages.get_official_languages(country_tag, de_facto=True): lang_tag = lang_map.get(lang_tag, lang_tag) sxng_tag = region_tag(babel.Locale.parse('%s_%s' % (lang_tag, country_tag.upper()))) - # print("%-20s: %s <-- %s" % (country_name, country_tag, sxng_tag)) + # print("%-20s: %s <-- %s" % (v['label'], country_tag, sxng_tag)) conflict = engine_traits.regions.get(sxng_tag) if conflict: