zaclys/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng synced 2024-01-01 19:24:07 +01:00
Code Issues Projects Releases Packages Wiki Activity

[mod] activate pyright checks (in CI)

Browse source 
We have been using a static type checker (pyright) for a long time, but its
check was not yet a prerequisite for passing the quality gate.  It was checked
in the CI, but the error messages were only logged.

As is always the case in life, with checks that you have to do but which have no
consequences; you neglect them :-)

We didn't activate the checks back then because we (even today) have too much
monkey patching in our code (not only in the engines, httpx and others objects
are also affected).

We want to replace monkey patching with clear interfaces for a long time, the
basis for this is increased typing and we can only achieve this if we make type
checking an integral part of the quality gate.

  This PR activates the type check; in order to pass the check, a few typings
  were corrected in the code, but most type inconsistencies were deactivated via
  inline comments.

This was particularly necessary in places where the code uses properties that
stick to the objects (monkey patching).  The sticking of properties only happens
in a few places, but the access to these properties extends over the entire
code, which is why there are many `# type: ignore` markers in the code ... which
we will hopefully be able to remove again successively in the future.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2024-03-11 18:17:56 +01:00
parent 648f43be1d
commit 86b4d2f2d0
100 changed files with 236 additions and 426 deletions
Download patch file Download diff file Expand all files Collapse all files

2
searx/engines/1337x.py
View file

@ -41,7 +41,7 @@ def response(resp):
seed = extract_text(eval_xpath(result, './/td[contains(@class, "seeds")]'))
leech = extract_text(eval_xpath(result, './/td[contains(@class, "leeches")]'))
filesize_info = extract_text(eval_xpath(result, './/td[contains(@class, "size")]/text()'))
filesize, filesize_multiplier = filesize_info.split()
filesize, filesize_multiplier = filesize_info.split() # type: ignore
filesize = get_torrent_size(filesize, filesize_multiplier)
results.append(

18
searx/engines/__builtins__.pyi Normal file
View file

@ -0,0 +1,18 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring
# Ugly hack to avoid errors from pyright when checking the engiens / sadly this
# *bultins* are now available in all modules !?!
#
# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md
import searx
import searx.enginelib.traits
logger = searx.logger
traits = searx.enginelib.traits.EngineTraits()
supported_languages = None
language_aliases = None
categories = []
del searx

6
searx/engines/__init__.py
View file

@ -14,16 +14,14 @@ import sys
import copy
from os.path import realpath, dirname
from typing import TYPE_CHECKING, Dict
from typing import Dict
import types
import inspect
from searx import logger, settings
from searx.enginelib import Engine
from searx.utils import load_module
if TYPE_CHECKING:
from searx.enginelib import Engine
logger = logger.getChild('engines')
ENGINE_DIR = dirname(realpath(__file__))
ENGINE_DEFAULT_ARGS = {

2
searx/engines/apkmirror.py
View file

@ -51,7 +51,7 @@ def response(resp):
link = eval_xpath_getindex(result, './/h5/a', 0)
url = base_url + link.attrib.get('href') + '#downloads'
url = base_url + link.attrib.get('href') + '#downloads' # type: ignore
title = extract_text(link)
img_src = base_url + eval_xpath_getindex(result, './/img/@src', 0)
res = {'url': url, 'title': title, 'img_src': img_src}

8
searx/engines/archlinux.py
View file

@ -8,7 +8,6 @@ Arch Wiki blocks access to it.
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode, urljoin, urlparse
import lxml
import babel
@ -17,13 +16,6 @@ from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
from searx.enginelib.traits import EngineTraits
from searx.locales import language_tag
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://wiki.archlinux.org/',

2
searx/engines/arxiv.py
View file

@ -5,7 +5,7 @@
from datetime import datetime
from lxml import etree
from lxml import etree # type: ignore
from lxml.etree import XPath
from searx.utils import eval_xpath, eval_xpath_list, eval_xpath_getindex

2
searx/engines/ask.py
View file

@ -2,7 +2,7 @@
"""Ask.com"""
from urllib.parse import urlencode
import dateutil
import dateutil.parser
from lxml import html
from searx import utils

2
searx/engines/base.py
View file

@ -6,7 +6,7 @@ from datetime import datetime
import re
from urllib.parse import urlencode
from lxml import etree
from lxml import etree # type: ignore
from searx.utils import searx_useragent
# about

7
searx/engines/bing.py
View file

@ -26,7 +26,6 @@ category for the Chinese market.
"""
# pylint: disable=too-many-branches, invalid-name
from typing import TYPE_CHECKING
import base64
import re
import time
@ -39,12 +38,6 @@ from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_ge
from searx.locales import language_tag, region_tag
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
about = {
"website": 'https://www.bing.com',

6
searx/engines/bing_images.py
View file

@ -4,7 +4,6 @@
# pylint: disable=invalid-name
from typing import TYPE_CHECKING
import json
from urllib.parse import urlencode
@ -15,11 +14,6 @@ from searx.engines.bing import set_bing_cookies
from searx.engines.bing import fetch_traits # pylint: disable=unused-import
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
# about

8
searx/engines/bing_news.py
View file

@ -9,7 +9,6 @@
# pylint: disable=invalid-name
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from lxml import html
@ -18,13 +17,6 @@ from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_ge
from searx.enginelib.traits import EngineTraits
from searx.engines.bing import set_bing_cookies
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {

9
searx/engines/bing_videos.py
View file

@ -3,24 +3,15 @@
"""Bing-Videos: description see :py:obj:`searx.engines.bing`.
"""
from typing import TYPE_CHECKING
import json
from urllib.parse import urlencode
from lxml import html
from searx.enginelib.traits import EngineTraits
from searx.engines.bing import set_bing_cookies
from searx.engines.bing import fetch_traits # pylint: disable=unused-import
from searx.engines.bing_images import time_map
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://www.bing.com/videos',

13
searx/engines/brave.py
View file

@ -118,7 +118,7 @@ Implementations
"""
from typing import Any, TYPE_CHECKING
from typing import Any
from urllib.parse import (
urlencode,
@ -139,13 +139,6 @@ from searx.utils import (
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://search.brave.com/',
"wikidata_id": 'Q22906900',
@ -228,10 +221,10 @@ def request(query, params):
params['cookies']['useLocation'] = '0'
params['cookies']['summarizer'] = '0'
engine_region = traits.get_region(params['searxng_locale'], 'all')
engine_region = traits.get_region(params['searxng_locale'], 'all') # type: ignore
params['cookies']['country'] = engine_region.split('-')[-1].lower() # type: ignore
ui_lang = locales.get_engine_locale(params['searxng_locale'], traits.custom["ui_lang"], 'en-us')
ui_lang = locales.get_engine_locale(params['searxng_locale'], traits.custom["ui_lang"], 'en-us') # type: ignore
params['cookies']['ui_lang'] = ui_lang
logger.debug("cookies %s", params['cookies'])

2
searx/engines/bt4g.py
View file

@ -40,7 +40,7 @@ import re
from datetime import datetime
from urllib.parse import quote
from lxml import etree
from lxml import etree # type: ignore
from searx.utils import get_torrent_size

2
searx/engines/btdigg.py
View file

@ -56,7 +56,7 @@ def response(resp):
content = html.tostring(excerpt, encoding='unicode', method='text', with_tail=False)
# it is better to emit <br/> instead of |, but html tags are verboten
content = content.strip().replace('\n', ' | ')
content = ' '.join(content.split())
content = ' '.join(content.split()) # type: ignore
filesize = result.xpath('.//span[@class="torrent_size"]/text()')[0].split()[0]
filesize_multiplier = result.xpath('.//span[@class="torrent_size"]/text()')[0].split()[1]

9
searx/engines/dailymotion.py
View file

@ -10,8 +10,6 @@ Dailymotion (Videos)
"""
from typing import TYPE_CHECKING
from datetime import datetime, timedelta
from urllib.parse import urlencode
import time
@ -23,14 +21,7 @@ from searx.exceptions import SearxEngineAPIException
from searx.locales import region_tag, language_tag
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://www.dailymotion.com',
"wikidata_id": 'Q769222',

4
searx/engines/destatis.py
View file

@ -57,10 +57,10 @@ def response(resp):
results.append(
{
'url': base_url + "/" + extract_text(eval_xpath(result, url_xpath)),
'url': base_url + "/" + extract_text(eval_xpath(result, url_xpath)), # type: ignore
'title': extract_text(eval_xpath(result, title_xpath)),
'content': extract_text(eval_xpath(result, content_xpath)),
'metadata': ', '.join(metadata),
'metadata': ', '.join(metadata), # type: ignore
}
)

12
searx/engines/duckduckgo.py
View file

@ -4,7 +4,6 @@ DuckDuckGo Lite
~~~~~~~~~~~~~~~
"""
from typing import TYPE_CHECKING
import re
from urllib.parse import urlencode
import json
@ -25,13 +24,6 @@ from searx.network import get # see https://github.com/searxng/searxng/issues/7
from searx import redisdb
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://lite.duckduckgo.com/lite/',
"wikidata_id": 'Q12805',
@ -110,7 +102,7 @@ def get_vqd(query):
key = 'SearXNG_ddg_web_vqd' + redislib.secret_hash(query)
value = c.get(key)
if value or value == b'':
value = value.decode('utf-8')
value = value.decode('utf-8') # type: ignore
logger.debug("re-use cached vqd value: %s", value)
return value
@ -129,7 +121,7 @@ def get_vqd(query):
return value
def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default: str = 'en_US') -> str:
"""Get DuckDuckGo's language identifier from SearXNG's locale.
DuckDuckGo defines its languages by region codes (see

7
searx/engines/duckduckgo_definitions.py
View file

@ -13,8 +13,6 @@ most of the features are based on English terms.
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode, urlparse, urljoin
from lxml import html
@ -22,12 +20,7 @@ from searx.data import WIKIDATA_UNITS
from searx.utils import extract_text, html_to_text, get_string_replaces_function
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
if TYPE_CHECKING:
import logging
logger: logging.Logger
# about
about = {
"website": 'https://duckduckgo.com/',
"wikidata_id": 'Q12805',

9
searx/engines/duckduckgo_extra.py
View file

@ -5,7 +5,6 @@ DuckDuckGo Extra (images, videos, news)
"""
from datetime import datetime
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
@ -13,16 +12,8 @@ from searx.engines.duckduckgo import (
get_ddg_lang,
get_vqd,
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://duckduckgo.com/',
"wikidata_id": 'Q12805',

10
searx/engines/duckduckgo_weather.py
View file

@ -4,7 +4,6 @@ DuckDuckGo Weather
~~~~~~~~~~~~~~~~~~
"""
from typing import TYPE_CHECKING
from json import loads
from urllib.parse import quote
@ -13,15 +12,6 @@ from flask_babel import gettext
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
from searx.engines.duckduckgo import get_ddg_lang
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://duckduckgo.com/',

7
searx/engines/flickr_noapi.py
View file

@ -3,20 +3,13 @@
"""
from typing import TYPE_CHECKING
import json
from time import time
import re
from urllib.parse import urlencode
from searx.utils import ecma_unescape, html_to_text
if TYPE_CHECKING:
import logging
logger: logging.Logger
# about
about = {
"website": 'https://www.flickr.com',
"wikidata_id": 'Q103204',

2
searx/engines/goodreads.py
View file

@ -46,7 +46,7 @@ def response(resp):
for result in eval_xpath_list(dom, results_xpath):
results.append(
{
'url': base_url + extract_text(eval_xpath(result, url_xpath)),
'url': base_url + extract_text(eval_xpath(result, url_xpath)), # type: ignore
'title': extract_text(eval_xpath(result, title_xpath)),
'img_src': extract_text(eval_xpath(result, thumbnail_xpath)),
'content': extract_text(eval_xpath(result, info_text_xpath)),

10
searx/engines/google.py
View file

@ -11,8 +11,6 @@ engines:
"""
from typing import TYPE_CHECKING
import re
from urllib.parse import urlencode
from lxml import html
@ -26,14 +24,6 @@ from searx.network import get # see https://github.com/searxng/searxng/issues/7
from searx.exceptions import SearxEngineCaptchaException
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://www.google.com',

10
searx/engines/google_images.py
View file

@ -13,8 +13,6 @@ This internal API offer results in
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from json import loads
@ -25,15 +23,7 @@ from searx.engines.google import (
detect_google_sorry,
)
if TYPE_CHECKING:
import logging
from searx.enginelib.traits import EngineTraits
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://images.google.com',
"wikidata_id": 'Q521550',

11
searx/engines/google_news.py
View file

@ -24,8 +24,6 @@ The google news API ignores some parameters from the common :ref:`google API`:
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
import base64
from lxml import html
@ -46,13 +44,6 @@ from searx.engines.google import (
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://news.google.com',
@ -301,4 +292,4 @@ def fetch_traits(engine_traits: EngineTraits):
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
continue
engine_traits.custom['ceid'][locales.region_tag(locale)] = ceid
engine_traits.custom['ceid'][locales.region_tag(locale)] = ceid # type: ignore

9
searx/engines/google_scholar.py
View file

@ -7,7 +7,6 @@ can make use of the :ref:`google API` to assemble the arguments of the GET
request.
"""
from typing import TYPE_CHECKING
from typing import Optional
from urllib.parse import urlencode
@ -28,14 +27,6 @@ from searx.engines.google import (
get_google_info,
time_range_dict,
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {

10
searx/engines/google_videos.py
View file

@ -13,8 +13,6 @@
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from lxml import html
@ -33,14 +31,6 @@ from searx.engines.google import (
suggestion_xpath,
detect_google_sorry,
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {

2
searx/engines/imgur.py
View file

@ -55,7 +55,7 @@ def response(resp):
results.append(
{
'template': 'images.html',
'url': base_url + extract_text(eval_xpath(result, url_xpath)),
'url': base_url + extract_text(eval_xpath(result, url_xpath)), # type: ignore
'title': extract_text(eval_xpath(result, title_xpath)),
'img_src': img_src,
'thumbnail_src': thumbnail_src,

6
searx/engines/ina.py
View file

@ -1,6 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""
INA (Videos)
"""INA (Videos)
"""
from html import unescape
@ -58,7 +58,7 @@ def response(resp):
thumbnail = extract_text(eval_xpath(result, thumbnail_xpath))
content = extract_text(eval_xpath(result, publishedDate_xpath)) + extract_text(
eval_xpath(result, content_xpath)
)
) # type: ignore
# append result
results.append(

2
searx/engines/json_engine.py
View file

@ -17,7 +17,7 @@ from urllib.parse import urlencode
from searx.utils import to_string, html_to_text
search_url = None
search_url: str = ''
url_query = None
url_prefix = ""
content_query = None

9
searx/engines/mediawiki.py
View file

@ -33,22 +33,13 @@ Implementations
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from datetime import datetime
from urllib.parse import urlencode, quote
from searx.utils import html_to_text
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": None,
"wikidata_id": None,

11
searx/engines/mullvad_leta.py
View file

@ -20,7 +20,6 @@ Otherwise, follow instructions provided by Mullvad for enabling the VPN on Linux
update of SearXNG!
"""
from typing import TYPE_CHECKING
from httpx import Response
from lxml import html
from searx.enginelib.traits import EngineTraits
@ -28,18 +27,10 @@ from searx.locales import region_tag, get_official_locales
from searx.utils import eval_xpath, extract_text, eval_xpath_list
from searx.exceptions import SearxEngineResponseException
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
use_cache: bool = True # non-cache use only has 100 searches per day!
search_url = "https://leta.mullvad.net"
# about
about = {
"website": search_url,
"wikidata_id": 'Q47008412', # the Mullvad id - not leta, but related
@ -145,7 +136,7 @@ def fetch_traits(engine_traits: EngineTraits):
if not isinstance(resp, Response):
print("ERROR: failed to get response from mullvad-leta. Are you connected to the VPN?")
return
if not resp.ok:
if not resp.ok: # type: ignore
print("ERROR: response from mullvad-leta is not OK. Are you connected to the VPN?")
return
dom = html.fromstring(resp.text)

6
searx/engines/odysee.py
View file

@ -14,8 +14,6 @@ from searx.network import get
from searx.locales import language_tag
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
# Engine metadata
about = {
"website": "https://odysee.com/",
@ -122,11 +120,11 @@ def fetch_traits(engine_traits: EngineTraits):
timeout=60,
)
if not resp.ok:
if not resp.ok: # type: ignore
print("ERROR: can't determine languages from Odysee")
return
for line in resp.text.split("\n")[1:-4]:
for line in resp.text.split("\n")[1:-4]: # type: ignore
lang_tag = line.strip().split(": ")[0].replace("'", "")
try:

2
searx/engines/peertube.py
View file

@ -17,8 +17,6 @@ from searx.locales import language_tag
from searx.utils import html_to_text
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
about = {
# pylint: disable=line-too-long
"website": 'https://joinpeertube.org',

12
searx/engines/pkg_go_dev.py
View file

@ -5,7 +5,7 @@ import re
from urllib.parse import urlencode
from dateutil import parser
import babel
import babel.numbers
import flask_babel
from lxml import html
from searx.utils import eval_xpath, eval_xpath_list, extract_text
@ -69,14 +69,18 @@ def response(resp):
results.append(
{
'template': 'packages.html',
'url': base_url + extract_text(eval_xpath(result, url_xpath)),
'url': base_url + extract_text(eval_xpath(result, url_xpath)), # type: ignore
'title': extract_text(eval_xpath(result, title_xpath)),
'content': extract_text(eval_xpath(result, content_xpath)),
'package_name': re.sub(r"\(|\)", "", extract_text(eval_xpath(result, package_name_xpath))),
'package_name': re.sub(
r"\(|\)",
"",
extract_text(eval_xpath(result, package_name_xpath)),
), # type: ignore
'version': extract_text(eval_xpath(result, version_xpath)),
'popularity': popularity,
'license_name': extract_text(eval_xpath(result, license_name_xpath)),
'license_url': base_url + extract_text(eval_xpath(result, license_url_xpath)),
'license_url': base_url + extract_text(eval_xpath(result, license_url_xpath)), # type: ignore
'publishedDate': publishedDate,
}
)

2
searx/engines/postgresql.py
View file

@ -63,7 +63,7 @@ def search(query, params):
query_params = {'query': query}
query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
with _connection:
with _connection: # type: ignore
with _connection.cursor() as cur:
cur.execute(query_to_run, query_params)
return _fetch_results(cur)

2
searx/engines/presearch.py
View file

@ -128,7 +128,7 @@ def _get_request_id(query, params):
# performs an IP-based geolocation of the user, we don't want that in
# SearXNG ;-)
if l.territory:
if l and l.territory:
headers['Accept-Language'] = f"{l.language}-{l.territory},{l.language};" "q=0.9,*;" "q=0.5"
resp_text = get(url, headers=headers).text # type: ignore

8
searx/engines/pubmed.py
View file

@ -6,7 +6,7 @@
from datetime import datetime
from urllib.parse import urlencode
from lxml import etree
from lxml import etree # type: ignore
from searx.network import get
from searx.utils import (
eval_xpath_getindex,
@ -77,8 +77,8 @@ def response(resp): # pylint: disable=too-many-locals
for entry in eval_xpath_list(search_results, '//PubmedArticle'):
medline = eval_xpath_getindex(entry, './MedlineCitation', 0)
title = eval_xpath_getindex(medline, './/Article/ArticleTitle', 0).text
pmid = eval_xpath_getindex(medline, './/PMID', 0).text
title = eval_xpath_getindex(medline, './/Article/ArticleTitle', 0).text # type: ignore
pmid = eval_xpath_getindex(medline, './/PMID', 0).text # type: ignore
url = pubmed_url + pmid
content = extract_text(
eval_xpath_getindex(medline, './/Abstract/AbstractText//text()', 0, default=None), allow_none=True
@ -120,7 +120,7 @@ def response(resp): # pylint: disable=too-many-locals
day = eval_xpath_getindex(accepted_date, './Day', 0)
try:
publishedDate = datetime.strptime(
year.text + '-' + month.text + '-' + day.text,
year.text + '-' + month.text + '-' + day.text, # type: ignore
'%Y-%m-%d',
)
res_dict['publishedDate'] = publishedDate

4
searx/engines/qwant.py
View file

@ -47,7 +47,7 @@ from json import loads
from urllib.parse import urlencode
from flask_babel import gettext
import babel
import lxml
import lxml.html
from searx.exceptions import SearxEngineAPIException, SearxEngineTooManyRequestsException
from searx.network import raise_for_httperror
@ -59,8 +59,6 @@ from searx.utils import (
extract_text,
)
traits: EngineTraits
# about
about = {
"website": 'https://www.qwant.com/',

4
searx/engines/recoll.py
View file

@ -66,9 +66,9 @@ paging = True
time_range_support = True
# parameters from settings.yml
base_url = None
base_url: str = ''
search_dir = ''
mount_prefix = None
mount_prefix: str = ''
dl_prefix = None
# embedded

6
searx/engines/redis_server.py
View file

@ -69,7 +69,7 @@ def search(query, _params):
ret = _redis_client.hgetall(query)
if ret:
ret['template'] = result_template
ret['template'] = result_template # type: ignore
return [ret]
if ' ' in query:
@ -98,7 +98,7 @@ def search_keys(query):
res = dict(enumerate(_redis_client.lrange(key, 0, -1)))
if res:
res['template'] = result_template
res['redis_key'] = key
res['template'] = result_template # type: ignore
res['redis_key'] = key # type: ignore
ret.append(res)
return ret

2
searx/engines/rumble.py
View file

@ -55,7 +55,7 @@ def response(resp):
return []
for result_dom in results_dom:
url = base_url + extract_text(result_dom.xpath(url_xpath))
url = base_url + extract_text(result_dom.xpath(url_xpath)) # type: ignore
thumbnail = extract_text(result_dom.xpath(thumbnail_xpath))
title = extract_text(result_dom.xpath(title_xpath))
p_date = extract_text(result_dom.xpath(published_date))

10
searx/engines/sepiasearch.py
View file

@ -5,8 +5,6 @@ peertube engines.
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from datetime import datetime
@ -17,14 +15,6 @@ from searx.engines.peertube import (
safesearch_table,
time_range_table,
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
# pylint: disable=line-too-long

4
searx/engines/soundcloud.py
View file

@ -44,7 +44,7 @@ guest_client_id = ''
def get_client_id():
resp = http_get("https://soundcloud.com")
if resp.ok:
if resp.ok: # type: ignore
tree = html.fromstring(resp.content)
# script_tags has been moved from /assets/app/ to /assets/ path. I
# found client_id in https://a-v2.sndcdn.com/assets/49-a0c01933-3.js
@ -55,7 +55,7 @@ def get_client_id():
for app_js_url in app_js_urls[::-1]:
# gets app_js and searches for the clientid
resp = http_get(app_js_url)
if resp.ok:
if resp.ok: # type: ignore
cids = cid_re.search(resp.content.decode())
if cids is not None and len(cids.groups()):
return cids.groups()[0]

8
searx/engines/startpage.py
View file

@ -79,7 +79,6 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
"""
from typing import TYPE_CHECKING
from collections import OrderedDict
import re
from unicodedata import normalize, combining
@ -96,14 +95,7 @@ from searx.exceptions import SearxEngineCaptchaException
from searx.locales import region_tag
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://startpage.com',
"wikidata_id": 'Q2333295',

2
searx/engines/svgrepo.py
View file

@ -36,7 +36,7 @@ def response(resp):
results.append(
{
'template': 'images.html',
'url': base_url + extract_text(eval_xpath(result, url_xpath)),
'url': base_url + extract_text(eval_xpath(result, url_xpath)), # type: ignore
'title': extract_text(eval_xpath(result, title_xpath)).replace(" SVG File", "").replace("Show ", ""),
'img_src': extract_text(eval_xpath(result, img_src_xpath)),
}

5
searx/engines/tagesschau.py
View file

@ -15,16 +15,11 @@ This SearXNG engine uses the `/api2u/search`_ API.
.. _OpenAPI: https://swagger.io/specification/
"""
from typing import TYPE_CHECKING
from datetime import datetime
from urllib.parse import urlencode
import re
if TYPE_CHECKING:
import logging
logger: logging.Logger
about = {
'website': "https://tagesschau.de",

2
searx/engines/tineye.py
View file

@ -166,7 +166,7 @@ def response(resp):
message = 'HTTP status: %s' % resp.status_code
error = json_data.get('error')
s_key = json_data.get('suggestions', {}).get('key', '')
s_key = json_data.get('suggestions', {}).get('key', '') # type: ignore
if error and s_key:
message = "%s (%s)" % (error, s_key)

9
searx/engines/torznab.py
View file

@ -48,21 +48,16 @@ Implementations
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import List, Dict, Any
from datetime import datetime
from urllib.parse import quote
from lxml import etree # type: ignore
import httpx
from searx.exceptions import SearxEngineAPIException
if TYPE_CHECKING:
import httpx
import logging
logger: logging.Logger
# engine settings
about: Dict[str, Any] = {
"website": None,

19
searx/engines/wikidata.py
View file

@ -5,7 +5,6 @@ from :ref:`wikipedia engine`.
"""
# pylint: disable=missing-class-docstring
from typing import TYPE_CHECKING
from hashlib import md5
from urllib.parse import urlencode, unquote
from json import loads
@ -23,14 +22,6 @@ from searx.engines.wikipedia import (
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://wikidata.org/',
"wikidata_id": 'Q2013',
@ -142,7 +133,7 @@ def get_headers():
return {'Accept': 'application/sparql-results+json', 'User-Agent': searx_useragent()}
def get_label_for_entity(entity_id, language):
def get_label_for_entity(entity_id, language): # type: ignore
name = WIKIDATA_PROPERTIES.get(entity_id)
if name is None:
name = WIKIDATA_PROPERTIES.get((entity_id, language))
@ -497,7 +488,7 @@ class WDAttribute:
def __init__(self, name):
self.name = name
def get_select(self):
def get_select(self) -> str:
return '(group_concat(distinct ?{name};separator=", ") as ?{name}s)'.replace('{name}', self.name)
def get_label(self, language):
@ -506,10 +497,10 @@ class WDAttribute:
def get_where(self):
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace('{name}', self.name)
def get_wikibase_label(self):
def get_wikibase_label(self) -> str:
return ""
def get_group_by(self):
def get_group_by(self) -> str:
return ""
def get_str(self, result, language): # pylint: disable=unused-argument
@ -702,7 +693,7 @@ class WDDateAttribute(WDAttribute):
# precision: minute
return (
get_datetime_format(format, locale=locale)
.replace("'", "")
.replace("'", "") # type: ignore
.replace('{0}', format_time(timestamp, 'full', tzinfo=None, locale=locale))
.replace('{1}', format_date(timestamp, 'short', locale=locale))
)

4
searx/engines/wikipedia.py
View file

@ -64,8 +64,6 @@ from searx import network as _network
from searx import locales
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
# about
about = {
"website": 'https://www.wikipedia.org/',
@ -277,7 +275,7 @@ def fetch_wikimedia_traits(engine_traits: EngineTraits):
engine_traits.regions[sxng_tag] = eng_tag
resp = _network.get(list_of_wikipedias)
if not resp.ok:
if not resp.ok: # type: ignore
print("ERROR: response from Wikipedia is not OK.")
dom = html.fromstring(resp.text)

2
searx/engines/wolframalpha_api.py
View file

@ -5,7 +5,7 @@
from urllib.parse import urlencode
from lxml import etree
from lxml import etree # type: ignore
# about
about = {

2
searx/engines/www1x.py
View file

@ -4,7 +4,7 @@
"""
from urllib.parse import urlencode, urljoin
from lxml import html, etree
from lxml import html, etree # type: ignore
from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

8
searx/engines/xpath.py
View file

@ -73,7 +73,7 @@ from lxml import html
from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
from searx.network import raise_for_httperror
search_url = None
search_url = ''
"""
Search URL of the engine. Example::
@ -270,7 +270,9 @@ def response(resp): # pylint: disable=too-many-branches
# add alternative cached url if available
if cached_xpath:
tmp_result['cached_url'] = cached_url + extract_text(eval_xpath_list(result, cached_xpath, min_len=1))
tmp_result['cached_url'] = cached_url + extract_text(
eval_xpath_list(result, cached_xpath, min_len=1)
) # type: ignore
if is_onion:
tmp_result['is_onion'] = True
@ -290,7 +292,7 @@ def response(resp): # pylint: disable=too-many-branches
'url': url,
'title': title,
'content': content,
'cached_url': cached_url + cached,
'cached_url': cached_url + cached, # type: ignore
'is_onion': is_onion,
}
)

8
searx/engines/yahoo.py
View file

@ -19,8 +19,6 @@ from searx.utils import (
)
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
# about
about = {
"website": 'https://search.yahoo.com/',
@ -86,7 +84,7 @@ def request(query, params):
'p': query,
'ei': 'UTF-8',
'fl': 1,
'vl': 'lang_' + lang,
'vl': 'lang_' + lang, # type: ignore
'btf': btf,
'fr2': 'time',
'age': age,
@ -95,7 +93,7 @@ def request(query, params):
}
)
domain = lang2domain.get(lang, '%s.search.yahoo.com' % lang)
domain = lang2domain.get(lang, '%s.search.yahoo.com' % lang) # type: ignore
params['url'] = 'https://%s/search?%s' % (domain, args)
return params
@ -158,7 +156,7 @@ def fetch_traits(engine_traits: EngineTraits):
engine_traits.all_locale = 'any'
resp = network.get('https://search.yahoo.com/preferences/languages')
if not resp.ok:
if not resp.ok: # type: ignore
print("ERROR: response from yahoo is not OK.")
dom = html.fromstring(resp.text)

2
searx/engines/yahoo_news.py
View file

@ -82,7 +82,7 @@ def response(resp):
item = {'url': url, 'title': title, 'content': content, 'img_src': img_src}
pub_date = extract_text(result.xpath('.//span[contains(@class,"s-time")]'))
ago = AGO_RE.search(pub_date)
ago = AGO_RE.search(pub_date) # type: ignore
if ago:
number = int(ago.group(1))
delta = AGO_TIMEDELTA[ago.group(2)]

10
searx/engines/zlibrary.py
View file

@ -32,11 +32,13 @@ Implementations
===============
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import List, Dict, Any, Optional
from datetime import datetime
from urllib.parse import quote
import httpx
from lxml import html
from flask_babel import gettext
@ -44,13 +46,7 @@ from searx.utils import extract_text, eval_xpath, eval_xpath_list
from searx.enginelib.traits import EngineTraits
from searx.data import ENGINE_TRAITS
if TYPE_CHECKING:
import httpx
import logging
logger: logging.Logger
# about
about: Dict[str, Any] = {
"website": "https://zlibrary-global.se",
"wikidata_id": "Q104863992",