# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring, invalid-name
import lxml.etree
from lxml import html
from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
from searx import utils
from tests import SearxTestCase
class TestUtils(SearxTestCase): # pylint: disable=missing-class-docstring
def test_gen_useragent(self):
self.assertIsInstance(utils.gen_useragent(), str)
self.assertIsNotNone(utils.gen_useragent())
self.assertTrue(utils.gen_useragent().startswith('Mozilla'))
def test_searx_useragent(self):
self.assertIsInstance(utils.searx_useragent(), str)
self.assertIsNotNone(utils.searx_useragent())
self.assertTrue(utils.searx_useragent().startswith('searx'))
def test_html_to_text(self):
html_str = """
Test text
"""
self.assertIsInstance(utils.html_to_text(html_str), str)
self.assertIsNotNone(utils.html_to_text(html_str))
self.assertEqual(utils.html_to_text(html_str), "Test text")
self.assertEqual(utils.html_to_text(r"regexp: (?
Test text
"""
dom = html.fromstring(html_str)
self.assertEqual(utils.extract_text(dom), 'Test text')
self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')
self.assertEqual(utils.extract_text(dom.xpath('//span/text()')), 'Test text')
self.assertEqual(utils.extract_text(dom.xpath('count(//span)')), '3.0')
self.assertEqual(utils.extract_text(dom.xpath('boolean(//span)')), 'True')
self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')
self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')
self.assertEqual(utils.extract_text(None, allow_none=True), None)
with self.assertRaises(ValueError):
utils.extract_text(None)
with self.assertRaises(ValueError):
utils.extract_text({})
def test_extract_url(self):
def f(html_str, search_url):
return utils.extract_url(html.fromstring(html_str), search_url)
self.assertEqual(f('https://example.com', 'http://example.com/'), 'https://example.com/')
self.assertEqual(f('https://example.com', 'http://example.com/'), 'https://example.com/')
self.assertEqual(f('//example.com', 'http://example.com/'), 'http://example.com/')
self.assertEqual(f('//example.com', 'https://example.com/'), 'https://example.com/')
self.assertEqual(f('/path?a=1', 'https://example.com'), 'https://example.com/path?a=1')
with self.assertRaises(lxml.etree.ParserError):
f('', 'https://example.com')
with self.assertRaises(Exception):
utils.extract_url([], 'https://example.com')
def test_html_to_text_invalid(self):
_html = '
Lorem ipsumdolor sit amet
' self.assertEqual(utils.html_to_text(_html), "Lorem ipsum") def test_ecma_unscape(self): self.assertEqual(utils.ecma_unescape('text%20with%20space'), 'text with space') self.assertEqual(utils.ecma_unescape('text using %xx: %F3'), 'text using %xx: ó') self.assertEqual(utils.ecma_unescape('text using %u: %u5409, %u4E16%u754c'), 'text using %u: 吉, 世界') class TestHTMLTextExtractor(SearxTestCase): # pylint: disable=missing-class-docstring def setUp(self): self.html_text_extractor = utils._HTMLTextExtractor() # pylint: disable=protected-access def test__init__(self): self.assertEqual(self.html_text_extractor.result, []) def test_handle_charref(self): self.html_text_extractor.handle_charref('xF') self.assertIn('\x0f', self.html_text_extractor.result) self.html_text_extractor.handle_charref('XF') self.assertIn('\x0f', self.html_text_extractor.result) self.html_text_extractor.handle_charref('97') self.assertIn('a', self.html_text_extractor.result) def test_handle_entityref(self): entity = 'test' self.html_text_extractor.handle_entityref(entity) self.assertIn(entity, self.html_text_extractor.result) def test_invalid_html(self): text = 'Lorem ipsumdolor sit amet
' with self.assertRaises(utils._HTMLTextExtractorException): # pylint: disable=protected-access self.html_text_extractor.feed(text) class TestXPathUtils(SearxTestCase): # pylint: disable=missing-class-docstring TEST_DOC = """