diff --git a/docs/index.rst b/docs/index.rst index 7100d4ed1..f9800f95b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,7 +33,6 @@ If you don't trust anyone, you can set up your own, see :ref:`installation`. user/index admin/index dev/index - searxng_extra/index utils/index src/index diff --git a/docs/searxng_extra/index.rst b/docs/searxng_extra/index.rst deleted file mode 100644 index a98c358f2..000000000 --- a/docs/searxng_extra/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _searxng_extra: - -====================================================== -Tooling box ``searxng_extra`` for developers and users -====================================================== - -In the folder :origin:`searxng_extra/` we maintain some tools useful for -developers and users. - -.. toctree:: - :maxdepth: 2 - :caption: Contents - - standalone_searx.py diff --git a/docs/searxng_extra/standalone_searx.py.rst b/docs/searxng_extra/standalone_searx.py.rst deleted file mode 100644 index 7cbbccfde..000000000 --- a/docs/searxng_extra/standalone_searx.py.rst +++ /dev/null @@ -1,9 +0,0 @@ - -.. _standalone_searx.py: - -===================================== -``searxng_extra/standalone_searx.py`` -===================================== - -.. automodule:: searxng_extra.standalone_searx - :members: diff --git a/searxng_extra/standalone_searx.py b/searxng_extra/standalone_searx.py deleted file mode 100755 index e243d9cb1..000000000 --- a/searxng_extra/standalone_searx.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -# lint: pylint - -# SPDX-License-Identifier: AGPL-3.0-or-later -# (C) Copyright Contributors to the SearXNG project. -# (C) Copyright Contributors to the searx project (2014 - 2021) - -"""Script to run SearXNG from terminal. - -Getting categories without initiate the engine will only return `['general']` - ->>> import searx.engines -... list(searx.engines.categories.keys()) -['general'] ->>> import searx.search -... searx.search.initialize() -... list(searx.engines.categories.keys()) -['general', 'it', 'science', 'images', 'news', 'videos', 'music', 'files', 'social media', 'map'] - -Example to use this script: - -.. code:: bash - - $ python3 searxng_extra/standalone_searx.py rain - -Example to run it from python: - ->>> import importlib -... import json -... import sys -... import searx.engines -... import searx.search -... search_query = 'rain' -... # initialize engines -... searx.search.initialize() -... # load engines categories once instead of each time the function called -... engine_cs = list(searx.engines.categories.keys()) -... # load module -... spec = importlib.util.spec_from_file_location( -... 'utils.standalone_searx', 'searxng_extra/standalone_searx.py') -... sas = importlib.util.module_from_spec(spec) -... spec.loader.exec_module(sas) -... # use function from module -... prog_args = sas.parse_argument([search_query], category_choices=engine_cs) -... search_q = sas.get_search_query(prog_args, engine_categories=engine_cs) -... res_dict = sas.to_dict(search_q) -... sys.stdout.write(json.dumps( -... res_dict, sort_keys=True, indent=4, ensure_ascii=False, -... default=sas.json_serial)) -{ - "answers": [], - "infoboxes": [ {...} ], - "paging": true, - "results": [... ], - "results_number": 820000000.0, - "search": { - "lang": "all", - "pageno": 1, - "q": "rain", - "safesearch": 0, - "timerange": null - }, - "suggestions": [...] -} -""" # pylint: disable=line-too-long - -import argparse -import sys -from datetime import datetime -from json import dumps -from typing import Any, Dict, List, Optional - -import searx -import searx.preferences -import searx.query -import searx.search -import searx.webadapter - -EngineCategoriesVar = Optional[List[str]] - - -def get_search_query( - args: argparse.Namespace, engine_categories: EngineCategoriesVar = None -) -> searx.search.SearchQuery: - """Get search results for the query""" - if engine_categories is None: - engine_categories = list(searx.engines.categories.keys()) - try: - category = args.category.decode('utf-8') - except AttributeError: - category = args.category - form = { - "q": args.query, - "categories": category, - "pageno": str(args.pageno), - "language": args.lang, - "time_range": args.timerange - } - preferences = searx.preferences.Preferences( - ['oscar'], engine_categories, searx.engines.engines, []) - preferences.key_value_settings['safesearch'].parse(args.safesearch) - - search_query = searx.webadapter.get_search_query_from_webapp( - preferences, form)[0] - return search_query - - -def no_parsed_url(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Remove parsed url from dict.""" - for result in results: - del result['parsed_url'] - return results - - -def json_serial(obj: Any) -> Any: - """JSON serializer for objects not serializable by default json code. - - :raise TypeError: raised when **obj** is not serializable - """ - if isinstance(obj, datetime): - serial = obj.isoformat() - return serial - if isinstance(obj, bytes): - return obj.decode('utf8') - if isinstance(obj, set): - return list(obj) - raise TypeError("Type ({}) not serializable".format(type(obj))) - - -def to_dict(search_query: searx.search.SearchQuery) -> Dict[str, Any]: - """Get result from parsed arguments.""" - result_container = searx.search.Search(search_query).search() - result_container_json = { - "search": { - "q": search_query.query, - "pageno": search_query.pageno, - "lang": search_query.lang, - "safesearch": search_query.safesearch, - "timerange": search_query.time_range, - }, - "results": no_parsed_url(result_container.get_ordered_results()), - "infoboxes": result_container.infoboxes, - "suggestions": list(result_container.suggestions), - "answers": list(result_container.answers), - "paging": result_container.paging, - "results_number": result_container.results_number() - } - return result_container_json - - -def parse_argument( - args: Optional[List[str]]=None, - category_choices: EngineCategoriesVar=None -) -> argparse.Namespace: - """Parse command line. - - :raise SystemExit: Query argument required on `args` - - Examples: - - >>> import importlib - ... # load module - ... spec = importlib.util.spec_from_file_location( - ... 'utils.standalone_searx', 'utils/standalone_searx.py') - ... sas = importlib.util.module_from_spec(spec) - ... spec.loader.exec_module(sas) - ... sas.parse_argument() - usage: ptipython [-h] [--category [{general}]] [--lang [LANG]] [--pageno [PAGENO]] [--safesearch [{0,1,2}]] [--timerange [{day,week,month,year}]] - query - SystemExit: 2 - >>> sas.parse_argument(['rain']) - Namespace(category='general', lang='all', pageno=1, query='rain', safesearch='0', timerange=None) - """ # noqa: E501 - if not category_choices: - category_choices = list(searx.engines.categories.keys()) - parser = argparse.ArgumentParser(description='Standalone searx.') - parser.add_argument('query', type=str, - help='Text query') - parser.add_argument('--category', type=str, nargs='?', - choices=category_choices, - default='general', - help='Search category') - parser.add_argument('--lang', type=str, nargs='?', default='all', - help='Search language') - parser.add_argument('--pageno', type=int, nargs='?', default=1, - help='Page number starting from 1') - parser.add_argument( - '--safesearch', type=str, nargs='?', - choices=['0', '1', '2'], default='0', - help='Safe content filter from none to strict') - parser.add_argument( - '--timerange', type=str, - nargs='?', choices=['day', 'week', 'month', 'year'], - help='Filter by time range') - return parser.parse_args(args) - - -if __name__ == '__main__': - searx.search.initialize() - engine_cs = list(searx.engines.categories.keys()) - prog_args = parse_argument(category_choices=engine_cs) - search_q = get_search_query(prog_args, engine_categories=engine_cs) - res_dict = to_dict(search_q) - sys.stdout.write(dumps( - res_dict, sort_keys=True, indent=4, ensure_ascii=False, - default=json_serial)) diff --git a/tests/unit/test_standalone_searx.py b/tests/unit/test_standalone_searx.py deleted file mode 100644 index c81598160..000000000 --- a/tests/unit/test_standalone_searx.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- -"""Test utils/standalone_searx.py""" -import datetime -import io -import sys - -from mock import Mock, patch -from nose2.tools import params - -from searx.search import SearchQuery, EngineRef, initialize -from searxng_extra import standalone_searx as sas -from tests import SearxTestCase - - -class StandaloneSearx(SearxTestCase): - """Unit test for standalone_searx.""" - - @classmethod - def setUpClass(cls): - engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1'}] - - initialize(engine_list) - - def test_parse_argument_no_args(self): - """Test parse argument without args.""" - with patch.object(sys, 'argv', ['standalone_searx']), \ - self.assertRaises(SystemExit): - sys.stderr = io.StringIO() - sas.parse_argument() - sys.stdout = sys.__stderr__ - - def test_parse_argument_basic_args(self): - """Test parse argument with basic args.""" - query = 'red box' - exp_dict = { - 'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1, - 'safesearch': '0', 'timerange': None} - args = ['standalone_searx', query] - with patch.object(sys, 'argv', args): - res = sas.parse_argument() - self.assertEqual(exp_dict, vars(res)) - res2 = sas.parse_argument(args[1:]) - self.assertEqual(exp_dict, vars(res2)) - - def test_to_dict(self): - """test to_dict.""" - self.assertEqual( - sas.to_dict( - sas.get_search_query(sas.parse_argument(['red box']))), - { - 'search': { - 'q': 'red box', 'pageno': 1, 'lang': 'all', - 'safesearch': 0, 'timerange': None - }, - 'results': [], 'infoboxes': [], 'suggestions': [], - 'answers': [], 'paging': False, 'results_number': 0 - } - ) - - def test_to_dict_with_mock(self): - """test to dict.""" - with patch.object(sas.searx.search, 'Search') as mock_s: - m_search = mock_s().search() - m_sq = Mock() - self.assertEqual( - sas.to_dict(m_sq), - { - 'answers': [], - 'infoboxes': m_search.infoboxes, - 'paging': m_search.paging, - 'results': m_search.get_ordered_results(), - 'results_number': m_search.results_number(), - 'search': { - 'lang': m_sq.lang, - 'pageno': m_sq.pageno, - 'q': m_sq.query, - 'safesearch': m_sq.safesearch, - 'timerange': m_sq.time_range, - }, - 'suggestions': [] - } - ) - - def test_get_search_query(self): - """test get_search_query.""" - args = sas.parse_argument(['rain', ]) - search_q = sas.get_search_query(args) - self.assertTrue(search_q) - self.assertEqual(search_q, SearchQuery('rain', [EngineRef('engine1', 'general')], - 'all', 0, 1, None, None, None)) - - def test_no_parsed_url(self): - """test no_parsed_url func""" - self.assertEqual( - sas.no_parsed_url([{'parsed_url': 'http://example.com'}]), - [{}] - ) - - @params( - (datetime.datetime(2020, 1, 1), '2020-01-01T00:00:00'), - ('a'.encode('utf8'), 'a'), - (set([1]), [1]) - ) - def test_json_serial(self, arg, exp_res): - """test json_serial func""" - self.assertEqual(sas.json_serial(arg), exp_res) - - def test_json_serial_error(self): - """test error on json_serial.""" - with self.assertRaises(TypeError): - sas.json_serial('a')