From 8cd544b2a60fce62aedd03dd920d2410c71f2381 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 23 May 2021 10:56:29 +0200 Subject: [PATCH] [doc] add documentation about the XPath engine - pylint searx/engines/xpath.py - fix indentation of some long lines - add logging - add doc-strings Signed-off-by: Markus Heiser --- docs/dev/engine_overview.rst | 6 +- docs/dev/index.rst | 1 + docs/dev/xpath_engine.rst | 9 +++ searx/engines/xpath.py | 122 +++++++++++++++++++++++++++-------- 4 files changed, 110 insertions(+), 28 deletions(-) create mode 100644 docs/dev/xpath_engine.rst diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst index 42c205d9d..2a661ec40 100644 --- a/docs/dev/engine_overview.rst +++ b/docs/dev/engine_overview.rst @@ -43,7 +43,7 @@ argument type information categories list pages, in which the engine is working paging boolean support multible pages time_range_support boolean support search time range -engine_type str ``online`` by default, other possibles values are +engine_type str ``online`` by default, other possibles values are ``offline``, ``online_dictionnary``, ``online_currency`` ======================= =========== ======================================================== @@ -100,6 +100,8 @@ example code paging = True +.. _engine request: + making a request ================ @@ -198,6 +200,8 @@ example code return params +.. _engine results: + returned results ================ diff --git a/docs/dev/index.rst b/docs/dev/index.rst index ba0a25a9c..82e5d3e84 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -9,6 +9,7 @@ Developer documentation quickstart contribution_guide engine_overview + xpath_engine search_api plugins translation diff --git a/docs/dev/xpath_engine.rst b/docs/dev/xpath_engine.rst new file mode 100644 index 000000000..4c73763d1 --- /dev/null +++ b/docs/dev/xpath_engine.rst @@ -0,0 +1,9 @@ +.. _xpath_engine: + +================ +The XPath engine +================ + +.. automodule:: searx.engines.xpath + :members: + diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index b215c2ba1..920adf4c6 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -1,51 +1,106 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +# pylint: disable=missing-function-docstring +"""The XPath engine is a *generic* engine with which it is possible to configure +engines in the settings. + +Here is a simple example of a XPath engine configured in the +:ref:`settings engine` section, further read :ref:`engines-dev`. + +.. code:: yaml + + - name : bitbucket + engine : xpath + paging : True + search_url : https://bitbucket.org/repo/all/{pageno}?name={query} + url_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]/@href + title_xpath : //article[@class="repo-summary"]//a[@class="repo-link"] + content_xpath : //article[@class="repo-summary"]/p + +""" + +from urllib.parse import urlencode from lxml import html -from urllib.parse import urlencode from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list +from searx import logger + +logger = logger.getChild('XPath engine') search_url = None -url_xpath = None -content_xpath = None -title_xpath = None -thumbnail_xpath = False -paging = False -suggestion_xpath = '' +""" +Search URL of the engine, replacements are: + +``{query}``: + Search terms from user. + +``{pageno}``: + Page number if engine supports pagging :py:obj:`paging` + +""" + +soft_max_redirects = 0 +'''Maximum redirects, soft limit. Record an error but don't stop the engine''' + results_xpath = '' +'''XPath selector for the list of result items''' + +url_xpath = None +'''XPath selector of result's ``url``.''' + +content_xpath = None +'''XPath selector of result's ``content``.''' + +title_xpath = None +'''XPath selector of result's ``title``.''' + +thumbnail_xpath = False +'''XPath selector of result's ``img_src``.''' + +suggestion_xpath = '' +'''XPath selector of result's ``suggestion``.''' + cached_xpath = '' cached_url = '' -soft_max_redirects = 0 -# parameters for engines with paging support -# -# number of results on each page -# (only needed if the site requires not a page number, but an offset) +paging = False +'''Engine supports paging [True or False].''' + page_size = 1 -# number of the first page (usually 0 or 1) -first_page_num = 1 +'''Number of results on each page. Only needed if the site requires not a page +number, but an offset.''' +first_page_num = 1 +'''Number of the first page (usually 0 or 1).''' def request(query, params): + '''Build request parameters (see :ref:`engine request`). + + ''' query = urlencode({'q': query})[2:] - fp = {'query': query} + fargs = {'query': query} if paging and search_url.find('{pageno}') >= 0: - fp['pageno'] = (params['pageno'] - 1) * page_size + first_page_num + fargs['pageno'] = (params['pageno'] - 1) * page_size + first_page_num - params['url'] = search_url.format(**fp) + params['url'] = search_url.format(**fargs) params['query'] = query params['soft_max_redirects'] = soft_max_redirects + logger.debug("query_url --> %s", params['url']) return params - def response(resp): + '''Scrap *results* from the response (see :ref:`engine results`). + + ''' results = [] dom = html.fromstring(resp.text) - is_onion = True if 'onions' in categories else False # pylint: disable=undefined-variable + is_onion = 'onions' in categories # pylint: disable=undefined-variable if results_xpath: for result in eval_xpath_list(dom, results_xpath): + url = extract_url(eval_xpath_list(result, url_xpath, min_len=1), search_url) title = extract_text(eval_xpath_list(result, title_xpath, min_len=1)) content = extract_text(eval_xpath_list(result, content_xpath, min_len=1)) @@ -59,13 +114,16 @@ def response(resp): # add alternative cached url if available if cached_xpath: - tmp_result['cached_url'] = cached_url\ + tmp_result['cached_url'] = ( + cached_url + extract_text(eval_xpath_list(result, cached_xpath, min_len=1)) + ) if is_onion: tmp_result['is_onion'] = True results.append(tmp_result) + else: if cached_xpath: for url, title, content, cached in zip( @@ -75,8 +133,12 @@ def response(resp): map(extract_text, eval_xpath_list(dom, content_xpath)), map(extract_text, eval_xpath_list(dom, cached_xpath)) ): - results.append({'url': url, 'title': title, 'content': content, - 'cached_url': cached_url + cached, 'is_onion': is_onion}) + results.append({ + 'url': url, + 'title': title, + 'content': content, + 'cached_url': cached_url + cached, 'is_onion': is_onion + }) else: for url, title, content in zip( (extract_url(x, search_url) for @@ -84,10 +146,16 @@ def response(resp): map(extract_text, eval_xpath_list(dom, title_xpath)), map(extract_text, eval_xpath_list(dom, content_xpath)) ): - results.append({'url': url, 'title': title, 'content': content, 'is_onion': is_onion}) + results.append({ + 'url': url, + 'title': title, + 'content': content, + 'is_onion': is_onion + }) - if not suggestion_xpath: - return results - for suggestion in eval_xpath(dom, suggestion_xpath): - results.append({'suggestion': extract_text(suggestion)}) + if suggestion_xpath: + for suggestion in eval_xpath(dom, suggestion_xpath): + results.append({'suggestion': extract_text(suggestion)}) + + logger.debug("found %s results", len(results)) return results