From f0102a95c96605cb0702a374f084f82074698cf2 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 18 Jan 2022 11:05:45 +0100 Subject: [PATCH 1/2] [fix] google engine: remove adds and fix mobile_ui selector 1. Fix issue reported in comment [1] 2. Fix XPath selector for the response of google's mobile UI, reported in comment [2] [1] https://github.com/searxng/searxng/pull/777#issuecomment-1015121322 [2] https://github.com/searxng/searxng/pull/777#issuecomment-1015236238 Signed-off-by: Markus Heiser --- searx/engines/google.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/searx/engines/google.py b/searx/engines/google.py index 5ee00cca3..b681da17c 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -112,7 +112,8 @@ filter_mapping = {0: 'off', 1: 'medium', 2: 'high'} # ------------------------ # google results are grouped into
-results_xpath = '//div[contains(@class, "g")]' +results_xpath = '//div[@id="search"]//div[contains(@class, "g ")]' +results_xpath_mobile_ui = '//div[contains(@class, "g ")]' # google *sections* are no usual *results*, we ignore them g_section_with_header = './g-section-with-header' @@ -336,7 +337,12 @@ def response(resp): logger.error(e, exc_info=True) # parse results - for result in eval_xpath_list(dom, results_xpath): + + _results_xpath = results_xpath + if use_mobile_ui: + _results_xpath = results_xpath_mobile_ui + + for result in eval_xpath_list(dom, _results_xpath): # google *sections* if extract_text(eval_xpath(result, g_section_with_header)): From 1a0760c10a54584c82987c5cb22d1c83cbc4252f Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 18 Jan 2022 13:23:35 +0100 Subject: [PATCH 2/2] [fix] googel engine - "some results are invalids: invalid content" Fix google issues listet in the `/stats?engine=google` and message:: some results are invalids: invalid content The log is:: DEBUG searx : result: invalid content: {'url': 'https://de.wikipedia.org/wiki/Foo', 'title': 'Foo - Wikipedia', 'content': None, 'engine': 'google'} WARNING searx.engines.google : ErrorContext('searx/search/processors/abstract.py', 111, 'result_container.extend(self.engine_name, search_results)', None, 'some results are invalids: invalid content', ()) True Signed-off-by: Markus Heiser --- searx/engines/google.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/searx/engines/google.py b/searx/engines/google.py index b681da17c..13d270113 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -353,20 +353,22 @@ def response(resp): title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None) if title_tag is None: # this not one of the common google results *section* - logger.debug('ingoring
section: missing title') + logger.debug('ingoring item from the result_xpath list: missing title') continue title = extract_text(title_tag) url = eval_xpath_getindex(result, href_xpath, 0, None) if url is None: continue content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True) + if content is None: + logger.debug('ingoring item from the result_xpath list: missing content of title "%s"', title) + continue + + logger.debug('add link to results: %s', title) results.append({'url': url, 'title': title, 'content': content}) + except Exception as e: # pylint: disable=broad-except logger.error(e, exc_info=True) - # from lxml import etree - # logger.debug(etree.tostring(result, pretty_print=True)) - # import pdb - # pdb.set_trace() continue # parse suggestion