From 4f90fb6a9236a9699e80c249153ba372bf308cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sp=C3=BChler=20Stefan?= Date: Mon, 2 Mar 2020 18:55:48 +0100 Subject: [PATCH 1/4] [Fix] Startpage ValueError on Spanish date format datetime.parser.parse() does not know the Spanish date format which leads to a ValueError. Fixes #1870 Traceback (most recent call last): File "/usr/local/searx/searx/search.py", line 160, in search_one_http_request_safe search_results = search_one_http_request(engine, query, request_params) File "/usr/local/searx/searx/search.py", line 97, in search_one_http_request return engine.response(response) File "/usr/local/searx/searx/engines/startpage.py", line 102, in response published_date = parser.parse(date_string, dayfirst=True) File "/usr/local/searx/searx-ve/lib/python3.6/site-packages/dateutil/parser/_parser.py", line 1358, in parse return DEFAULTPARSER.parse(timestr, **kwargs) File "/usr/local/searx/searx-ve/lib/python3.6/site-packages/dateutil/parser/_parser.py", line 649, in parse raise ValueError("Unknown string format:", timestr) ValueError: ('Unknown string format:', '24 Ene 2013') --- searx/engines/startpage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 76567396f..953734934 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -99,11 +99,14 @@ def response(resp): if re.match(r"^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]{2} [0-9]{4} \.\.\. ", content): date_pos = content.find('...') + 4 date_string = content[0:date_pos - 5] - published_date = parser.parse(date_string, dayfirst=True) - # fix content string content = content[date_pos:] + try: + published_date = parser.parse(date_string, dayfirst=True) + except ValueError: + pass + # check if search result starts with something like: "5 days ago ... " elif re.match(r"^[0-9]+ days? ago \.\.\. ", content): date_pos = content.find('...') + 4 From 3973951770146f35d52664e971b76026af15d9f9 Mon Sep 17 00:00:00 2001 From: Dalf Date: Wed, 8 Apr 2020 15:58:26 +0200 Subject: [PATCH 2/4] [mod] requirements.txt: upgrade dependency versions. --- requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index ea4a5a7a0..5d508d7ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -certifi==2019.3.9 +certifi==2020.4.5.1 babel==2.7.0 flask-babel==1.0.0 -flask==1.0.2 -idna==2.8 -jinja2==2.10.1 -lxml==4.3.3 +flask==1.1.2 +idna==2.9 +jinja2==2.11.1 +lxml==4.5.0 pygments==2.1.3 -pyopenssl==19.0.0 +pyopenssl==19.1.0 python-dateutil==2.8.0 -pyyaml==5.1 -requests[socks]==2.22.0 +pyyaml==5.3.1 +requests[socks]==2.23.0 From ba7c8d7b960c3a3f288db162a51b76a2a935a605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 15 Apr 2020 23:24:12 +0200 Subject: [PATCH 3/4] [fix] remove usage of request context where not available --- searx/webapp.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/searx/webapp.py b/searx/webapp.py index c910230ab..9d76d8441 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -56,6 +56,7 @@ from flask import ( from babel.support import Translations import flask_babel from flask_babel import Babel, gettext, format_date, format_decimal +from flask.ctx import has_request_context from flask.json import jsonify from searx import brand from searx import settings, searx_dir, searx_debug @@ -165,13 +166,11 @@ _flask_babel_get_translations = flask_babel.get_translations # monkey patch for flask_babel.get_translations def _get_translations(): - translation_locale = request.form.get('use-translation') - if translation_locale: + if has_request_context() and request.form.get('use-translation') == 'oc': babel_ext = flask_babel.current_app.extensions['babel'] - translation = Translations.load(next(babel_ext.translation_directories), 'oc') - else: - translation = _flask_babel_get_translations() - return translation + return Translations.load(next(babel_ext.translation_directories), 'oc') + + return _flask_babel_get_translations() flask_babel.get_translations = _get_translations From bce3830b8dd9322e4e93d0003aa86e03c03e2dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 17 Apr 2020 16:31:02 +0200 Subject: [PATCH 4/4] [fix] translate engine errors to Occitan when configured --- searx/results.py | 4 ++-- searx/search.py | 17 +++++------------ searx/webapp.py | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/searx/results.py b/searx/results.py index 3b1e4bd62..02ab9efb1 100644 --- a/searx/results.py +++ b/searx/results.py @@ -345,8 +345,8 @@ class ResultContainer(object): return 0 return resultnum_sum / len(self._number_of_results) - def add_unresponsive_engine(self, engine_error): - self.unresponsive_engines.add(engine_error) + def add_unresponsive_engine(self, engine_name, error_type, error_message=None): + self.unresponsive_engines.add((engine_name, error_type, error_message)) def add_timing(self, engine_name, engine_time, page_load_time): self.timings.append({ diff --git a/searx/search.py b/searx/search.py index 2dcc4c8f7..661add7ca 100644 --- a/searx/search.py +++ b/searx/search.py @@ -127,11 +127,7 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e)) except Exception as e: record_offline_engine_stats_on_error(engine, result_container, start_time) - - result_container.add_unresponsive_engine(( - engine_name, - u'{0}: {1}'.format(gettext('unexpected crash'), e), - )) + result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e)) logger.exception('engine {0} : exception : {1}'.format(engine_name, e)) @@ -186,24 +182,21 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont engine.stats['errors'] += 1 if (issubclass(e.__class__, requests.exceptions.Timeout)): - result_container.add_unresponsive_engine((engine_name, gettext('timeout'))) + result_container.add_unresponsive_engine(engine_name, 'timeout') # requests timeout (connect or read) logger.error("engine {0} : HTTP requests timeout" "(search duration : {1} s, timeout: {2} s) : {3}" .format(engine_name, engine_time, timeout_limit, e.__class__.__name__)) requests_exception = True elif (issubclass(e.__class__, requests.exceptions.RequestException)): - result_container.add_unresponsive_engine((engine_name, gettext('request exception'))) + result_container.add_unresponsive_engine(engine_name, 'request exception') # other requests exception logger.exception("engine {0} : requests exception" "(search duration : {1} s, timeout: {2} s) : {3}" .format(engine_name, engine_time, timeout_limit, e)) requests_exception = True else: - result_container.add_unresponsive_engine(( - engine_name, - u'{0}: {1}'.format(gettext('unexpected crash'), e), - )) + result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e)) # others errors logger.exception('engine {0} : exception : {1}'.format(engine_name, e)) @@ -238,7 +231,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim remaining_time = max(0.0, timeout_limit - (time() - start_time)) th.join(remaining_time) if th.isAlive(): - result_container.add_unresponsive_engine((th._engine_name, gettext('timeout'))) + result_container.add_unresponsive_engine(th._engine_name, 'timeout') logger.warning('engine timeout: {0}'.format(th._engine_name)) diff --git a/searx/webapp.py b/searx/webapp.py index 9d76d8441..8c3531069 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -626,7 +626,7 @@ def index(): 'corrections': list(result_container.corrections), 'infoboxes': result_container.infoboxes, 'suggestions': list(result_container.suggestions), - 'unresponsive_engines': list(result_container.unresponsive_engines)}, + 'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)}, # noqa default=lambda item: list(item) if isinstance(item, set) else item), mimetype='application/json') elif output_format == 'csv': @@ -694,7 +694,7 @@ def index(): corrections=correction_urls, infoboxes=result_container.infoboxes, paging=result_container.paging, - unresponsive_engines=result_container.unresponsive_engines, + unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines), current_language=match_language(search_query.lang, LANGUAGE_CODES, fallback=request.preferences.get_value("language")), @@ -705,6 +705,16 @@ def index(): ) +def __get_translated_errors(unresponsive_engines): + translated_errors = [] + for unresponsive_engine in unresponsive_engines: + error_msg = gettext(unresponsive_engine[1]) + if unresponsive_engine[2]: + error_msg = "{} {}".format(error_msg, unresponsive_engine[2]) + translated_errors.append((unresponsive_engine[0], error_msg)) + return translated_errors + + @app.route('/about', methods=['GET']) def about(): """Render about page"""