Sync upstream

searx/engines/yahoo.py

@@ -16,6 +16,7 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
+    html_to_text,
 )
 from searx.enginelib.traits import EngineTraits
 
@@ -133,12 +134,20 @@ def response(resp):
         url = parse_url(url)
 
         title = eval_xpath_getindex(result, './/h3//a/@aria-label', 0, default='')
-        title = extract_text(title)
+        title: str = extract_text(title)
         content = eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0, default='')
-        content = extract_text(content, allow_none=True)
+        content: str = extract_text(content, allow_none=True)
 
         # append result
-        results.append({'url': url, 'title': title, 'content': content})
+        results.append(
+            {
+                'url': url,
+                # title sometimes contains HTML tags / see
+                # https://github.com/searxng/searxng/issues/3790
+                'title': " ".join(html_to_text(title).strip().split()),
+                'content': " ".join(html_to_text(content).strip().split()),
+            }
+        )
 
     for suggestion in eval_xpath_list(dom, '//div[contains(@class, "AlsoTry")]//table//a'):
         # append suggestion

searx/plugins/calculator.py

@@ -4,17 +4,21 @@
 
 import ast
 import operator
+from multiprocessing import Process, Queue
 
 from flask_babel import gettext
-from searx import settings
+
+from searx.plugins import logger
 
 name = "Basic Calculator"
 description = gettext("Calculate mathematical expressions via the search bar")
-default_on = False
+default_on = True
 
 preference_section = 'general'
 plugin_id = 'calculator'
 
+logger = logger.getChild(plugin_id)
+
 operators = {
     ast.Add: operator.add,
     ast.Sub: operator.sub,
@@ -51,10 +55,31 @@ def _eval(node):
     raise TypeError(node)
 
 
+def timeout_func(timeout, func, *args, **kwargs):
+
+    def handler(q: Queue, func, args, **kwargs):  # pylint:disable=invalid-name
+        try:
+            q.put(func(*args, **kwargs))
+        except:
+            q.put(None)
+            raise
+
+    que = Queue()
+    p = Process(target=handler, args=(que, func, args), kwargs=kwargs)
+    p.start()
+    p.join(timeout=timeout)
+    ret_val = None
+    if not p.is_alive():
+        ret_val = que.get()
+    else:
+        logger.debug("terminate function after timeout is exceeded")
+        p.terminate()
+    p.join()
+    p.close()
+    return ret_val
+
+
 def post_search(_request, search):
-    # don't run on public instances due to possible attack surfaces
-    if settings['server']['public_instance']:
-        return True
 
     # only show the result of the expression on the first page
     if search.search_query.pageno > 1:
@@ -74,15 +99,13 @@ def post_search(_request, search):
 
     # in python, powers are calculated via **
     query_py_formatted = query.replace("^", "**")
-    try:
-        result = str(_eval_expr(query_py_formatted))
-        if result != query:
-            search.result_container.answers['calculate'] = {'answer': f"{query} = {result}"}
-    except (TypeError, SyntaxError, ArithmeticError):
-        pass
 
+    # Prevent the runtime from being longer than 50 ms
+    result = timeout_func(0.05, _eval_expr, query_py_formatted)
+    if result is None:
+        return True
+    result = str(result)
+
+    if result != query:
+        search.result_container.answers['calculate'] = {'answer': f"{query} = {result}"}
     return True
-
-
-def is_allowed():
-    return not settings['server']['public_instance']

searx/settings.yml

@@ -219,13 +219,13 @@ outgoing:
 #
 # enabled_plugins:
 #   # these plugins are enabled if nothing is configured ..
+#   - 'Basic Calculator'
 #   - 'Hash plugin'
 #   - 'Self Information'
 #   - 'Tracker URL remover'
 #   - 'Ahmia blacklist'  # activation depends on outgoing.using_tor_proxy
 #   # these plugins are disabled if nothing is configured ..
 #   - 'Hostnames plugin'  # see 'hostnames' configuration below
-#   - 'Basic Calculator'
 #   - 'Open Access DOI rewrite'
 #   - 'Tor check plugin'
 #   # Read the docs before activate: auto-detection of the language could be
@@ -562,33 +562,6 @@ engines:
     categories: general
     shortcut: cc
 
-  - name: bahnhof
-    engine: json_engine
-    search_url: https://www.bahnhof.de/api/stations/search/{query}
-    url_prefix: https://www.bahnhof.de/
-    url_query: slug
-    title_query: name
-    content_query: state
-    shortcut: bf
-    disabled: true
-    about:
-      website: https://www.bahn.de
-      wikidata_id: Q22811603
-      use_official_api: false
-      require_api_key: false
-      results: JSON
-      language: de
-    tests:
-      bahnhof:
-        matrix:
-          query: berlin
-          lang: en
-        result_container:
-          - not_empty
-          - ['one_title_contains', 'Berlin Hauptbahnhof']
-        test:
-          - unique_results
-
   - name: deezer
     engine: deezer
     shortcut: dz

searx/static/themes/simple/src/less/style.less

@@ -748,6 +748,20 @@ summary.title {
   }
 }
 
+#engines_msg {
+  .engine-name {
+    width: 10rem;
+  }
+
+  .response-error {
+    color: var(--color-error);
+  }
+
+  .bar-chart-value {
+    width: auto;
+  }
+}
+
 #search_url {
   div.selectable_url {
     pre {

searx/templates/simple/elements/engines_msg.html

@@ -1,27 +1,34 @@
 <div id="engines_msg">
   {% if not results and not answers %}
   <details class="sidebar-collapsable" open>
+    <summary class="title" id="engines_msg-title">{{ _('Messages from the search engines') }}</summary>
   {% else %}
   <details class="sidebar-collapsable">
+    <summary class="title" id="engines_msg-title">{{ _('Response time') }}: {{ max_response_time | round(1) }} {{ _('seconds') }}</summary>
   {% endif %}
-    <summary class="title" id="engines_msg-title">{{ _('Messages from the search engines') }}</summary>
+    <table class="engine-stats" id="engines_msg-table">
-    <div class="dialog-error" role="alert">
+      {%- for engine_name, error_type in unresponsive_engines -%}
-      {{ icon_big('warning') }}
+      <tr>
-      <div>
+        <td class="engine-name">
-        <p>
+          <a href="{{ url_for('stats', engine=engine_name|e) }}"
-          <strong>{{ _('Error!') }}</strong>
+             title="{{ _('View error logs and submit a bug report') }}">
-          {{ _('Engines cannot retrieve results') }}:
+             {{- engine_name -}}
-        </p>
+          </a>
-        {%- for engine_name, error_type in unresponsive_engines -%}
+        </td>
-          <p>{{- engine_name }} (
+        <td class="response-error">{{- error_type -}}</td>
-            <a href="{{ url_for('stats', engine=engine_name|e) }}"
+      </tr>
-               title="{{ _('View error logs and submit a bug report') }}">
+      {%- endfor -%}
-               {{- error_type -}}
+      {%- for engine_name, response_time in timings -%}
-            </a>
+      <tr>
-            ){{- '' -}}
+        <td class="engine-name"><a href="{{ url_for('stats', engine=engine_name|e) }}">{{ engine_name }}</a></td>
-          </p>
+        <td class="response-time">
-        {%- endfor -%}
+          <div class="bar-chart-value">{{- response_time | round(1) -}}</div>
-      </div>
+          <div class="bar-chart-graph" aria-labelledby="{{engine_name}}_time" aria-hidden="true">
-    </div>
+            <div class="bar-chart-bar bar{{ (100 * response_time / max_response_time) | round | int }}"></div>
+          </div>
+        </td>
+      </tr>
+      {%- endfor -%}
+    </table>
   </details>
 </div>

searx/templates/simple/preferences.html

@@ -38,7 +38,7 @@
 
 {%- macro plugin_preferences(section) -%}
 {%- for plugin in plugins -%}
-{%- if plugin.preference_section == section and (plugin.is_allowed() if plugin.is_allowed else True) -%}
+{%- if plugin.preference_section == section -%}
 <fieldset>{{- '' -}}
     <legend>{{ _(plugin.name) }}</legend>{{- '' -}}
     <div class="value">

searx/templates/simple/results.html

@@ -57,14 +57,12 @@
           {%- include 'simple/elements/suggestions.html' -%}
         {%- endif -%}
 
+        {%- include 'simple/elements/engines_msg.html' -%}
+
         {%- if method == 'POST' -%}
           {%- include 'simple/elements/search_url.html' -%}
         {%- endif -%}
 
-        {%- if unresponsive_engines -%}
-          {%- include 'simple/elements/engines_msg.html' -%}
-	{%- endif -%}
-
         {%- if search_formats -%}
           {%- include 'simple/elements/apis.html' -%}
         {%- endif -%}

searx/webapp.py

@@ -761,6 +761,11 @@ def search():
         )
     )
 
+    # engine_timings: get engine response times sorted from slowest to fastest
+    engine_timings = sorted(result_container.get_timings(), reverse=True, key=lambda e: e.total)
+    max_response_time = engine_timings[0].total if engine_timings else None
+    engine_timings_pairs = [(timing.engine, timing.total) for timing in engine_timings]
+
     # search_query.lang contains the user choice (all, auto, en, ...)
     # when the user choice is "auto", search.search_query.lang contains the detected language
     # otherwise it is equals to search_query.lang
@@ -789,7 +794,9 @@ def search():
             settings['search']['languages'],
             fallback=request.preferences.get_value("language")
         ),
-        timeout_limit = request.form.get('timeout_limit', None)
+        timeout_limit = request.form.get('timeout_limit', None),
+        timings = engine_timings_pairs,
+        max_response_time = max_response_time
         # fmt: on
     )