mirror of https://github.com/searxng/searxng.git
[mod] /stats : add reliability column and sort by column links
This commit is contained in:
Alexandre Flament
parent
c54bf42cb9
commit
09e7ecdce2
|
|
@ -97,12 +97,12 @@ def initialize(engine_names=None):
|
|||
histogram_storage.configure(histogram_width, histogram_size, 'engine', engine_name, 'time', 'total')
|
||||
|
||||
|
||||
def get_engine_errors(engline_list):
|
||||
def get_engine_errors(engline_name_list):
|
||||
result = {}
|
||||
engine_names = list(errors_per_engines.keys())
|
||||
engine_names.sort()
|
||||
for engine_name in engine_names:
|
||||
if engine_name not in engline_list:
|
||||
if engine_name not in engline_name_list:
|
||||
continue
|
||||
|
||||
error_stats = errors_per_engines[engine_name]
|
||||
|
|
@ -126,61 +126,86 @@ def get_engine_errors(engline_list):
|
|||
return result
|
||||
|
||||
|
||||
def to_percentage(stats, maxvalue):
|
||||
for engine_stat in stats:
|
||||
if maxvalue:
|
||||
engine_stat['percentage'] = int(engine_stat['avg'] / maxvalue * 100)
|
||||
def get_reliabilities(engline_name_list, checker_results):
|
||||
reliabilities = {}
|
||||
|
||||
engine_errors = get_engine_errors(engline_name_list)
|
||||
|
||||
for engine_name in engline_name_list:
|
||||
checker_result = checker_results.get(engine_name, {})
|
||||
checker_success = checker_result.get('success', True)
|
||||
errors = engine_errors.get(engine_name) or []
|
||||
if counter('engine', engine_name, 'search', 'count', 'sent') == 0:
|
||||
# no request
|
||||
reliablity = None
|
||||
elif checker_success and not errors:
|
||||
reliablity = 100
|
||||
elif 'simple' in checker_result.get('errors', {}):
|
||||
# the basic (simple) test doesn't work: the engine is broken accoding to the checker
|
||||
# even if there is no exception
|
||||
reliablity = 0
|
||||
else:
|
||||
engine_stat['percentage'] = 0
|
||||
return stats
|
||||
reliablity = 100 - sum([error['percentage'] for error in errors if not error.get('secondary')])
|
||||
|
||||
reliabilities[engine_name] = {
|
||||
'reliablity': reliablity,
|
||||
'errors': errors,
|
||||
'checker': checker_results.get(engine_name, {}).get('errors', {}).keys(),
|
||||
}
|
||||
return reliabilities
|
||||
|
||||
|
||||
def get_engines_stats(engine_list):
|
||||
def round_or_none(number, digits):
|
||||
return round(number, digits) if number else number
|
||||
|
||||
|
||||
def get_engines_stats(engine_name_list):
|
||||
assert counter_storage is not None
|
||||
assert histogram_storage is not None
|
||||
|
||||
list_time = []
|
||||
|
||||
max_time_total = max_result_count = None # noqa
|
||||
for engine_name in engine_list:
|
||||
successful_count = counter('engine', engine_name, 'search', 'count', 'successful')
|
||||
if successful_count == 0:
|
||||
for engine_name in engine_name_list:
|
||||
sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
|
||||
if sent_count == 0:
|
||||
continue
|
||||
|
||||
result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
|
||||
successful_count = counter('engine', engine_name, 'search', 'count', 'successful')
|
||||
|
||||
time_total = histogram('engine', engine_name, 'time', 'total').percentage(50)
|
||||
time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
|
||||
time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
|
||||
time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
|
||||
time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
|
||||
time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
|
||||
result_count = result_count_sum / float(successful_count)
|
||||
|
||||
if result_count:
|
||||
result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
|
||||
result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
|
||||
if successful_count and result_count_sum:
|
||||
score = counter('engine', engine_name, 'score') # noqa
|
||||
score_per_result = score / float(result_count_sum)
|
||||
else:
|
||||
score = score_per_result = 0.0
|
||||
|
||||
max_time_total = max(time_total, max_time_total or 0)
|
||||
max_result_count = max(result_count, max_result_count or 0)
|
||||
max_time_total = max(time_total or 0, max_time_total or 0)
|
||||
max_result_count = max(result_count or 0, max_result_count or 0)
|
||||
|
||||
list_time.append({
|
||||
'total': round(time_total, 1),
|
||||
'total_p80': round(time_total_p80, 1),
|
||||
'total_p95': round(time_total_p95, 1),
|
||||
'http': round(time_http, 1),
|
||||
'http_p80': round(time_http_p80, 1),
|
||||
'http_p95': round(time_http_p95, 1),
|
||||
'name': engine_name,
|
||||
'processing': round(time_total - time_http, 1),
|
||||
'processing_p80': round(time_total_p80 - time_http_p80, 1),
|
||||
'processing_p95': round(time_total_p95 - time_http_p95, 1),
|
||||
'total': round_or_none(time_total, 1),
|
||||
'total_p80': round_or_none(time_total_p80, 1),
|
||||
'total_p95': round_or_none(time_total_p95, 1),
|
||||
'http': round_or_none(time_http, 1),
|
||||
'http_p80': round_or_none(time_http_p80, 1),
|
||||
'http_p95': round_or_none(time_http_p95, 1),
|
||||
'processing': round(time_total - time_http, 1) if time_total else None,
|
||||
'processing_p80': round(time_total_p80 - time_http_p80, 1) if time_total else None,
|
||||
'processing_p95': round(time_total_p95 - time_http_p95, 1) if time_total else None,
|
||||
'score': score,
|
||||
'score_per_result': score_per_result,
|
||||
'result_count': result_count,
|
||||
})
|
||||
|
||||
return {
|
||||
'time': list_time,
|
||||
'max_time': math.ceil(max_time_total or 0),
|
||||
|
|
|
|||
|
|
@ -682,6 +682,7 @@ input[type=checkbox]:not(:checked) + .label_hide_if_checked + .label_hide_if_not
|
|||
padding: 0.5rem 1rem;
|
||||
margin: 0rem 0 0 2rem;
|
||||
border: 1px solid #ddd;
|
||||
box-shadow: 2px 2px 2px 0px rgba(0, 0, 0, 0.1);
|
||||
background: white;
|
||||
font-size: 14px;
|
||||
font-weight: normal;
|
||||
|
|
@ -756,3 +757,21 @@ td:hover .engine-tooltip,
|
|||
padding: 0.4rem 0;
|
||||
width: 1px;
|
||||
}
|
||||
.stacked-bar-chart-serie1 {
|
||||
display: flex;
|
||||
flex-shrink: 0;
|
||||
flex-grow: 0;
|
||||
flex-basis: unset;
|
||||
background: #5bc0de;
|
||||
box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
|
||||
padding: 0.4rem 0;
|
||||
}
|
||||
.stacked-bar-chart-serie2 {
|
||||
display: flex;
|
||||
flex-shrink: 0;
|
||||
flex-grow: 0;
|
||||
flex-basis: unset;
|
||||
background: #deb15b;
|
||||
box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
|
||||
padding: 0.4rem 0;
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -8,6 +8,7 @@
|
|||
padding: 0.5rem 1rem;
|
||||
margin: 0rem 0 0 2rem;
|
||||
border: 1px solid #ddd;
|
||||
box-shadow: 2px 2px 2px 0px rgba(0,0,0,0.1);
|
||||
background: white;
|
||||
font-size: 14px;
|
||||
font-weight: normal;
|
||||
|
|
@ -77,3 +78,17 @@ th:hover .engine-tooltip, td:hover .engine-tooltip, .engine-tooltip:hover {
|
|||
width: 1px;
|
||||
}
|
||||
|
||||
.stacked-bar-chart-serie1 {
|
||||
.stacked-bar-chart-base();
|
||||
background: #5bc0de;
|
||||
box-shadow: inset 0 -1px 0 rgba(0,0,0,.15);
|
||||
padding: 0.4rem 0;
|
||||
}
|
||||
|
||||
.stacked-bar-chart-serie2 {
|
||||
.stacked-bar-chart-base();
|
||||
background: #deb15b;
|
||||
box-shadow: inset 0 -1px 0 rgba(0,0,0,.15);
|
||||
padding: 0.4rem 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
{% extends "oscar/base.html" %}
|
||||
{% block styles %}
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/charts.min.css') }}" type="text/css" />
|
||||
<style>
|
||||
#engine-times {
|
||||
--labels-size: 20rem;
|
||||
|
|
@ -12,6 +11,15 @@
|
|||
</style>
|
||||
{% endblock %}
|
||||
{% block title %}{{ _('stats') }} - {% endblock %}
|
||||
|
||||
{%- macro th_sort(column_order, column_name) -%}
|
||||
{% if column_order==sort_order %}
|
||||
{{ column_name }} {{ icon('chevron-down') }}
|
||||
{% else %}
|
||||
<a href="{{ url_for('stats', sort=column_order) }}">{{ column_name }}
|
||||
{% endif %}
|
||||
{%- endmacro -%}
|
||||
|
||||
{% block content %}
|
||||
<div class="container-fluid">
|
||||
<h1>{{ _('Engine stats') }}</h1>
|
||||
|
|
@ -25,27 +33,33 @@
|
|||
{% else %}
|
||||
<table class="table table-hover table-condensed table-striped">
|
||||
<tr>
|
||||
<th scope="col" style="width:20rem;">{{ _("Engine name") }}</th>
|
||||
<th scope="col" style="width:7rem; text-align: right;">{{ _('Scores') }}</th>
|
||||
<th scope="col">{{ _('Number of results') }}</th>
|
||||
<th scope="col">{{ _('Response time') }}</th>
|
||||
<th scope="col" style="width:20rem;">{{ th_sort('name', _("Engine name")) }}</th>
|
||||
<th scope="col" style="width:7rem; text-align: right;">{{ th_sort('score', _('Scores')) }}</th>
|
||||
<th scope="col">{{ th_sort('result_count', _('Result count')) }}</th>
|
||||
<th scope="col">{{ th_sort('time', _('Response time')) }}</th>
|
||||
<th scope="col" style="text-align: right;">{{ th_sort('reliability', _('Reliability')) }}</th>
|
||||
</tr>
|
||||
{% for engine_stat in engine_stats.get('time', []) %}
|
||||
<tr>
|
||||
<td>{{ engine_stat.name }}</td>
|
||||
<td style="text-align: right;">
|
||||
{% if engine_stat.score %}
|
||||
<span aria-labelledby="{{engine_stat.name}}_score" >{{ engine_stat.score|round(1) }}</span>
|
||||
<div class="engine-tooltip text-left" role="tooltip" id="{{engine_name}}_score">{{- "" -}}
|
||||
<p>{{ _('Scores per result') }}: {{ engine_stat.score_per_result | round(3) }}</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{%- if engine_stat.result_count -%}
|
||||
<span class="stacked-bar-chart-value">{{- engine_stat.result_count | int -}}</span>{{- "" -}}
|
||||
<span class="stacked-bar-chart" aria-labelledby="{{engine_stat.name}}_chart_result_count" aria-hidden="true">{{- "" -}}
|
||||
<span style="width: calc(max(2px, 100%*{{ (engine_stat.result_count / engine_stats.max_result_count )|round(3) }}))" class="stacked-bar-chart-serie1"></span>{{- "" -}}
|
||||
</span>{{- "" -}}
|
||||
</span>
|
||||
{%- endif -%}
|
||||
</td>
|
||||
<td>
|
||||
{%- if engine_stat.total -%}
|
||||
<span class="stacked-bar-chart-value">{{- engine_stat.total | round(1) -}}</span>{{- "" -}}
|
||||
<span class="stacked-bar-chart" aria-labelledby="{{engine_stat.name}}_chart" aria-hidden="true">{{- "" -}}
|
||||
<span style="width: calc(max(2px, 100%*{{ (engine_stat.http / engine_stats.max_time )|round(3) }}))" class="stacked-bar-chart-serie1"></span>{{- "" -}}
|
||||
|
|
@ -79,7 +93,9 @@
|
|||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
{%- endif -%}
|
||||
</td>
|
||||
<td style="text-align: right;"> {{ engine_reliabilities.get(engine_stat.name, {}).get('reliablity') }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
|
|||
from searx.answerers import answerers
|
||||
from searx.network import stream as http_stream
|
||||
from searx.answerers import ask
|
||||
from searx.metrics import get_engines_stats, get_engine_errors, histogram, counter
|
||||
from searx.metrics import get_engines_stats, get_engine_errors, get_reliabilities, histogram, counter
|
||||
|
||||
# serve pages with HTTP/1.1
|
||||
from werkzeug.serving import WSGIRequestHandler
|
||||
|
|
@ -1073,11 +1073,47 @@ def image_proxy():
|
|||
@app.route('/stats', methods=['GET'])
|
||||
def stats():
|
||||
"""Render engine statistics page."""
|
||||
checker_results = checker_get_result()
|
||||
checker_results = checker_results['engines'] \
|
||||
if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
|
||||
|
||||
filtered_engines = dict(filter(lambda kv: (kv[0], request.preferences.validate_token(kv[1])), engines.items()))
|
||||
engine_stats = get_engines_stats(filtered_engines)
|
||||
engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
|
||||
|
||||
sort_order = request.args.get('sort', default='name', type=str)
|
||||
|
||||
SORT_PARAMETERS = {
|
||||
'name': (False, 'name', ''),
|
||||
'score': (True, 'score', 0),
|
||||
'result_count': (True, 'result_count', 0),
|
||||
'time': (False, 'total', 0),
|
||||
'reliability': (False, 'reliability', 100),
|
||||
}
|
||||
|
||||
if sort_order not in SORT_PARAMETERS:
|
||||
sort_order = 'name'
|
||||
|
||||
reverse, key_name, default_value = SORT_PARAMETERS[sort_order]
|
||||
|
||||
def get_key(engine_stat):
|
||||
reliability = engine_reliabilities.get(engine_stat['name']).get('reliablity', 0)
|
||||
reliability_order = 0 if reliability else 1
|
||||
if key_name == 'reliability':
|
||||
key = reliability
|
||||
reliability_order = 0
|
||||
else:
|
||||
key = engine_stat.get(key_name) or default_value
|
||||
if reverse:
|
||||
reliability_order = 1 - reliability_order
|
||||
return (reliability_order, key, engine_stat['name'])
|
||||
|
||||
engine_stats['time'] = sorted(engine_stats['time'], reverse=reverse, key=get_key)
|
||||
return render(
|
||||
'stats.html',
|
||||
engine_stats=engine_stats
|
||||
sort_order=sort_order,
|
||||
engine_stats=engine_stats,
|
||||
engine_reliabilities=engine_reliabilities,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue