From e33858d6ef81d6b48f1e0fbd56252383ed6e032e Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 30 May 2021 19:20:46 +0200 Subject: [PATCH 1/3] [fix] correct kv template formatting and remove internal data Slightly modified merge of commit [ea7ccf24] from searx. [ea7ccf24] https://github.com/searx/searx/commit/ea7ccf24 Signed-off-by: Markus Heiser --- searx/templates/oscar/result_templates/key-value.html | 4 ++-- searx/templates/simple/result_templates/key-value.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/searx/templates/oscar/result_templates/key-value.html b/searx/templates/oscar/result_templates/key-value.html index d5c56a189..9d75682d4 100644 --- a/searx/templates/oscar/result_templates/key-value.html +++ b/searx/templates/oscar/result_templates/key-value.html @@ -1,8 +1,8 @@ {% from 'oscar/macros.html' import result_footer, result_footer_rtl with context %} -
+
{% for key, value in result.items() %} - {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions'] %} + {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions', 'pretty_url', 'parsed_url'] %} {% continue %} {% endif %} diff --git a/searx/templates/simple/result_templates/key-value.html b/searx/templates/simple/result_templates/key-value.html index eebaa2c85..cb7f829ee 100644 --- a/searx/templates/simple/result_templates/key-value.html +++ b/searx/templates/simple/result_templates/key-value.html @@ -1,6 +1,6 @@
{% for key, value in result.items() %} - {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions'] %} + {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions', 'pretty_url', 'parsed_url'] %} {% continue %} {% endif %} From e4b6558339ae5c78071a78bfbdc5924470882af7 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Sun, 30 May 2021 19:20:17 +0200 Subject: [PATCH 2/3] [enh] add redis offline engine / https://redis.io/ Slightly modified merge of commit [97269be6], [01a8a5814a] and [c8d2b5eb] from searx. [97269be6] https://github.com/searx/searx/commit/97269be6 [01a8a581] https://github.com/searx/searx/commit/01a8a581 [c8d2b5eb] https://github.com/searx/searx/commit/c8d2b5eb Signed-off-by: Markus Heiser --- searx/engines/redis_server.py | 75 +++++++++++++++++++++++++++++++++++ searx/settings.yml | 9 +++++ 2 files changed, 84 insertions(+) create mode 100644 searx/engines/redis_server.py diff --git a/searx/engines/redis_server.py b/searx/engines/redis_server.py new file mode 100644 index 000000000..da7cf5a4f --- /dev/null +++ b/searx/engines/redis_server.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +# pylint: disable=missing-function-docstring +"""Redis engine (offline) + +""" + +import redis # pylint: disable=import-error + +engine_type = 'offline' + +# redis connection variables +host = '127.0.0.1' +port = 6379 +password = '' +db = 0 + +# engine specific variables +paging = False +result_template = 'key-value.html' +exact_match_only = True + +redis_client = None +def init(_engine_settings): + # pylint: disable=global-statement + global redis_client + redis_client = redis.StrictRedis( + host = host, + port = port, + db = db, + password = password or None, + decode_responses = True, + ) + +def search(query, _params): + + if not exact_match_only: + return search_keys(query) + + ret = redis_client.hgetall(query) + if ret: + ret['template'] = result_template + return [ret] + + if ' ' in query: + qset, rest = query.split(' ', 1) + ret = [] + for res in redis_client.hscan_iter( + qset, match='*{}*'.format(rest) + ): + ret.append({ + res[0]: res[1], + 'template': result_template, + }) + return ret + return [] + +def search_keys(query): + ret = [] + for key in redis_client.scan_iter( + match='*{}*'.format(query) + ): + key_type = redis_client.type(key) + res = None + + if key_type == 'hash': + res = redis_client.hgetall(key) + elif key_type == 'list': + res = dict(enumerate(redis_client.lrange(key, 0, -1))) + + if res: + res['template'] = result_template + res['redis_key'] = key + ret.append(res) + return ret diff --git a/searx/settings.yml b/searx/settings.yml index 0c4ba5f54..63d100ca5 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -960,6 +960,15 @@ engines: timeout : 10.0 disabled : True + # Required dependency: redis + # - name: myredis + # engine: redis_server + # exact_match_only: False + # host: '127.0.0.1' + # port: 6379 + # password: '' + # db: 0 + # tmp suspended: bad certificate # - name : scanr structures # shortcut: scs From 39c18274c62f2eb8edf2b55b2e4e66a138376913 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 2 Jun 2021 09:54:58 +0200 Subject: [PATCH 3/3] [fix] enigine redis - avoid error when the engine is loaded Should be _redis_client to avoid an error when the engine is loaded. Suggested-by: @dalf https://github.com/searxng/searxng/pull/124#pullrequestreview-673885664 Signed-off-by: Markus Heiser --- searx/engines/redis_server.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/searx/engines/redis_server.py b/searx/engines/redis_server.py index da7cf5a4f..e8c248e32 100644 --- a/searx/engines/redis_server.py +++ b/searx/engines/redis_server.py @@ -20,11 +20,10 @@ paging = False result_template = 'key-value.html' exact_match_only = True -redis_client = None +_redis_client = None def init(_engine_settings): - # pylint: disable=global-statement - global redis_client - redis_client = redis.StrictRedis( + global _redis_client # pylint: disable=global-statement + _redis_client = redis.StrictRedis( host = host, port = port, db = db, @@ -33,11 +32,12 @@ def init(_engine_settings): ) def search(query, _params): + global _redis_client # pylint: disable=global-statement if not exact_match_only: return search_keys(query) - ret = redis_client.hgetall(query) + ret = _redis_client.hgetall(query) if ret: ret['template'] = result_template return [ret] @@ -45,7 +45,7 @@ def search(query, _params): if ' ' in query: qset, rest = query.split(' ', 1) ret = [] - for res in redis_client.hscan_iter( + for res in _redis_client.hscan_iter( qset, match='*{}*'.format(rest) ): ret.append({ @@ -56,17 +56,19 @@ def search(query, _params): return [] def search_keys(query): + global _redis_client # pylint: disable=global-statement + ret = [] - for key in redis_client.scan_iter( + for key in _redis_client.scan_iter( match='*{}*'.format(query) ): - key_type = redis_client.type(key) + key_type = _redis_client.type(key) res = None if key_type == 'hash': - res = redis_client.hgetall(key) + res = _redis_client.hgetall(key) elif key_type == 'list': - res = dict(enumerate(redis_client.lrange(key, 0, -1))) + res = dict(enumerate(_redis_client.lrange(key, 0, -1))) if res: res['template'] = result_template