[enh] add offline engine for sqlite database

To test & demonstrate this implementation download:

  https://liste.mediathekview.de/filmliste-v2.db.bz2

and unpack into searx/data/filmliste-v2.db, in your settings.yml define a sqlite
engine named "demo"::

    - name : demo
      engine : sqlite
      shortcut: demo
      categories: general
      result_template: default.html
      database : searx/data/filmliste-v2.db
      query_str :  >-
        SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
               COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
               description AS content
          FROM film
         WHERE title LIKE :wildcard OR description LIKE :wildcard
         ORDER BY duration DESC
      disabled : False

Query to test: "!demo concert"

This is a rewrite of the implementation from commit [1]

[1] 8e90a214ce

Suggested-by: @virtadpt https://github.com/searx/searx/issues/2808
Signed-off-by: Markus Heiser <markus@darmarit.de>
This commit is contained in:
Markus Heiser 2021-05-18 18:42:58 +02:00
parent 8283ce3d79
commit d286d88747
2 changed files with 98 additions and 0 deletions

78
searx/engines/sqlite.py Normal file
View file

@ -0,0 +1,78 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pylint: disable=missing-function-docstring
"""SQLite database (Offline)
"""
import sqlite3
from searx import logger
logger = logger.getChild('SQLite engine')
engine_type = 'offline'
database = ""
query_str = ""
limit = 10
paging = True
result_template = 'key-value.html'
class SQLiteDB:
"""
Implements a `Context Manager`_ for a SQLite.
usage::
with SQLiteDB('test.db') as cur:
print(cur.execute('select sqlite_version();').fetchall()[0][0])
.. _Context Manager: https://docs.python.org/3/library/stdtypes.html#context-manager-types
"""
def __init__(self, db):
self.database = db
self.connect = None
def __enter__(self):
self.connect = sqlite3.connect(self.database)
self.connect.row_factory = sqlite3.Row
return self.connect.cursor()
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self.connect.commit()
self.connect.close()
def init(engine_settings):
if 'query_str' not in engine_settings:
raise ValueError('query_str cannot be empty')
if not engine_settings['query_str'].lower().startswith('select '):
raise ValueError('only SELECT query is supported')
def search(query, params):
global database, query_str, result_template # pylint: disable=global-statement
results = []
query_params = {
'query': query,
'wildcard': r'%' + query.replace(' ',r'%') + r'%',
'limit': limit,
'offset': (params['pageno'] - 1) * limit
}
query_to_run = query_str + ' LIMIT :limit OFFSET :offset'
with SQLiteDB(database) as cur:
cur.execute(query_to_run, query_params)
col_names = [cn[0] for cn in cur.description]
for row in cur.fetchall():
item = dict( zip(col_names, map(str, row)) )
item['template'] = result_template
logger.debug("append result --> %s", item)
results.append(item)
return results

View file

@ -1003,6 +1003,26 @@ engines:
timeout : 3.0
disabled : True
# For this demo of the sqlite engine download:
# https://liste.mediathekview.de/filmliste-v2.db.bz2
# and unpack into searx/data/filmliste-v2.db
# Query to test: "!demo concert"
#
# - name : demo
# engine : sqlite
# shortcut: demo
# categories: general
# result_template: default.html
# database : searx/data/filmliste-v2.db
# query_str : >-
# SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
# COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
# description AS content
# FROM film
# WHERE title LIKE :wildcard OR description LIKE :wildcard
# ORDER BY duration DESC
# disabled : False
- name : torrentz
engine : torrentz
shortcut : tor