[mod] lower memory footprint by lazy loading JSON data

This patch implements lazy loading of the JSON data.

Motivation: in most requests not all JSON data is needed, but loaded.  By
example these four JSON files:

- currencies.json ~550KB
- engine_descriptions.json ~1,3MB
- external_bangs.json ~1,3MB
- osm_keys_tags.json ~ 2,2MB

most often not used and consume a lot of memory and BTW they also extend the
time required to instantiate a walker.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2024-04-29 18:36:26 +02:00
parent e45a7cc063
commit 82fd0dac60
15 changed files with 73 additions and 48 deletions

View file

@ -2,7 +2,7 @@
# pylint: disable=missing-module-docstring
from hashlib import md5
from searx.data import ahmia_blacklist_loader
from searx import data
name = "Ahmia blacklist"
description = "Filter out onion results that appear in Ahmia's blacklist. (See https://ahmia.fi/blacklist)"
@ -24,5 +24,5 @@ def init(_app, settings):
if not settings['outgoing']['using_tor_proxy']:
# disable the plugin
return False
ahmia_blacklist = ahmia_blacklist_loader()
ahmia_blacklist = data.ahmia_blacklist_loader()
return True

View file

@ -4,7 +4,7 @@
from flask_babel import gettext
from searx.data import WIKIDATA_UNITS
from searx import data
name = "Unit converter plugin"
description = gettext("Convert between units")
@ -38,7 +38,7 @@ def _parse_text_and_convert(search, splitted_query):
from_unit = None
to_unit = None
for unit in WIKIDATA_UNITS.values():
for unit in data.WIKIDATA_UNITS.values():
if unit['symbol'] == from_unit_key:
from_unit = unit