[mod] lower memory footprint by lazy loading JSON data

This patch implements lazy loading of the JSON data.

Motivation: in most requests not all JSON data is needed, but loaded.  By
example these four JSON files:

- currencies.json ~550KB
- engine_descriptions.json ~1,3MB
- external_bangs.json ~1,3MB
- osm_keys_tags.json ~ 2,2MB

most often not used and consume a lot of memory and BTW they also extend the
time required to instantiate a walker.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2024-04-29 18:36:26 +02:00
parent e45a7cc063
commit 82fd0dac60
15 changed files with 73 additions and 48 deletions

View file

@ -16,7 +16,7 @@ import types
from typing import Dict, Literal, Iterable, Union, Callable, Optional, TYPE_CHECKING
from searx import locales
from searx.data import data_dir, ENGINE_TRAITS
from searx import data
if TYPE_CHECKING:
from . import Engine
@ -193,7 +193,7 @@ class EngineTraits:
class EngineTraitsMap(Dict[str, EngineTraits]):
"""A python dictionary to map :class:`EngineTraits` by engine name."""
ENGINE_TRAITS_FILE = (data_dir / 'engine_traits.json').resolve()
ENGINE_TRAITS_FILE = (data.data_dir / 'engine_traits.json').resolve()
"""File with persistence of the :py:obj:`EngineTraitsMap`."""
def save_data(self):
@ -205,7 +205,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
def from_data(cls) -> 'EngineTraitsMap':
"""Instantiate :class:`EngineTraitsMap` object from :py:obj:`ENGINE_TRAITS`"""
obj = cls()
for k, v in ENGINE_TRAITS.items():
for k, v in data.ENGINE_TRAITS.items():
obj[k] = EngineTraits(**v)
return obj