[mod] lower memory footprint by lazy loading JSON data

This patch implements lazy loading of the JSON data.

Motivation: in most requests not all JSON data is needed, but loaded.  By
example these four JSON files:

- currencies.json ~550KB
- engine_descriptions.json ~1,3MB
- external_bangs.json ~1,3MB
- osm_keys_tags.json ~ 2,2MB

most often not used and consume a lot of memory and BTW they also extend the
time required to instantiate a walker.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/annas_archive.py

@@ -37,9 +37,9 @@ from typing import List, Dict, Any, Optional
 from urllib.parse import quote
 from lxml import html
 
+from searx import data
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 from searx.enginelib.traits import EngineTraits
-from searx.data import ENGINE_TRAITS
 
 # about
 about: Dict[str, Any] = {
@@ -86,7 +86,7 @@ aa_ext: str = ''
 
 def init(engine_settings=None):  # pylint: disable=unused-argument
     """Check of engine's settings."""
-    traits = EngineTraits(**ENGINE_TRAITS['annas archive'])
+    traits = EngineTraits(**data.ENGINE_TRAITS['annas archive'])
 
     if aa_content and aa_content not in traits.custom['content']:
         raise ValueError(f'invalid setting content: {aa_content}')

searx/engines/duckduckgo.py

@@ -12,6 +12,7 @@ import babel
 import lxml.html
 
 from searx import (
+    data,
     locales,
     redislib,
     external_bang,
@@ -230,7 +231,7 @@ def quote_ddg_bangs(query):
     for val in re.split(r'(\s+)', query):
         if not val.strip():
             continue
-        if val.startswith('!') and external_bang.get_node(external_bang.EXTERNAL_BANGS, val[1:]):
+        if val.startswith('!') and external_bang.get_node(data.EXTERNAL_BANGS, val[1:]):
             val = f"'{val}'"
         query_parts.append(val)
     return ' '.join(query_parts)

searx/engines/duckduckgo_definitions.py

@@ -18,7 +18,7 @@ from typing import TYPE_CHECKING
 from urllib.parse import urlencode, urlparse, urljoin
 from lxml import html
 
-from searx.data import WIKIDATA_UNITS
+from searx import data
 from searx.utils import extract_text, html_to_text, get_string_replaces_function
 from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
 
@@ -238,7 +238,7 @@ def unit_to_str(unit):
     for prefix in WIKIDATA_PREFIX:
         if unit.startswith(prefix):
             wikidata_entity = unit[len(prefix) :]
-            real_unit = WIKIDATA_UNITS.get(wikidata_entity)
+            real_unit = data.WIKIDATA_UNITS.get(wikidata_entity)
             if real_unit is None:
                 return unit
             return real_unit['symbol']

searx/engines/openstreetmap.py

@@ -10,7 +10,7 @@ from functools import partial
 
 from flask_babel import gettext
 
-from searx.data import OSM_KEYS_TAGS, CURRENCIES
+from searx import data as searx_data
 from searx.utils import searx_useragent
 from searx.external_urls import get_external_url
 from searx.engines.wikidata import send_wikidata_query, sparql_string_escape, get_thumbnail
@@ -435,7 +435,7 @@ def get_label(labels, lang):
 def get_tag_label(tag_category, tag_name, lang):
     """Get tag label from OSM_KEYS_TAGS"""
     tag_name = '' if tag_name is None else tag_name
-    tag_labels = OSM_KEYS_TAGS['tags'].get(tag_category, {}).get(tag_name, {})
+    tag_labels = searx_data.OSM_KEYS_TAGS['tags'].get(tag_category, {}).get(tag_name, {})
     return get_label(tag_labels, lang)
 
 
@@ -449,12 +449,12 @@ def get_key_label(key_name, lang):
         # https://taginfo.openstreetmap.org/keys/currency#values
         currency = key_name.split(':')
         if len(currency) > 1:
-            o = CURRENCIES['iso4217'].get(currency[1])
+            o = searx_data.CURRENCIES['iso4217'].get(currency[1])
             if o:
                 return get_label(o, lang).lower()
             return currency[1]
 
-    labels = OSM_KEYS_TAGS['keys']
+    labels = searx_data.OSM_KEYS_TAGS['keys']
     for k in key_name.split(':') + ['*']:
         labels = labels.get(k)
         if labels is None:

searx/engines/wikidata.py

@@ -13,7 +13,7 @@ from json import loads
 from dateutil.parser import isoparse
 from babel.dates import format_datetime, format_date, format_time, get_datetime_format
 
-from searx.data import WIKIDATA_UNITS
+from searx import data
 from searx.network import post, get
 from searx.utils import searx_useragent, get_string_replaces_function
 from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
@@ -762,7 +762,7 @@ def debug_explain_wikidata_query(query, method='GET'):
 
 def init(engine_settings=None):  # pylint: disable=unused-argument
     # WIKIDATA_PROPERTIES : add unit symbols
-    WIKIDATA_PROPERTIES.update(WIKIDATA_UNITS)
+    WIKIDATA_PROPERTIES.update(data.WIKIDATA_UNITS)
 
     # WIKIDATA_PROPERTIES : add property labels
     wikidata_property_names = []

searx/engines/zlibrary.py

@@ -40,9 +40,9 @@ from urllib.parse import quote
 from lxml import html
 from flask_babel import gettext
 
+from searx import data
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 from searx.enginelib.traits import EngineTraits
-from searx.data import ENGINE_TRAITS
 
 if TYPE_CHECKING:
     import httpx
@@ -80,7 +80,7 @@ zlib_ext: str = ""
 
 def init(engine_settings=None) -> None:  # pylint: disable=unused-argument
     """Check of engine's settings."""
-    traits: EngineTraits = EngineTraits(**ENGINE_TRAITS["z-library"])
+    traits: EngineTraits = EngineTraits(**data.ENGINE_TRAITS["z-library"])
 
     if zlib_ext and zlib_ext not in traits.custom["ext"]:
         raise ValueError(f"invalid setting ext: {zlib_ext}")