data: currencies: use SQLite instead of JSON

searx/data/__init__.py

@@ -7,7 +7,6 @@
 
 __all__ = [
     'ENGINE_TRAITS',
-    'CURRENCIES',
     'USER_AGENTS',
     'EXTERNAL_URLS',
     'WIKIDATA_UNITS',
@@ -16,12 +15,17 @@ __all__ = [
     'LOCALES',
     'ahmia_blacklist_loader',
     'fetch_engine_descriptions',
+    'fetch_iso4217_from_user',
+    'fetch_name_from_iso4217',
 ]
 
+import re
+import unicodedata
 import json
 import sqlite3
 from contextlib import contextmanager
-from typing import Dict, Generator, List
+from typing import Dict, Generator, List, Optional
+from functools import lru_cache
 from pathlib import Path
 
 
@@ -69,6 +73,42 @@ def fetch_engine_descriptions(language) -> Dict[str, List[str]]:
         return {result[0]: [result[1], result[2]] for result in res.fetchall()}
 
 
+def _normalize_name(name):
+    name = name.lower().replace('-', ' ').rstrip('s')
+    name = re.sub(' +', ' ', name)
+    return unicodedata.normalize('NFKD', name).lower()
+
+
+@lru_cache(10)
+def fetch_iso4217_from_user(name: str) -> Optional[str]:
+    with sql_connection("currencies.db") as connection:
+        # try the iso4217
+        res = connection.execute("SELECT iso4217 FROM currencies WHERE lower(iso4217)=? LIMIT 1", (name.lower(),))
+        result = res.fetchone()
+        if result:
+            return result[0]
+
+        # try the currency names
+        name = _normalize_name(name)
+        res = connection.execute("SELECT iso4217 FROM currencies WHERE name=?", (name,))
+        result = list(set(result[0] for result in res.fetchall()))
+        if len(result) == 1:
+            return result[0]
+
+        # ambiguity --> return nothing
+        return None
+
+
+@lru_cache(10)
+def fetch_name_from_iso4217(iso4217: str, language: str) -> Optional[str]:
+    with sql_connection("currencies.db") as connection:
+        res = connection.execute("SELECT name FROM currencies WHERE iso4217=? AND language=?", (iso4217, language))
+        result = [result[0] for result in res.fetchall()]
+        if len(result) == 1:
+            return result[0]
+        return None
+
+
 def ahmia_blacklist_loader():
     """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion
     names.  The MD5 values are fetched by::
@@ -82,7 +122,6 @@ def ahmia_blacklist_loader():
         return f.read().split()
 
 
-CURRENCIES = _load('currencies.json')
 USER_AGENTS = _load('useragents.json')
 EXTERNAL_URLS = _load('external_urls.json')
 WIKIDATA_UNITS = _load('wikidata_units.json')

searx/engines/openstreetmap.py

@@ -10,7 +10,7 @@ from functools import partial
 
 from flask_babel import gettext
 
-from searx.data import OSM_KEYS_TAGS, CURRENCIES
+from searx.data import OSM_KEYS_TAGS, fetch_name_from_iso4217
 from searx.utils import searx_useragent
 from searx.external_urls import get_external_url
 from searx.engines.wikidata import send_wikidata_query, sparql_string_escape, get_thumbnail
@@ -449,9 +449,9 @@ def get_key_label(key_name, lang):
         # https://taginfo.openstreetmap.org/keys/currency#values
         currency = key_name.split(':')
         if len(currency) > 1:
-            o = CURRENCIES['iso4217'].get(currency[1])
-            if o:
-                return get_label(o, lang).lower()
+            label = fetch_name_from_iso4217(currency[1], lang)
+            if label:
+                return label
             return currency[1]
 
     labels = OSM_KEYS_TAGS['keys']

searx/search/processors/online_currency.py

@@ -3,33 +3,14 @@
 
 """
 
-import unicodedata
 import re
 
-from searx.data import CURRENCIES
+from searx.data import fetch_iso4217_from_user, fetch_name_from_iso4217
 from .online import OnlineProcessor
 
 parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
 
 
-def normalize_name(name):
-    name = name.lower().replace('-', ' ').rstrip('s')
-    name = re.sub(' +', ' ', name)
-    return unicodedata.normalize('NFKD', name).lower()
-
-
-def name_to_iso4217(name):
-    name = normalize_name(name)
-    currency = CURRENCIES['names'].get(name, [name])
-    if isinstance(currency, str):
-        return currency
-    return currency[0]
-
-
-def iso4217_to_name(iso4217, language):
-    return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)
-
-
 class OnlineCurrencyProcessor(OnlineProcessor):
     """Processor class used by ``online_currency`` engines."""
 
@@ -52,14 +33,17 @@ class OnlineCurrencyProcessor(OnlineProcessor):
             amount = float(amount_str)
         except ValueError:
             return None
-        from_currency = name_to_iso4217(from_currency.strip())
-        to_currency = name_to_iso4217(to_currency.strip())
+        from_currency = fetch_iso4217_from_user(from_currency.strip())
+        to_currency = fetch_iso4217_from_user(to_currency.strip())
+
+        if from_currency is None or to_currency is None:
+            return None
 
         params['amount'] = amount
         params['from'] = from_currency
         params['to'] = to_currency
-        params['from_name'] = iso4217_to_name(from_currency, 'en')
-        params['to_name'] = iso4217_to_name(to_currency, 'en')
+        params['from_name'] = fetch_name_from_iso4217(from_currency, 'en')
+        params['to_name'] = fetch_name_from_iso4217(to_currency, 'en')
         return params
 
     def get_default_tests(self):