From 848c8d0544eb0138da4414a750f79b22883f91b2 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 25 May 2025 10:40:57 +0200 Subject: [mod] data: implement a simple currencies (SQL) database (#4836) To reduce the memory footprint, this patch no longer loads the JSON data completely into memory. Instead, there is an SQL database based on `ExpireCacheSQLite`. The class CurrenciesDB is a simple DB application that encapsulates the DB (queries and initialization) and provides convenient methods like `name_to_iso4217` and `iso4217_to_name`. Related: - https://github.com/searxng/searxng/discussions/1892 - https://github.com/searxng/searxng/pull/3458#issuecomment-2900807671 - https://github.com/searxng/searxng/pull/4650 Signed-off-by: Markus Heiser --- searx/data/__init__.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'searx/data/__init__.py') diff --git a/searx/data/__init__.py b/searx/data/__init__.py index 48455ebe3..9be1cd67e 100644 --- a/searx/data/__init__.py +++ b/searx/data/__init__.py @@ -4,21 +4,17 @@ make data.all """ - +from __future__ import annotations __all__ = ["ahmia_blacklist_loader"] import json -from pathlib import Path import typing -from searx import logger - -log = logger.getChild("data") +from .core import log, data_dir +from .currencies import CurrenciesDB -data_dir = Path(__file__).parent - -CURRENCIES: dict[str, typing.Any] +CURRENCIES: CurrenciesDB USER_AGENTS: dict[str, typing.Any] EXTERNAL_URLS: dict[str, typing.Any] WIKIDATA_UNITS: dict[str, typing.Any] @@ -29,7 +25,7 @@ ENGINE_TRAITS: dict[str, typing.Any] LOCALES: dict[str, typing.Any] lazy_globals = { - "CURRENCIES": None, + "CURRENCIES": CurrenciesDB(), "USER_AGENTS": None, "EXTERNAL_URLS": None, "WIKIDATA_UNITS": None, @@ -41,7 +37,6 @@ lazy_globals = { } data_json_files = { - "CURRENCIES": "currencies.json", "USER_AGENTS": "useragents.json", "EXTERNAL_URLS": "external_urls.json", "WIKIDATA_UNITS": "wikidata_units.json", @@ -63,6 +58,7 @@ def __getattr__(name): return data log.debug("init searx.data.%s", name) + with open(data_dir / data_json_files[name], encoding='utf-8') as f: lazy_globals[name] = json.load(f) -- cgit v1.2.3