summaryrefslogtreecommitdiff
path: root/searx/enginelib
diff options
context:
space:
mode:
Diffstat (limited to 'searx/enginelib')
-rw-r--r--searx/enginelib/__init__.py52
-rw-r--r--searx/enginelib/traits.py46
2 files changed, 58 insertions, 40 deletions
diff --git a/searx/enginelib/__init__.py b/searx/enginelib/__init__.py
index 3fa4edabb..a78981561 100644
--- a/searx/enginelib/__init__.py
+++ b/searx/enginelib/__init__.py
@@ -22,21 +22,25 @@ an example in which the command line is called in the development environment::
-----
"""
-from __future__ import annotations
__all__ = ["EngineCache", "Engine", "ENGINES_CACHE"]
-from typing import List, Callable, TYPE_CHECKING, Any
+import typing as t
+import abc
+from collections.abc import Callable
+import logging
import string
import typer
-from ..cache import ExpireCache, ExpireCacheCfg
+from ..cache import ExpireCacheSQLite, ExpireCacheCfg
-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
from searx.enginelib import traits
+ from searx.enginelib.traits import EngineTraits
+ from searx.extended_types import SXNG_Response
+ from searx.result_types import EngineResults
-
-ENGINES_CACHE = ExpireCache.build_cache(
+ENGINES_CACHE: ExpireCacheSQLite = ExpireCacheSQLite.build_cache(
ExpireCacheCfg(
name="ENGINES_CACHE",
MAXHOLD_TIME=60 * 60 * 24 * 7, # 7 days
@@ -62,7 +66,7 @@ def state():
title = f"properties of {ENGINES_CACHE.cfg.name}"
print(title)
print("=" * len(title))
- print(str(ENGINES_CACHE.properties)) # type: ignore
+ print(str(ENGINES_CACHE.properties))
@app.command()
@@ -152,11 +156,11 @@ class EngineCache:
"""
def __init__(self, engine_name: str, expire: int | None = None):
- self.expire = expire or ENGINES_CACHE.cfg.MAXHOLD_TIME
+ self.expire: int = expire or ENGINES_CACHE.cfg.MAXHOLD_TIME
_valid = "-_." + string.ascii_letters + string.digits
- self.table_name = "".join([c if c in _valid else "_" for c in engine_name])
+ self.table_name: str = "".join([c if c in _valid else "_" for c in engine_name])
- def set(self, key: str, value: Any, expire: int | None = None) -> bool:
+ def set(self, key: str, value: t.Any, expire: int | None = None) -> bool:
return ENGINES_CACHE.set(
key=key,
value=value,
@@ -164,14 +168,14 @@ class EngineCache:
ctx=self.table_name,
)
- def get(self, key: str, default=None) -> Any:
+ def get(self, key: str, default: t.Any = None) -> t.Any:
return ENGINES_CACHE.get(key, default=default, ctx=self.table_name)
def secret_hash(self, name: str | bytes) -> str:
return ENGINES_CACHE.secret_hash(name=name)
-class Engine: # pylint: disable=too-few-public-methods
+class Engine(abc.ABC): # pylint: disable=too-few-public-methods
"""Class of engine instances build from YAML settings.
Further documentation see :ref:`general engine configuration`.
@@ -181,6 +185,8 @@ class Engine: # pylint: disable=too-few-public-methods
This class is currently never initialized and only used for type hinting.
"""
+ logger: logging.Logger
+
# Common options in the engine module
engine_type: str
@@ -220,15 +226,15 @@ class Engine: # pylint: disable=too-few-public-methods
region: fr-BE
"""
- fetch_traits: Callable
+ fetch_traits: "Callable[[EngineTraits, bool], None]"
"""Function to to fetch engine's traits from origin."""
- traits: traits.EngineTraits
+ traits: "traits.EngineTraits"
"""Traits of the engine."""
# settings.yml
- categories: List[str]
+ categories: list[str]
"""Specifies to which :ref:`engine categories` the engine should be added."""
name: str
@@ -269,7 +275,7 @@ class Engine: # pylint: disable=too-few-public-methods
inactive: bool
"""Remove the engine from the settings (*disabled & removed*)."""
- about: dict
+ about: dict[str, dict[str, str]]
"""Additional fields describing the engine.
.. code:: yaml
@@ -291,9 +297,21 @@ class Engine: # pylint: disable=too-few-public-methods
the user is used to build and send a ``Accept-Language`` header in the
request to the origin search engine."""
- tokens: List[str]
+ tokens: list[str]
"""A list of secret tokens to make this engine *private*, more details see
:ref:`private engines`."""
weight: int
"""Weighting of the results of this engine (:ref:`weight <settings engines>`)."""
+
+ def init(self, engine_settings: dict[str, t.Any]) -> None: # pyright: ignore[reportUnusedParameter]
+ """Initialization of the engine. If no initialization is needed, drop
+ this init function."""
+
+ @abc.abstractmethod
+ def request(self, query: str, params: dict[str, t.Any]) -> None:
+ """Build up the params for the online request."""
+
+ @abc.abstractmethod
+ def response(self, resp: "SXNG_Response") -> "EngineResults":
+ """Parse out the result items from the response."""
diff --git a/searx/enginelib/traits.py b/searx/enginelib/traits.py
index 8bb4e0554..ad59a7a35 100644
--- a/searx/enginelib/traits.py
+++ b/searx/enginelib/traits.py
@@ -15,12 +15,12 @@ import os
import json
import dataclasses
import types
-from typing import Dict, Literal, Iterable, Union, Callable, Optional, TYPE_CHECKING
-
+import typing as t
+import pathlib
from searx import locales
from searx.data import data_dir, ENGINE_TRAITS
-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
from . import Engine
@@ -28,7 +28,7 @@ class EngineTraitsEncoder(json.JSONEncoder):
"""Encodes :class:`EngineTraits` to a serializable object, see
:class:`json.JSONEncoder`."""
- def default(self, o):
+ def default(self, o: t.Any) -> t.Any:
"""Return dictionary of a :class:`EngineTraits` object."""
if isinstance(o, EngineTraits):
return o.__dict__
@@ -39,7 +39,7 @@ class EngineTraitsEncoder(json.JSONEncoder):
class EngineTraits:
"""The class is intended to be instantiated for each engine."""
- regions: Dict[str, str] = dataclasses.field(default_factory=dict)
+ regions: dict[str, str] = dataclasses.field(default_factory=dict)
"""Maps SearXNG's internal representation of a region to the one of the engine.
SearXNG's internal representation can be parsed by babel and the value is
@@ -56,7 +56,7 @@ class EngineTraits:
...
"""
- languages: Dict[str, str] = dataclasses.field(default_factory=dict)
+ languages: dict[str, str] = dataclasses.field(default_factory=dict)
"""Maps SearXNG's internal representation of a language to the one of the engine.
SearXNG's internal representation can be parsed by babel and the value is
@@ -73,20 +73,20 @@ class EngineTraits:
...
"""
- all_locale: Optional[str] = None
+ all_locale: str | None = None
"""To which locale value SearXNG's ``all`` language is mapped (shown a "Default
language").
"""
- data_type: Literal['traits_v1'] = 'traits_v1'
+ data_type: t.Literal['traits_v1'] = 'traits_v1'
"""Data type, default is 'traits_v1'.
"""
- custom: Dict[str, Union[Dict[str, Dict], Iterable[str]]] = dataclasses.field(default_factory=dict)
+ custom: dict[str, t.Any] = dataclasses.field(default_factory=dict)
"""A place to store engine's custom traits, not related to the SearXNG core.
"""
- def get_language(self, searxng_locale: str, default=None):
+ def get_language(self, searxng_locale: str, default: t.Any = None):
"""Return engine's language string that *best fits* to SearXNG's locale.
:param searxng_locale: SearXNG's internal representation of locale
@@ -102,7 +102,7 @@ class EngineTraits:
return self.all_locale
return locales.get_engine_locale(searxng_locale, self.languages, default=default)
- def get_region(self, searxng_locale: str, default=None):
+ def get_region(self, searxng_locale: str, default: t.Any = None) -> t.Any:
"""Return engine's region string that best fits to SearXNG's locale.
:param searxng_locale: SearXNG's internal representation of locale
@@ -133,10 +133,10 @@ class EngineTraits:
def copy(self):
"""Create a copy of the dataclass object."""
- return EngineTraits(**dataclasses.asdict(self))
+ return EngineTraits(**dataclasses.asdict(self)) # type: ignore
@classmethod
- def fetch_traits(cls, engine: Engine) -> Union['EngineTraits', None]:
+ def fetch_traits(cls, engine: "Engine | types.ModuleType") -> "EngineTraits | None":
"""Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch
and set properties from the origin engine in the object ``engine_traits``. If
function does not exists, ``None`` is returned.
@@ -150,7 +150,7 @@ class EngineTraits:
fetch_traits(engine_traits)
return engine_traits
- def set_traits(self, engine: Engine):
+ def set_traits(self, engine: "Engine | types.ModuleType"):
"""Set traits from self object in a :py:obj:`.Engine` namespace.
:param engine: engine instance build by :py:func:`searx.engines.load_engine`
@@ -161,14 +161,14 @@ class EngineTraits:
else:
raise TypeError('engine traits of type %s is unknown' % self.data_type)
- def _set_traits_v1(self, engine: Engine):
+ def _set_traits_v1(self, engine: "Engine | types.ModuleType"):
# For an engine, when there is `language: ...` in the YAML settings the engine
# does support only this one language (region)::
#
# - name: google italian
# engine: google
# language: it
- # region: it-IT # type: ignore
+ # region: it-IT
traits = self.copy()
@@ -186,16 +186,16 @@ class EngineTraits:
raise ValueError(_msg % (engine.name, 'region', engine.region))
traits.regions = {engine.region: regions[engine.region]}
- engine.language_support = bool(traits.languages or traits.regions)
+ engine.language_support = bool(traits.languages or traits.regions) # type: ignore
# set the copied & modified traits in engine's namespace
- engine.traits = traits
+ engine.traits = traits # pyright: ignore[reportAttributeAccessIssue]
-class EngineTraitsMap(Dict[str, EngineTraits]):
+class EngineTraitsMap(dict[str, EngineTraits]):
"""A python dictionary to map :class:`EngineTraits` by engine name."""
- ENGINE_TRAITS_FILE = (data_dir / 'engine_traits.json').resolve()
+ ENGINE_TRAITS_FILE: pathlib.Path = (data_dir / 'engine_traits.json').resolve()
"""File with persistence of the :py:obj:`EngineTraitsMap`."""
def save_data(self):
@@ -212,7 +212,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
return obj
@classmethod
- def fetch_traits(cls, log: Callable) -> 'EngineTraitsMap':
+ def fetch_traits(cls, log: t.Callable[[str], None]) -> 'EngineTraitsMap':
from searx import engines # pylint: disable=cyclic-import, import-outside-toplevel
names = list(engines.engines)
@@ -220,7 +220,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
obj = cls()
for engine_name in names:
- engine = engines.engines[engine_name]
+ engine: Engine | types.ModuleType = engines.engines[engine_name]
traits = None
# pylint: disable=broad-exception-caught
@@ -242,7 +242,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
return obj
- def set_traits(self, engine: Engine | types.ModuleType):
+ def set_traits(self, engine: "Engine | types.ModuleType"):
"""Set traits in a :py:obj:`Engine` namespace.
:param engine: engine instance build by :py:func:`searx.engines.load_engine`
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-03 19:03:03 +0000