From 8c1a65d32fb6a0859c0052d668d01f08325f11ad Mon Sep 17 00:00:00 2001
From: Alexandre Flament <alex@al-f.net>
Date: Wed, 5 May 2021 13:08:54 +0200
Subject: [mod] multithreading only in searx.search.* packages

it prepares the new architecture change,
everything about multithreading in moved in the searx.search.* packages

previously the call to the "init" function of the engines was done in searx.engines:
* the network was not set (request not sent using the defined proxy)
* it requires to monkey patch the code to avoid HTTP requests during the tests
---
 searx/search/processors/__init__.py | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

(limited to 'searx/search/processors/__init__.py')

diff --git a/searx/search/processors/__init__.py b/searx/search/processors/__init__.py
index caac74e65..d5ebdb70c 100644
--- a/searx/search/processors/__init__.py
+++ b/searx/search/processors/__init__.py
@@ -11,9 +11,11 @@ __all__ = [
     'OnlineProcessor',
     'OnlineDictionaryProcessor',
     'OnlineCurrencyProcessor',
-    'processors',
+    'PROCESSORS',
 ]
 
+import threading
+
 from searx import logger
 import searx.engines as engines
 
@@ -24,7 +26,7 @@ from .online_currency import OnlineCurrencyProcessor
 from .abstract import EngineProcessor
 
 logger = logger.getChild('search.processors')
-processors = {}
+PROCESSORS = {}
 """Cache request processores, stored by *engine-name* (:py:func:`initialize`)"""
 
 def get_processor_class(engine_type):
@@ -34,6 +36,7 @@ def get_processor_class(engine_type):
             return c
     return None
 
+
 def get_processor(engine, engine_name):
     """Return processor instance that fits to ``engine.engine.type``)"""
     engine_type = getattr(engine, 'engine_type', 'online')
@@ -42,12 +45,26 @@ def get_processor(engine, engine_name):
         return processor_class(engine, engine_name)
     return None
 
+
+def initialize_processor(processor):
+    """Initialize one processor
+
+    Call the init function of the engine
+    """
+    if processor.has_initialize_function:
+        t = threading.Thread(target=processor.initialize, daemon=True)
+        t.start()
+
+
 def initialize(engine_list):
-    """Initialize all engines and store a processor for each engine in :py:obj:`processors`."""
-    engines.initialize_engines(engine_list)
-    for engine_name, engine in engines.engines.items():
-        processor = get_processor(engine, engine_name)
-        if processor is None:
-            logger.error('Error get processor for engine %s', engine_name)
-        else:
-            processors[engine_name] = processor
+    """Initialize all engines and store a processor for each engine in :py:obj:`PROCESSORS`."""
+    for engine_data in engine_list:
+        engine_name = engine_data['name']
+        engine = engines.engines.get(engine_name)
+        if engine:
+            processor = get_processor(engine, engine_name)
+            initialize_processor(processor)
+            if processor is None:
+                logger.error('Error get processor for engine %s', engine_name)
+            else:
+                PROCESSORS[engine_name] = processor
-- 
cgit v1.2.3