From 8c1a65d32fb6a0859c0052d668d01f08325f11ad Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 5 May 2021 13:08:54 +0200 Subject: [mod] multithreading only in searx.search.* packages it prepares the new architecture change, everything about multithreading in moved in the searx.search.* packages previously the call to the "init" function of the engines was done in searx.engines: * the network was not set (request not sent using the defined proxy) * it requires to monkey patch the code to avoid HTTP requests during the tests --- searx/search/processors/online.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'searx/search/processors/online.py') diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index 93a9c6cbf..48a514e8a 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -5,7 +5,7 @@ """ -from time import time +from timeit import default_timer import asyncio import httpx @@ -40,6 +40,15 @@ class OnlineProcessor(EngineProcessor): engine_type = 'online' + def initialize(self): + # set timeout for all HTTP requests + searx.network.set_timeout_for_thread(self.engine.timeout, start_time=default_timer()) + # reset the HTTP total time + searx.network.reset_time_for_thread() + # set the network + searx.network.set_context_network_name(self.engine_name) + super().initialize() + def get_params(self, search_query, engine_category): params = super().get_params(search_query, engine_category) if params is None: @@ -139,7 +148,7 @@ class OnlineProcessor(EngineProcessor): self.handle_exception(result_container, e, suspend=True) logger.error("engine {0} : HTTP requests timeout" "(search duration : {1} s, timeout: {2} s) : {3}" - .format(self.engine_name, time() - start_time, + .format(self.engine_name, default_timer() - start_time, timeout_limit, e.__class__.__name__)) except (httpx.HTTPError, httpx.StreamError) as e: @@ -147,7 +156,7 @@ class OnlineProcessor(EngineProcessor): self.handle_exception(result_container, e, suspend=True) logger.exception("engine {0} : requests exception" "(search duration : {1} s, timeout: {2} s) : {3}" - .format(self.engine_name, time() - start_time, + .format(self.engine_name, default_timer() - start_time, timeout_limit, e)) except SearxEngineCaptchaException as e: -- cgit v1.2.3