summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.pylintrc1
-rw-r--r--requirements-dev.txt2
-rw-r--r--requirements.txt4
-rw-r--r--searx/engines/command.py2
-rw-r--r--searx/engines/demo_offline.py1
-rw-r--r--searx/engines/redis_server.py4
-rw-r--r--searx/engines/sqlite.py2
-rw-r--r--searx/engines/yggtorrent.py1
-rw-r--r--searx/external_bang.py2
-rw-r--r--searx/locales.py2
-rw-r--r--searx/metrics/error_recorder.py6
-rw-r--r--searx/network/__init__.py8
-rw-r--r--searx/network/client.py105
-rw-r--r--searx/network/network.py13
-rw-r--r--searx/search/processors/online_currency.py2
-rw-r--r--tests/unit/test_external_bangs.py6
16 files changed, 61 insertions, 100 deletions
diff --git a/.pylintrc b/.pylintrc
index 141715258..f2dd716d3 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -62,6 +62,7 @@ confidence=
disable=bad-whitespace,
duplicate-code,
missing-function-docstring,
+ consider-using-f-string,
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
diff --git a/requirements-dev.txt b/requirements-dev.txt
index eb47858ca..606a18f67 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -2,7 +2,7 @@ mock==4.0.3
nose2[coverage_plugin]==0.10.0
cov-core==1.15.0
pycodestyle==2.7.0
-pylint==2.10.2
+pylint==2.11.1
splinter==0.15.0
selenium==3.141.0
twine==3.4.2
diff --git a/requirements.txt b/requirements.txt
index 371883b3e..b38aedcb4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,10 +7,10 @@ lxml==4.6.3
pygments==2.10.0
python-dateutil==2.8.2
pyyaml==5.4.1
-httpx[http2]==0.17.1
+httpx[http2]==0.19.0
Brotli==1.0.9
uvloop==0.16.0; python_version >= '3.7'
uvloop==0.14.0; python_version < '3.7'
-httpx-socks[asyncio]==0.3.1
+httpx-socks[asyncio]==0.4.1
langdetect==1.0.9
setproctitle==1.2.2
diff --git a/searx/engines/command.py b/searx/engines/command.py
index 33270d245..aca379c67 100644
--- a/searx/engines/command.py
+++ b/searx/engines/command.py
@@ -35,7 +35,7 @@ def init(engine_settings):
if 'command' not in engine_settings:
raise ValueError('engine command : missing configuration key: command')
- global command, working_dir, result_template, delimiter, parse_regex, timeout, environment_variables
+ global command, working_dir, delimiter, parse_regex, environment_variables
command = engine_settings['command']
diff --git a/searx/engines/demo_offline.py b/searx/engines/demo_offline.py
index 06609d2c3..a4a632180 100644
--- a/searx/engines/demo_offline.py
+++ b/searx/engines/demo_offline.py
@@ -56,7 +56,6 @@ def search(query, request_params):
results.
"""
- global _my_offline_engine # pylint: disable=global-statement
ret_val = []
result_list = json.loads(_my_offline_engine)
diff --git a/searx/engines/redis_server.py b/searx/engines/redis_server.py
index a48f0775b..f9726033d 100644
--- a/searx/engines/redis_server.py
+++ b/searx/engines/redis_server.py
@@ -31,8 +31,6 @@ def init(_engine_settings):
)
def search(query, _params):
- global _redis_client # pylint: disable=global-statement
-
if not exact_match_only:
return search_keys(query)
@@ -55,8 +53,6 @@ def search(query, _params):
return []
def search_keys(query):
- global _redis_client # pylint: disable=global-statement
-
ret = []
for key in _redis_client.scan_iter(
match='*{}*'.format(query)
diff --git a/searx/engines/sqlite.py b/searx/engines/sqlite.py
index 292448602..43a85efbb 100644
--- a/searx/engines/sqlite.py
+++ b/searx/engines/sqlite.py
@@ -35,7 +35,6 @@ def sqlite_cursor():
* https://docs.python.org/3/library/sqlite3.html#sqlite3.connect
* https://www.sqlite.org/uri.html
"""
- global database # pylint: disable=global-statement
uri = 'file:' + database + '?mode=ro'
with contextlib.closing(sqlite3.connect(uri, uri=True)) as connect:
connect.row_factory = sqlite3.Row
@@ -44,7 +43,6 @@ def sqlite_cursor():
def search(query, params):
- global query_str, result_template # pylint: disable=global-statement
results = []
query_params = {
diff --git a/searx/engines/yggtorrent.py b/searx/engines/yggtorrent.py
index ca008c472..31a0408b8 100644
--- a/searx/engines/yggtorrent.py
+++ b/searx/engines/yggtorrent.py
@@ -37,7 +37,6 @@ cookies = dict()
def init(engine_settings=None):
- global cookies
# initial cookies
resp = http_get(url, allow_redirects=False)
if resp.ok:
diff --git a/searx/external_bang.py b/searx/external_bang.py
index 8798a0a65..f8e872693 100644
--- a/searx/external_bang.py
+++ b/searx/external_bang.py
@@ -45,7 +45,6 @@ def resolve_bang_definition(bang_definition, query):
def get_bang_definition_and_autocomplete(bang, external_bangs_db=None):
- global EXTERNAL_BANGS
if external_bangs_db is None:
external_bangs_db = EXTERNAL_BANGS
@@ -78,7 +77,6 @@ def get_bang_url(search_query, external_bangs_db=None):
:param search_query: This is a search_query object which contains preferences and the submitted queries.
:return: None if the bang was invalid, else a string of the redirect url.
"""
- global EXTERNAL_BANGS
if external_bangs_db is None:
external_bangs_db = EXTERNAL_BANGS
diff --git a/searx/locales.py b/searx/locales.py
index 80defceb7..42d5854c2 100644
--- a/searx/locales.py
+++ b/searx/locales.py
@@ -57,7 +57,7 @@ def initialize_locales(directory):
"""Initialize global names :py:obj:`LOCALE_NAMES`, :py:obj:`UI_LOCALE_CODES` and
:py:obj:`RTL_LOCALES`.
"""
- global LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES # pylint: disable=global-statement
+ global UI_LOCALE_CODES # pylint: disable=global-statement
for dirname in sorted(os.listdir(directory)):
# Based on https://flask-babel.tkte.ch/_modules/flask_babel.html#Babel.list_translations
if not os.path.isdir( os.path.join(directory, dirname, 'LC_MESSAGES') ):
diff --git a/searx/metrics/error_recorder.py b/searx/metrics/error_recorder.py
index 6963cda2f..37594e5e8 100644
--- a/searx/metrics/error_recorder.py
+++ b/searx/metrics/error_recorder.py
@@ -74,9 +74,11 @@ def get_request_exception_messages(exc: HTTPError)\
status_code = None
reason = None
hostname = None
- if hasattr(exc, 'request') and exc.request is not None:
+ if hasattr(exc, '_request') and exc._request is not None:
+ # exc.request is property that raise an RuntimeException
+ # if exc._request is not defined.
url = exc.request.url
- if url is None and hasattr(exc, 'response') and exc.respones is not None:
+ if url is None and hasattr(exc, 'response') and exc.response is not None:
url = exc.response.url
if url is not None:
hostname = url.host
diff --git a/searx/network/__init__.py b/searx/network/__init__.py
index 21c4c27b5..260d4f105 100644
--- a/searx/network/__init__.py
+++ b/searx/network/__init__.py
@@ -43,24 +43,20 @@ THREADLOCAL = threading.local()
"""Thread-local data is data for thread specific values."""
def reset_time_for_thread():
- global THREADLOCAL
THREADLOCAL.total_time = 0
def get_time_for_thread():
"""returns thread's total time or None"""
- global THREADLOCAL
return THREADLOCAL.__dict__.get('total_time')
def set_timeout_for_thread(timeout, start_time=None):
- global THREADLOCAL
THREADLOCAL.timeout = timeout
THREADLOCAL.start_time = start_time
def set_context_network_name(network_name):
- global THREADLOCAL
THREADLOCAL.network = get_network(network_name)
@@ -69,13 +65,11 @@ def get_context_network():
If unset, return value from :py:obj:`get_network`.
"""
- global THREADLOCAL
return THREADLOCAL.__dict__.get('network') or get_network()
def request(method, url, **kwargs):
"""same as requests/requests/api.py request(...)"""
- global THREADLOCAL
time_before_request = default_timer()
# timeout (httpx)
@@ -172,7 +166,7 @@ async def stream_chunk_to_queue(network, queue, method, url, **kwargs):
async for chunk in response.aiter_raw(65536):
if len(chunk) > 0:
queue.put(chunk)
- except httpx.ResponseClosed:
+ except httpx.StreamClosed:
# the response was queued before the exception.
# the exception was raised on aiter_raw.
# we do nothing here: in the finally block, None will be queued
diff --git a/searx/network/client.py b/searx/network/client.py
index e1abff05a..925c0fdd3 100644
--- a/searx/network/client.py
+++ b/searx/network/client.py
@@ -5,6 +5,7 @@
import asyncio
import logging
import threading
+
import httpcore
import httpx
from httpx_socks import AsyncProxyTransport
@@ -26,19 +27,22 @@ else:
uvloop.install()
-logger = logger.getChild('searx.http.client')
+logger = logger.getChild('searx.network.client')
LOOP = None
SSLCONTEXTS = {}
TRANSPORT_KWARGS = {
- 'backend': 'asyncio',
+ # use anyio :
+ # * https://github.com/encode/httpcore/issues/344
+ # * https://github.com/encode/httpx/discussions/1511
+ 'backend': 'anyio',
'trust_env': False,
}
# pylint: disable=protected-access
async def close_connections_for_url(
- connection_pool: httpcore.AsyncConnectionPool,
- url: httpcore._utils.URL ):
+ connection_pool: httpcore.AsyncConnectionPool, url: httpcore._utils.URL
+):
origin = httpcore._utils.url_to_origin(url)
logger.debug('Drop connections for %r', origin)
@@ -47,97 +51,93 @@ async def close_connections_for_url(
await connection_pool._remove_from_pool(connection)
try:
await connection.aclose()
- except httpcore.NetworkError as e:
+ except httpx.NetworkError as e:
logger.warning('Error closing an existing connection', exc_info=e)
# pylint: enable=protected-access
def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False):
- global SSLCONTEXTS
key = (proxy_url, cert, verify, trust_env, http2)
if key not in SSLCONTEXTS:
SSLCONTEXTS[key] = httpx.create_ssl_context(cert, verify, trust_env, http2)
return SSLCONTEXTS[key]
-class AsyncHTTPTransportNoHttp(httpcore.AsyncHTTPTransport):
+class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
"""Block HTTP request"""
- async def arequest(self, method, url, headers=None, stream=None, ext=None):
- raise httpcore.UnsupportedProtocol("HTTP protocol is disabled")
+ async def handle_async_request(
+ self, method, url, headers=None, stream=None, extensions=None
+ ):
+ raise httpx.UnsupportedProtocol('HTTP protocol is disabled')
class AsyncProxyTransportFixed(AsyncProxyTransport):
"""Fix httpx_socks.AsyncProxyTransport
- Map python_socks exceptions to httpcore.ProxyError
-
- Map socket.gaierror to httpcore.ConnectError
+ Map python_socks exceptions to httpx.ProxyError / httpx.ConnectError
- Note: keepalive_expiry is ignored, AsyncProxyTransport should call:
- * self._keepalive_sweep()
- * self._response_closed(self, connection)
+ Map socket.gaierror to httpx.ConnectError
Note: AsyncProxyTransport inherit from AsyncConnectionPool
-
- Note: the API is going to change on httpx 0.18.0
- see https://github.com/encode/httpx/pull/1522
"""
- async def arequest(self, method, url, headers=None, stream=None, ext=None):
+ async def handle_async_request(
+ self, method, url, headers=None, stream=None, extensions=None
+ ):
retry = 2
while retry > 0:
retry -= 1
try:
- return await super().arequest(method, url, headers, stream, ext)
+ return await super().handle_async_request(
+ method, url, headers=headers, stream=stream, extensions=extensions
+ )
except (ProxyConnectionError, ProxyTimeoutError, ProxyError) as e:
- raise httpcore.ProxyError(e)
+ raise httpx.ProxyError from e
except OSError as e:
# socket.gaierror when DNS resolution fails
- raise httpcore.NetworkError(e)
- except httpcore.RemoteProtocolError as e:
- # in case of httpcore.RemoteProtocolError: Server disconnected
- await close_connections_for_url(self, url)
- logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
- # retry
- except (httpcore.NetworkError, httpcore.ProtocolError) as e:
- # httpcore.WriteError on HTTP/2 connection leaves a new opened stream
+ raise httpx.ConnectError from e
+ except httpx.NetworkError as e:
+ # httpx.WriteError on HTTP/2 connection leaves a new opened stream
# then each new request creates a new stream and raise the same WriteError
await close_connections_for_url(self, url)
raise e
+ except httpx.RemoteProtocolError as e:
+ # in case of httpx.RemoteProtocolError: Server disconnected
+ await close_connections_for_url(self, url)
+ logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
+ # retry
class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport):
"""Fix httpx.AsyncHTTPTransport"""
- async def arequest(self, method, url, headers=None, stream=None, ext=None):
+ async def handle_async_request(
+ self, method, url, headers=None, stream=None, extensions=None
+ ):
retry = 2
while retry > 0:
retry -= 1
try:
- return await super().arequest(method, url, headers, stream, ext)
+ return await super().handle_async_request(
+ method, url, headers=headers, stream=stream, extensions=extensions
+ )
except OSError as e:
# socket.gaierror when DNS resolution fails
- raise httpcore.ConnectError(e)
- except httpcore.CloseError as e:
- # httpcore.CloseError: [Errno 104] Connection reset by peer
- # raised by _keepalive_sweep()
- # from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198 # pylint: disable=line-too-long
+ raise httpx.ConnectError from e
+ except httpx.NetworkError as e:
+ # httpx.WriteError on HTTP/2 connection leaves a new opened stream
+ # then each new request creates a new stream and raise the same WriteError
await close_connections_for_url(self._pool, url)
- logger.warning('httpcore.CloseError: retry', exc_info=e)
- # retry
- except httpcore.RemoteProtocolError as e:
- # in case of httpcore.RemoteProtocolError: Server disconnected
+ raise e
+ except httpx.RemoteProtocolError as e:
+ # in case of httpx.RemoteProtocolError: Server disconnected
await close_connections_for_url(self._pool, url)
- logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
+ logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
# retry
- except (httpcore.ProtocolError, httpcore.NetworkError) as e:
- await close_connections_for_url(self._pool, url)
- raise e
def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit, retries):
- global TRANSPORT_KWARGS
# support socks5h (requests compatibility):
# https://requests.readthedocs.io/en/master/user/advanced/#socks
# socks5:// hostname is resolved on client side
@@ -167,7 +167,6 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
def get_transport(verify, http2, local_address, proxy_url, limit, retries):
- global TRANSPORT_KWARGS
verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify
return AsyncHTTPTransportFixed(
# pylint: disable=protected-access
@@ -181,15 +180,6 @@ def get_transport(verify, http2, local_address, proxy_url, limit, retries):
)
-def iter_proxies(proxies):
- # https://www.python-httpx.org/compatibility/#proxy-keys
- if isinstance(proxies, str):
- yield 'all://', proxies
- elif isinstance(proxies, dict):
- for pattern, proxy_url in proxies.items():
- yield pattern, proxy_url
-
-
def new_client(
# pylint: disable=too-many-arguments
enable_http, verify, enable_http2,
@@ -202,8 +192,8 @@ def new_client(
)
# See https://www.python-httpx.org/advanced/#routing
mounts = {}
- for pattern, proxy_url in iter_proxies(proxies):
- if not enable_http and (pattern == 'http' or pattern.startswith('http://')):
+ for pattern, proxy_url in proxies.items():
+ if not enable_http and pattern.startswith('http://'):
continue
if (proxy_url.startswith('socks4://')
or proxy_url.startswith('socks5://')
@@ -235,7 +225,6 @@ def new_client(
def get_loop():
- global LOOP
return LOOP
diff --git a/searx/network/network.py b/searx/network/network.py
index 9954f0507..d58070f18 100644
--- a/searx/network/network.py
+++ b/searx/network/network.py
@@ -138,12 +138,10 @@ class Network:
request = response.request
status = f"{response.status_code} {response.reason_phrase}"
response_line = f"{response.http_version} {status}"
- if hasattr(response, "_elapsed"):
- elapsed_time = f"{response.elapsed.total_seconds()} sec"
- else:
- elapsed_time = "stream"
+ content_type = response.headers.get("Content-Type")
+ content_type = f' ({content_type})' if content_type else ''
self._logger.debug(
- f'HTTP Request: {request.method} {request.url} "{response_line}" ({elapsed_time})'
+ f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}'
)
def get_client(self, verify=None, max_redirects=None):
@@ -225,12 +223,10 @@ class Network:
@classmethod
async def aclose_all(cls):
- global NETWORKS
await asyncio.gather(*[network.aclose() for network in NETWORKS.values()], return_exceptions=False)
def get_network(name=None):
- global NETWORKS
return NETWORKS.get(name or DEFAULT_NAME)
@@ -240,8 +236,6 @@ def initialize(settings_engines=None, settings_outgoing=None):
from searx import settings
# pylint: enable=import-outside-toplevel)
- global NETWORKS
-
settings_engines = settings_engines or settings['engines']
settings_outgoing = settings_outgoing or settings['outgoing']
@@ -328,7 +322,6 @@ def done():
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register
"""
- global NETWORKS
try:
loop = get_loop()
if loop:
diff --git a/searx/search/processors/online_currency.py b/searx/search/processors/online_currency.py
index 3213a11e5..4e5c57264 100644
--- a/searx/search/processors/online_currency.py
+++ b/searx/search/processors/online_currency.py
@@ -18,7 +18,6 @@ def normalize_name(name):
return unicodedata.normalize('NFKD', name).lower()
def name_to_iso4217(name):
- global CURRENCIES # pylint: disable=global-statement
name = normalize_name(name)
currency = CURRENCIES['names'].get(name, [name])
if isinstance(currency, str):
@@ -26,7 +25,6 @@ def name_to_iso4217(name):
return currency[0]
def iso4217_to_name(iso4217, language):
- global CURRENCIES # pylint: disable=global-statement
return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)
class OnlineCurrencyProcessor(OnlineProcessor):
diff --git a/tests/unit/test_external_bangs.py b/tests/unit/test_external_bangs.py
index f320d3037..68b3b5a78 100644
--- a/tests/unit/test_external_bangs.py
+++ b/tests/unit/test_external_bangs.py
@@ -72,31 +72,26 @@ class TestResolveBangDefinition(SearxTestCase):
class TestGetBangDefinitionAndAutocomplete(SearxTestCase):
def test_found(self):
- global TEST_DB
bang_definition, new_autocomplete = get_bang_definition_and_autocomplete('exam', external_bangs_db=TEST_DB)
self.assertEqual(bang_definition, TEST_DB['trie']['exam']['*'])
self.assertEqual(new_autocomplete, ['example'])
def test_found_optimized(self):
- global TEST_DB
bang_definition, new_autocomplete = get_bang_definition_and_autocomplete('example', external_bangs_db=TEST_DB)
self.assertEqual(bang_definition, TEST_DB['trie']['exam']['ple'])
self.assertEqual(new_autocomplete, [])
def test_partial(self):
- global TEST_DB
bang_definition, new_autocomplete = get_bang_definition_and_autocomplete('examp', external_bangs_db=TEST_DB)
self.assertEqual(bang_definition, None)
self.assertEqual(new_autocomplete, ['example'])
def test_partial2(self):
- global TEST_DB
bang_definition, new_autocomplete = get_bang_definition_and_autocomplete('sea', external_bangs_db=TEST_DB)
self.assertEqual(bang_definition, TEST_DB['trie']['sea']['*'])
self.assertEqual(new_autocomplete, ['search', 'searching', 'seascapes', 'season'])
def test_error(self):
- global TEST_DB
bang_definition, new_autocomplete = get_bang_definition_and_autocomplete('error', external_bangs_db=TEST_DB)
self.assertEqual(bang_definition, None)
self.assertEqual(new_autocomplete, [])
@@ -114,7 +109,6 @@ class TestExternalBangJson(SearxTestCase):
self.assertEqual(result, None)
def test_get_bang_url(self):
- global TEST_DB
url = get_bang_url(SearchQuery('test', engineref_list=[], external_bang='example'), external_bangs_db=TEST_DB)
self.assertEqual(url, 'https://example.com/test')
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-01 18:44:40 +0000