summaryrefslogtreecommitdiff
path: root/searx/network
diff options
context:
space:
mode:
Diffstat (limited to 'searx/network')
-rw-r--r--searx/network/__init__.py8
-rw-r--r--searx/network/client.py105
-rw-r--r--searx/network/network.py13
3 files changed, 51 insertions, 75 deletions
diff --git a/searx/network/__init__.py b/searx/network/__init__.py
index 21c4c27b5..260d4f105 100644
--- a/searx/network/__init__.py
+++ b/searx/network/__init__.py
@@ -43,24 +43,20 @@ THREADLOCAL = threading.local()
"""Thread-local data is data for thread specific values."""
def reset_time_for_thread():
- global THREADLOCAL
THREADLOCAL.total_time = 0
def get_time_for_thread():
"""returns thread's total time or None"""
- global THREADLOCAL
return THREADLOCAL.__dict__.get('total_time')
def set_timeout_for_thread(timeout, start_time=None):
- global THREADLOCAL
THREADLOCAL.timeout = timeout
THREADLOCAL.start_time = start_time
def set_context_network_name(network_name):
- global THREADLOCAL
THREADLOCAL.network = get_network(network_name)
@@ -69,13 +65,11 @@ def get_context_network():
If unset, return value from :py:obj:`get_network`.
"""
- global THREADLOCAL
return THREADLOCAL.__dict__.get('network') or get_network()
def request(method, url, **kwargs):
"""same as requests/requests/api.py request(...)"""
- global THREADLOCAL
time_before_request = default_timer()
# timeout (httpx)
@@ -172,7 +166,7 @@ async def stream_chunk_to_queue(network, queue, method, url, **kwargs):
async for chunk in response.aiter_raw(65536):
if len(chunk) > 0:
queue.put(chunk)
- except httpx.ResponseClosed:
+ except httpx.StreamClosed:
# the response was queued before the exception.
# the exception was raised on aiter_raw.
# we do nothing here: in the finally block, None will be queued
diff --git a/searx/network/client.py b/searx/network/client.py
index e1abff05a..925c0fdd3 100644
--- a/searx/network/client.py
+++ b/searx/network/client.py
@@ -5,6 +5,7 @@
import asyncio
import logging
import threading
+
import httpcore
import httpx
from httpx_socks import AsyncProxyTransport
@@ -26,19 +27,22 @@ else:
uvloop.install()
-logger = logger.getChild('searx.http.client')
+logger = logger.getChild('searx.network.client')
LOOP = None
SSLCONTEXTS = {}
TRANSPORT_KWARGS = {
- 'backend': 'asyncio',
+ # use anyio :
+ # * https://github.com/encode/httpcore/issues/344
+ # * https://github.com/encode/httpx/discussions/1511
+ 'backend': 'anyio',
'trust_env': False,
}
# pylint: disable=protected-access
async def close_connections_for_url(
- connection_pool: httpcore.AsyncConnectionPool,
- url: httpcore._utils.URL ):
+ connection_pool: httpcore.AsyncConnectionPool, url: httpcore._utils.URL
+):
origin = httpcore._utils.url_to_origin(url)
logger.debug('Drop connections for %r', origin)
@@ -47,97 +51,93 @@ async def close_connections_for_url(
await connection_pool._remove_from_pool(connection)
try:
await connection.aclose()
- except httpcore.NetworkError as e:
+ except httpx.NetworkError as e:
logger.warning('Error closing an existing connection', exc_info=e)
# pylint: enable=protected-access
def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False):
- global SSLCONTEXTS
key = (proxy_url, cert, verify, trust_env, http2)
if key not in SSLCONTEXTS:
SSLCONTEXTS[key] = httpx.create_ssl_context(cert, verify, trust_env, http2)
return SSLCONTEXTS[key]
-class AsyncHTTPTransportNoHttp(httpcore.AsyncHTTPTransport):
+class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
"""Block HTTP request"""
- async def arequest(self, method, url, headers=None, stream=None, ext=None):
- raise httpcore.UnsupportedProtocol("HTTP protocol is disabled")
+ async def handle_async_request(
+ self, method, url, headers=None, stream=None, extensions=None
+ ):
+ raise httpx.UnsupportedProtocol('HTTP protocol is disabled')
class AsyncProxyTransportFixed(AsyncProxyTransport):
"""Fix httpx_socks.AsyncProxyTransport
- Map python_socks exceptions to httpcore.ProxyError
-
- Map socket.gaierror to httpcore.ConnectError
+ Map python_socks exceptions to httpx.ProxyError / httpx.ConnectError
- Note: keepalive_expiry is ignored, AsyncProxyTransport should call:
- * self._keepalive_sweep()
- * self._response_closed(self, connection)
+ Map socket.gaierror to httpx.ConnectError
Note: AsyncProxyTransport inherit from AsyncConnectionPool
-
- Note: the API is going to change on httpx 0.18.0
- see https://github.com/encode/httpx/pull/1522
"""
- async def arequest(self, method, url, headers=None, stream=None, ext=None):
+ async def handle_async_request(
+ self, method, url, headers=None, stream=None, extensions=None
+ ):
retry = 2
while retry > 0:
retry -= 1
try:
- return await super().arequest(method, url, headers, stream, ext)
+ return await super().handle_async_request(
+ method, url, headers=headers, stream=stream, extensions=extensions
+ )
except (ProxyConnectionError, ProxyTimeoutError, ProxyError) as e:
- raise httpcore.ProxyError(e)
+ raise httpx.ProxyError from e
except OSError as e:
# socket.gaierror when DNS resolution fails
- raise httpcore.NetworkError(e)
- except httpcore.RemoteProtocolError as e:
- # in case of httpcore.RemoteProtocolError: Server disconnected
- await close_connections_for_url(self, url)
- logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
- # retry
- except (httpcore.NetworkError, httpcore.ProtocolError) as e:
- # httpcore.WriteError on HTTP/2 connection leaves a new opened stream
+ raise httpx.ConnectError from e
+ except httpx.NetworkError as e:
+ # httpx.WriteError on HTTP/2 connection leaves a new opened stream
# then each new request creates a new stream and raise the same WriteError
await close_connections_for_url(self, url)
raise e
+ except httpx.RemoteProtocolError as e:
+ # in case of httpx.RemoteProtocolError: Server disconnected
+ await close_connections_for_url(self, url)
+ logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
+ # retry
class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport):
"""Fix httpx.AsyncHTTPTransport"""
- async def arequest(self, method, url, headers=None, stream=None, ext=None):
+ async def handle_async_request(
+ self, method, url, headers=None, stream=None, extensions=None
+ ):
retry = 2
while retry > 0:
retry -= 1
try:
- return await super().arequest(method, url, headers, stream, ext)
+ return await super().handle_async_request(
+ method, url, headers=headers, stream=stream, extensions=extensions
+ )
except OSError as e:
# socket.gaierror when DNS resolution fails
- raise httpcore.ConnectError(e)
- except httpcore.CloseError as e:
- # httpcore.CloseError: [Errno 104] Connection reset by peer
- # raised by _keepalive_sweep()
- # from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198 # pylint: disable=line-too-long
+ raise httpx.ConnectError from e
+ except httpx.NetworkError as e:
+ # httpx.WriteError on HTTP/2 connection leaves a new opened stream
+ # then each new request creates a new stream and raise the same WriteError
await close_connections_for_url(self._pool, url)
- logger.warning('httpcore.CloseError: retry', exc_info=e)
- # retry
- except httpcore.RemoteProtocolError as e:
- # in case of httpcore.RemoteProtocolError: Server disconnected
+ raise e
+ except httpx.RemoteProtocolError as e:
+ # in case of httpx.RemoteProtocolError: Server disconnected
await close_connections_for_url(self._pool, url)
- logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
+ logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
# retry
- except (httpcore.ProtocolError, httpcore.NetworkError) as e:
- await close_connections_for_url(self._pool, url)
- raise e
def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit, retries):
- global TRANSPORT_KWARGS
# support socks5h (requests compatibility):
# https://requests.readthedocs.io/en/master/user/advanced/#socks
# socks5:// hostname is resolved on client side
@@ -167,7 +167,6 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
def get_transport(verify, http2, local_address, proxy_url, limit, retries):
- global TRANSPORT_KWARGS
verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify
return AsyncHTTPTransportFixed(
# pylint: disable=protected-access
@@ -181,15 +180,6 @@ def get_transport(verify, http2, local_address, proxy_url, limit, retries):
)
-def iter_proxies(proxies):
- # https://www.python-httpx.org/compatibility/#proxy-keys
- if isinstance(proxies, str):
- yield 'all://', proxies
- elif isinstance(proxies, dict):
- for pattern, proxy_url in proxies.items():
- yield pattern, proxy_url
-
-
def new_client(
# pylint: disable=too-many-arguments
enable_http, verify, enable_http2,
@@ -202,8 +192,8 @@ def new_client(
)
# See https://www.python-httpx.org/advanced/#routing
mounts = {}
- for pattern, proxy_url in iter_proxies(proxies):
- if not enable_http and (pattern == 'http' or pattern.startswith('http://')):
+ for pattern, proxy_url in proxies.items():
+ if not enable_http and pattern.startswith('http://'):
continue
if (proxy_url.startswith('socks4://')
or proxy_url.startswith('socks5://')
@@ -235,7 +225,6 @@ def new_client(
def get_loop():
- global LOOP
return LOOP
diff --git a/searx/network/network.py b/searx/network/network.py
index 9954f0507..d58070f18 100644
--- a/searx/network/network.py
+++ b/searx/network/network.py
@@ -138,12 +138,10 @@ class Network:
request = response.request
status = f"{response.status_code} {response.reason_phrase}"
response_line = f"{response.http_version} {status}"
- if hasattr(response, "_elapsed"):
- elapsed_time = f"{response.elapsed.total_seconds()} sec"
- else:
- elapsed_time = "stream"
+ content_type = response.headers.get("Content-Type")
+ content_type = f' ({content_type})' if content_type else ''
self._logger.debug(
- f'HTTP Request: {request.method} {request.url} "{response_line}" ({elapsed_time})'
+ f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}'
)
def get_client(self, verify=None, max_redirects=None):
@@ -225,12 +223,10 @@ class Network:
@classmethod
async def aclose_all(cls):
- global NETWORKS
await asyncio.gather(*[network.aclose() for network in NETWORKS.values()], return_exceptions=False)
def get_network(name=None):
- global NETWORKS
return NETWORKS.get(name or DEFAULT_NAME)
@@ -240,8 +236,6 @@ def initialize(settings_engines=None, settings_outgoing=None):
from searx import settings
# pylint: enable=import-outside-toplevel)
- global NETWORKS
-
settings_engines = settings_engines or settings['engines']
settings_outgoing = settings_outgoing or settings['outgoing']
@@ -328,7 +322,6 @@ def done():
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register
"""
- global NETWORKS
try:
loop = get_loop()
if loop:
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-05 05:41:25 +0000