Merge pull request #1 from metasearch-lab/httpx_networks

Httpx networks

19 files changed, 28 insertions, 31 deletions

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index 2238ea1b9..95eda6dde 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -27,7 +27,7 @@ from searx import settings
 from searx import logger
 from searx.data import ENGINES_LANGUAGES
 from searx.exceptions import SearxEngineResponseException
-from searx.poolrequests import get, get_proxy_cycles
+from searx.network import get, initialize as initialize_network, set_context_network_name
 from searx.utils import load_module, match_language, get_engine_from_settings, gen_useragent
 
 
@@ -89,8 +89,6 @@ def load_engine(engine_data):
                 engine.categories = []
             else:
                 engine.categories = list(map(str.strip, param_value.split(',')))
-        elif param_name == 'proxies':
-            engine.proxies = get_proxy_cycles(param_value)
         else:
             setattr(engine, param_name, param_value)
 
@@ -289,9 +287,11 @@ def load_engines(engine_list):
 
 def initialize_engines(engine_list):
     load_engines(engine_list)
+    initialize_network(engine_list, settings['outgoing'])
 
     def engine_init(engine_name, init_fn):
         try:
+            set_context_network_name(engine_name)
             init_fn(get_engine_from_settings(engine_name))
         except SearxEngineResponseException as exc:
             logger.warn('%s engine: Fail to initialize // %s', engine_name, exc)
diff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py
index 2483c0805..eaa8b6ab4 100644
--- a/searx/engines/dictzone.py
+++ b/searx/engines/dictzone.py
@@ -52,7 +52,7 @@ def response(resp):
                 to_results.append(to_result.text_content())
 
         results.append({
-            'url': urljoin(resp.url, '?%d' % k),
+            'url': urljoin(str(resp.url), '?%d' % k),
             'title': from_result.text_content(),
             'content': '; '.join(to_results)
         })
diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
index ae1e36686..3c086f81b 100644
--- a/searx/engines/duckduckgo.py
+++ b/searx/engines/duckduckgo.py
@@ -6,7 +6,7 @@
 from lxml.html import fromstring
 from json import loads
 from searx.utils import extract_text, match_language, eval_xpath, dict_subset
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py
index 305eb1ca1..0daaf41e9 100644
--- a/searx/engines/duckduckgo_images.py
+++ b/searx/engines/duckduckgo_images.py
@@ -8,7 +8,7 @@ from urllib.parse import urlencode
 from searx.exceptions import SearxEngineAPIException
 from searx.engines.duckduckgo import get_region_code
 from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/elasticsearch.py b/searx/engines/elasticsearch.py
index da7f98074..db84a5c13 100644
--- a/searx/engines/elasticsearch.py
+++ b/searx/engines/elasticsearch.py
@@ -4,7 +4,6 @@
 """
 
 from json import loads, dumps
-from requests.auth import HTTPBasicAuth
 from searx.exceptions import SearxEngineAPIException
 
 
@@ -32,7 +31,7 @@ def request(query, params):
         return params
 
     if username and password:
-        params['auth'] = HTTPBasicAuth(username, password)
+        params['auth'] = (username, password)
 
     params['url'] = search_url
     params['method'] = 'GET'
diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py
index 248991df9..bbd9e20d2 100644
--- a/searx/engines/gigablast.py
+++ b/searx/engines/gigablast.py
@@ -8,7 +8,7 @@ import re
 from json import loads
 from urllib.parse import urlencode
 # from searx import logger
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/google.py b/searx/engines/google.py
index 8c20029a3..dcb65df57 100644
--- a/searx/engines/google.py
+++ b/searx/engines/google.py
@@ -10,7 +10,7 @@ Definitions`_.
 
 # pylint: disable=invalid-name, missing-function-docstring
 
-from urllib.parse import urlencode, urlparse
+from urllib.parse import urlencode
 from lxml import html
 from searx import logger
 from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
@@ -186,8 +186,7 @@ def get_lang_info(params, lang_list, custom_aliases):
     return ret_val
 
 def detect_google_sorry(resp):
-    resp_url = urlparse(resp.url)
-    if resp_url.netloc == 'sorry.google.com' or resp_url.path.startswith('/sorry'):
+    if resp.url.host == 'sorry.google.com' or resp.url.path.startswith('/sorry'):
         raise SearxEngineCaptchaException()
 
 
diff --git a/searx/engines/pubmed.py b/searx/engines/pubmed.py
index da02f91ca..5d88d398e 100644
--- a/searx/engines/pubmed.py
+++ b/searx/engines/pubmed.py
@@ -7,7 +7,7 @@ from flask_babel import gettext
 from lxml import etree
 from datetime import datetime
 from urllib.parse import urlencode
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index 13dcf1250..d01dc0acc 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -8,7 +8,7 @@ from json import loads
 from urllib.parse import urlencode
 from searx.utils import html_to_text, match_language
 from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 # about
 about = {
diff --git a/searx/engines/seznam.py b/searx/engines/seznam.py
index faceb0550..042088dbe 100644
--- a/searx/engines/seznam.py
+++ b/searx/engines/seznam.py
@@ -3,9 +3,9 @@
  Seznam
 """
 
-from urllib.parse import urlencode, urlparse
+from urllib.parse import urlencode
 from lxml import html
-from searx.poolrequests import get
+from searx.network import get
 from searx.exceptions import SearxEngineAccessDeniedException
 from searx.utils import (
     extract_text,
@@ -46,8 +46,7 @@ def request(query, params):
 
 
 def response(resp):
-    resp_url = urlparse(resp.url)
-    if resp_url.path.startswith('/verify'):
+    if resp.url.path.startswith('/verify'):
         raise SearxEngineAccessDeniedException()
 
     results = []
diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py
index b3e3383bd..a6f923855 100644
--- a/searx/engines/soundcloud.py
+++ b/searx/engines/soundcloud.py
@@ -9,7 +9,7 @@ from lxml import html
 from dateutil import parser
 from urllib.parse import quote_plus, urlencode
 from searx import logger
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {
diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py
index 0ad8bfe32..6816fe672 100644
--- a/searx/engines/spotify.py
+++ b/searx/engines/spotify.py
@@ -5,9 +5,10 @@
 
 from json import loads
 from urllib.parse import urlencode
-import requests
 import base64
 
+from searx.network import post as http_post
+
 # about
 about = {
     "website": 'https://www.spotify.com',
@@ -38,7 +39,7 @@ def request(query, params):
 
     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
-    r = requests.post(
+    r = http_post(
         'https://accounts.spotify.com/api/token',
         data={'grant_type': 'client_credentials'},
         headers={'Authorization': 'Basic ' + base64.b64encode(
diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py
index 91eaa68e9..8fc2cdb3a 100644
--- a/searx/engines/stackoverflow.py
+++ b/searx/engines/stackoverflow.py
@@ -3,7 +3,7 @@
  Stackoverflow (IT)
 """
 
-from urllib.parse import urlencode, urljoin, urlparse
+from urllib.parse import urlencode, urljoin
 from lxml import html
 from searx.utils import extract_text
 from searx.exceptions import SearxEngineCaptchaException
@@ -41,8 +41,7 @@ def request(query, params):
 
 # get response from search-request
 def response(resp):
-    resp_url = urlparse(resp.url)
-    if resp_url.path.startswith('/nocaptcha'):
+    if resp.url.path.startswith('/nocaptcha'):
         raise SearxEngineCaptchaException()
 
     results = []
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index c8e4cfae6..ddcce9085 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -12,7 +12,7 @@ from babel.dates import format_datetime, format_date, format_time, get_datetime_
 
 from searx import logger
 from searx.data import WIKIDATA_UNITS
-from searx.poolrequests import post, get
+from searx.network import post, get
 from searx.utils import match_language, searx_useragent, get_string_replaces_function
 from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
 from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index 3ad8748fb..5e34db9a7 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -7,7 +7,7 @@ from urllib.parse import quote
 from json import loads
 from lxml.html import fromstring
 from searx.utils import match_language, searx_useragent
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 # about
 about = {
diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py
index 8e427d575..1f2cfa4e6 100644
--- a/searx/engines/wolframalpha_noapi.py
+++ b/searx/engines/wolframalpha_noapi.py
@@ -7,7 +7,7 @@ from json import loads
 from time import time
 from urllib.parse import urlencode
 
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {
diff --git a/searx/engines/wordnik.py b/searx/engines/wordnik.py
index 3abe9efa2..4bfeb4070 100644
--- a/searx/engines/wordnik.py
+++ b/searx/engines/wordnik.py
@@ -6,7 +6,7 @@
 from lxml.html import fromstring
 from searx import logger
 from searx.utils import extract_text
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 logger = logger.getChild('Wordnik engine')
 
diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py
index c194ca451..fbd99c47b 100644
--- a/searx/engines/yacy.py
+++ b/searx/engines/yacy.py
@@ -7,7 +7,7 @@ from json import loads
 from dateutil import parser
 from urllib.parse import urlencode
 
-from requests.auth import HTTPDigestAuth
+from httpx import DigestAuth
 
 from searx.utils import html_to_text
 
@@ -56,7 +56,7 @@ def request(query, params):
                           search_type=search_type)
 
     if http_digest_auth_user and http_digest_auth_pass:
-        params['auth'] = HTTPDigestAuth(http_digest_auth_user, http_digest_auth_pass)
+        params['auth'] = DigestAuth(http_digest_auth_user, http_digest_auth_pass)
 
     # add language tag if specified
     if params['language'] != 'all':
diff --git a/searx/engines/yggtorrent.py b/searx/engines/yggtorrent.py
index 8dfc0a0f2..f5af91f46 100644
--- a/searx/engines/yggtorrent.py
+++ b/searx/engines/yggtorrent.py
@@ -8,7 +8,7 @@ from operator import itemgetter
 from datetime import datetime
 from urllib.parse import quote
 from searx.utils import extract_text, get_torrent_size
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {