diff options
Diffstat (limited to 'searx/search.py')
| -rw-r--r-- | searx/search.py | 23 |
1 files changed, 18 insertions, 5 deletions
diff --git a/searx/search.py b/searx/search.py index 0e7aaed8f..19469cc52 100644 --- a/searx/search.py +++ b/searx/search.py @@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <asciimoo@gmail.com> ''' -import grequests +import requests as requests_lib +import threading import re from itertools import izip_longest, chain from datetime import datetime @@ -32,6 +33,18 @@ from searx.query import Query number_of_searches = 0 +def threaded_requests(requests): + for fn, url, request_args in requests: + th = threading.Thread( + target=fn, args=(url,), kwargs=request_args, name=url, + ) + th.start() + + for th in threading.enumerate(): + if th.name.startswith('http'): + th.join() + + # get default reqest parameter def default_request_params(): return { @@ -471,9 +484,9 @@ class Search(object): # specific type of request (GET or POST) if request_params['method'] == 'GET': - req = grequests.get + req = requests_lib.get else: - req = grequests.post + req = requests_lib.post request_args['data'] = request_params['data'] # ignoring empty urls @@ -481,10 +494,10 @@ class Search(object): continue # append request to list - requests.append(req(request_params['url'], **request_args)) + requests.append((req, request_params['url'], request_args)) # send all search-request - grequests.map(requests) + threaded_requests(requests) # update engine-specific stats for engine_name, engine_results in results.items(): |