Merge branch 'master' into fix-engine-spotify

1 files changed, 112 insertions, 20 deletions

diff --git a/searx/search.py b/searx/search.py
index 1472073bd..5c268cc5d 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -45,6 +45,16 @@ if sys.version_info[0] == 3:
 logger = logger.getChild('search')
 
 number_of_searches = 0
+max_request_timeout = settings.get('outgoing', {}).get('max_request_timeout' or None)
+if max_request_timeout is None:
+    logger.info('max_request_timeout={0}'.format(max_request_timeout))
+else:
+    if isinstance(max_request_timeout, float):
+        logger.info('max_request_timeout={0} second(s)'.format(max_request_timeout))
+    else:
+        logger.critical('outgoing.max_request_timeout if defined has to be float')
+        from sys import exit
+        exit(1)
 
 
 def send_http_request(engine, request_params):
@@ -67,7 +77,7 @@ def send_http_request(engine, request_params):
     return req(request_params['url'], **request_args)
 
 
-def search_one_request(engine, query, request_params):
+def search_one_http_request(engine, query, request_params):
     # update request parameters dependent on
     # search-engine (contained in engines folder)
     engine.request(query, request_params)
@@ -87,7 +97,53 @@ def search_one_request(engine, query, request_params):
     return engine.response(response)
 
 
+def search_one_offline_request(engine, query, request_params):
+    return engine.search(query, request_params)
+
+
 def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
+    if engines[engine_name].offline:
+        return search_one_offline_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit)  # noqa
+    return search_one_http_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit)
+
+
+def search_one_offline_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
+    engine = engines[engine_name]
+
+    try:
+        search_results = search_one_offline_request(engine, query, request_params)
+
+        if search_results:
+            result_container.extend(engine_name, search_results)
+
+            engine_time = time() - start_time
+            result_container.add_timing(engine_name, engine_time, engine_time)
+            with threading.RLock():
+                engine.stats['engine_time'] += engine_time
+                engine.stats['engine_time_count'] += 1
+
+    except ValueError as e:
+        record_offline_engine_stats_on_error(engine, result_container, start_time)
+        logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))
+    except Exception as e:
+        record_offline_engine_stats_on_error(engine, result_container, start_time)
+
+        result_container.add_unresponsive_engine((
+            engine_name,
+            u'{0}: {1}'.format(gettext('unexpected crash'), e),
+        ))
+        logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
+
+
+def record_offline_engine_stats_on_error(engine, result_container, start_time):
+    engine_time = time() - start_time
+    result_container.add_timing(engine.name, engine_time, engine_time)
+
+    with threading.RLock():
+        engine.stats['errors'] += 1
+
+
+def search_one_http_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
     # set timeout for all HTTP requests
     requests_lib.set_timeout_for_thread(timeout_limit, start_time=start_time)
     # reset the HTTP total time
@@ -101,7 +157,7 @@ def search_one_request_safe(engine_name, query, request_params, result_container
 
     try:
         # send requests and parse the results
-        search_results = search_one_request(engine, query, request_params)
+        search_results = search_one_http_request(engine, query, request_params)
 
         # check if the engine accepted the request
         if search_results is not None:
@@ -265,6 +321,18 @@ def get_search_query_from_webapp(preferences, form):
     # query_engines
     query_engines = raw_text_query.engines
 
+    # timeout_limit
+    query_timeout = raw_text_query.timeout_limit
+    if query_timeout is None and 'timeout_limit' in form:
+        raw_time_limit = form.get('timeout_limit')
+        if raw_time_limit in ['None', '']:
+            raw_time_limit = None
+        else:
+            try:
+                query_timeout = float(raw_time_limit)
+            except ValueError:
+                raise SearxParameterException('timeout_limit', raw_time_limit)
+
     # query_categories
     query_categories = []
 
@@ -338,7 +406,8 @@ def get_search_query_from_webapp(preferences, form):
     query_engines = deduplicate_query_engines(query_engines)
 
     return (SearchQuery(query, query_engines, query_categories,
-                        query_lang, query_safesearch, query_pageno, query_time_range),
+                        query_lang, query_safesearch, query_pageno,
+                        query_time_range, query_timeout),
             raw_text_query)
 
 
@@ -351,6 +420,7 @@ class Search(object):
         super(Search, self).__init__()
         self.search_query = search_query
         self.result_container = ResultContainer()
+        self.actual_timeout = None
 
     # do search-request
     def search(self):
@@ -380,7 +450,7 @@ class Search(object):
         search_query = self.search_query
 
         # max of all selected engine timeout
-        timeout_limit = 0
+        default_timeout = 0
 
         # start search-reqest for all selected engines
         for selected_engine in search_query.engines:
@@ -403,29 +473,51 @@ class Search(object):
                 continue
 
             # set default request parameters
-            request_params = default_request_params()
-            request_params['headers']['User-Agent'] = user_agent
-            request_params['category'] = selected_engine['category']
-            request_params['pageno'] = search_query.pageno
+            request_params = {}
+            if not engine.offline:
+                request_params = default_request_params()
+                request_params['headers']['User-Agent'] = user_agent
 
-            if hasattr(engine, 'language') and engine.language:
-                request_params['language'] = engine.language
-            else:
-                request_params['language'] = search_query.lang
+                if hasattr(engine, 'language') and engine.language:
+                    request_params['language'] = engine.language
+                else:
+                    request_params['language'] = search_query.lang
 
-            # 0 = None, 1 = Moderate, 2 = Strict
-            request_params['safesearch'] = search_query.safesearch
-            request_params['time_range'] = search_query.time_range
+                request_params['safesearch'] = search_query.safesearch
+                request_params['time_range'] = search_query.time_range
+
+            request_params['category'] = selected_engine['category']
+            request_params['pageno'] = search_query.pageno
 
             # append request to list
             requests.append((selected_engine['name'], search_query.query, request_params))
 
-            # update timeout_limit
-            timeout_limit = max(timeout_limit, engine.timeout)
-
+            # update default_timeout
+            default_timeout = max(default_timeout, engine.timeout)
+
+        # adjust timeout
+        self.actual_timeout = default_timeout
+        query_timeout = self.search_query.timeout_limit
+
+        if max_request_timeout is None and query_timeout is None:
+            # No max, no user query: default_timeout
+            pass
+        elif max_request_timeout is None and query_timeout is not None:
+            # No max, but user query: From user query except if above default
+            self.actual_timeout = min(default_timeout, query_timeout)
+        elif max_request_timeout is not None and query_timeout is None:
+            # Max, no user query: Default except if above max
+            self.actual_timeout = min(default_timeout, max_request_timeout)
+        elif max_request_timeout is not None and query_timeout is not None:
+            # Max & user query: From user query except if above max
+            self.actual_timeout = min(query_timeout, max_request_timeout)
+
+        logger.debug("actual_timeout={0} (default_timeout={1}, ?timeout_limit={2}, max_request_timeout={3})"
+                     .format(self.actual_timeout, default_timeout, query_timeout, max_request_timeout))
+
+        # send all search-request
         if requests:
-            # send all search-request
-            search_multiple_requests(requests, self.result_container, start_time, timeout_limit)
+            search_multiple_requests(requests, self.result_container, start_time, self.actual_timeout)
             start_new_thread(gc.collect, tuple())
 
         # return results, suggestions, answers and infoboxes