Merge pull request #85 from searxng/fix-metrics-offline-engine

Fix metrics offline engine

8 files changed, 86 insertions, 61 deletions

diff --git a/searx/metrics/__init__.py b/searx/metrics/__init__.py
index c2dcfaae4..2ef73149b 100644
--- a/searx/metrics/__init__.py
+++ b/searx/metrics/__init__.py
@@ -156,57 +156,76 @@ def get_reliabilities(engline_name_list, checker_results):
     return reliabilities
 
 
-def round_or_none(number, digits):
-    return round(number, digits) if number else number
-
-
 def get_engines_stats(engine_name_list):
     assert counter_storage is not None
     assert histogram_storage is not None
 
     list_time = []
+    max_time_total = max_result_count = None
 
-    max_time_total = max_result_count = None  # noqa
     for engine_name in engine_name_list:
+
         sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
         if sent_count == 0:
             continue
 
-        successful_count = counter('engine', engine_name, 'search', 'count', 'successful')
-
-        time_total = histogram('engine', engine_name, 'time', 'total').percentage(50)
-        time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
-        time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
-        time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
-        time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
-        time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
-
         result_count = histogram('engine', engine_name, 'result', 'count').percentage(50)
         result_count_sum = histogram('engine', engine_name, 'result', 'count').sum
-        if successful_count and result_count_sum:
-            score = counter('engine', engine_name, 'score')  # noqa
-            score_per_result = score / float(result_count_sum)
-        else:
-            score = score_per_result = 0.0
+        successful_count = counter('engine', engine_name, 'search', 'count', 'successful')
 
+        time_total = histogram('engine', engine_name, 'time', 'total').percentage(50)
         max_time_total = max(time_total or 0, max_time_total or 0)
         max_result_count = max(result_count or 0, max_result_count or 0)
 
-        list_time.append({
+        stats = {
             'name': engine_name,
-            'total': round_or_none(time_total, 1),
-            'total_p80': round_or_none(time_total_p80, 1),
-            'total_p95': round_or_none(time_total_p95, 1),
-            'http': round_or_none(time_http, 1),
-            'http_p80': round_or_none(time_http_p80, 1),
-            'http_p95': round_or_none(time_http_p95, 1),
-            'processing': round(time_total - time_http, 1) if time_total else None,
-            'processing_p80': round(time_total_p80 - time_http_p80, 1) if time_total else None,
-            'processing_p95': round(time_total_p95 - time_http_p95, 1) if time_total else None,
-            'score': score,
-            'score_per_result': score_per_result,
+            'total': None,
+            'total_p80': None,
+            'total_p95': None,
+            'http': None,
+            'http_p80': None,
+            'http_p95': None,
+            'processing': None,
+            'processing_p80': None,
+            'processing_p95': None,
+            'score': 0,
+            'score_per_result': 0,
             'result_count': result_count,
-        })
+        }
+
+        if successful_count and result_count_sum:
+            score = counter('engine', engine_name, 'score')
+
+            stats['score'] = score
+            stats['score_per_result'] = score / float(result_count_sum)
+
+        time_http = histogram('engine', engine_name, 'time', 'http').percentage(50)
+        time_http_p80 = time_http_p95 = 0
+
+        if time_http is not None:
+
+            time_http_p80 = histogram('engine', engine_name, 'time', 'http').percentage(80)
+            time_http_p95 = histogram('engine', engine_name, 'time', 'http').percentage(95)
+
+            stats['http'] = round(time_http, 1)
+            stats['http_p80'] = round(time_http_p80, 1)
+            stats['http_p95'] = round(time_http_p95, 1)
+
+        if time_total is not None:
+
+            time_total_p80 = histogram('engine', engine_name, 'time', 'total').percentage(80)
+            time_total_p95 = histogram('engine', engine_name, 'time', 'total').percentage(95)
+
+            stats['total'] = round(time_total, 1)
+            stats['total_p80'] = round(time_total_p80, 1)
+            stats['total_p95'] = round(time_total_p95, 1)
+
+            stats['processing'] = round(time_total - (time_http or 0), 1)
+            stats['processing_p80'] = round(time_total_p80 - time_http_p80, 1)
+            stats['processing_p95'] = round(time_total_p95 - time_http_p95, 1)
+
+        list_time.append(stats)
+
     return {
         'time': list_time,
         'max_time': math.ceil(max_time_total or 0),
diff --git a/searx/network/__init__.py b/searx/network/__init__.py
index 981b2261a..587198144 100644
--- a/searx/network/__init__.py
+++ b/searx/network/__init__.py
@@ -44,7 +44,8 @@ def reset_time_for_thread():
 
 
 def get_time_for_thread():
-    return THREADLOCAL.total_time
+    """returns thread's total time or None"""
+    return THREADLOCAL.__dict__.get('total_time')
 
 
 def set_timeout_for_thread(timeout, start_time=None):
@@ -57,10 +58,11 @@ def set_context_network_name(network_name):
 
 
 def get_context_network():
-    try:
-        return THREADLOCAL.network
-    except AttributeError:
-        return get_network()
+    """If set return thread's network.
+
+    If unset, return value from :py:obj:`get_network`.
+    """
+    return THREADLOCAL.__dict__.get('network') or get_network()
 
 
 def request(method, url, **kwargs):
diff --git a/searx/results.py b/searx/results.py
index a1c1d8527..d0cb4df3f 100644
--- a/searx/results.py
+++ b/searx/results.py
@@ -371,11 +371,12 @@ class ResultContainer:
             self.unresponsive_engines.add((engine_name, error_type, error_message, suspended))
 
     def add_timing(self, engine_name, engine_time, page_load_time):
-        self.timings.append({
+        timing = {
             'engine': engines[engine_name].shortcut,
             'total': engine_time,
-            'load': page_load_time
-        })
+            'load': page_load_time,
+        }
+        self.timings.append(timing)
 
     def get_timings(self):
         return self.timings
diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html
index 33fb41061..352b90eed 100644
--- a/searx/templates/oscar/preferences.html
+++ b/searx/templates/oscar/preferences.html
@@ -25,11 +25,13 @@
 <td class="{{ label }}" style="padding: 2px">{{- "" -}}
     {%- if stats[engine_name].time != None -%}
     <span class="stacked-bar-chart-value">{{- stats[engine_name].time -}}</span>{{- "" -}}
-    <span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">{{- "" -}}
+    <span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">
+        {%- if max_rate95 is not none and max_rate95 > 0 -%}
         <span style="width: calc(max(2px, 100%*{{ (stats[engine_name].time / max_rate95)|round(3) }}))" class="stacked-bar-chart-median"></span>{{- "" -}}
         <span style="width: calc(100%*{{ ((stats[engine_name].rate80 - stats[engine_name].time) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate80"></span>{{- "" -}}
         <span style="width: calc(100%*{{ ((stats[engine_name].rate95 - stats[engine_name].rate80) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate95"></span>{{- "" -}}
-        <span class="stacked-bar-chart-rate100"></span>{{- "" -}}
+        <span class="stacked-bar-chart-rate100"></span>
+        {%- endif -%}
     </span>{{- "" -}}
     <div class="engine-tooltip text-left" role="tooltip" id="{{engine_name}}_graph">{{- "" -}}
         <p>{{ _('Median') }}: {{ stats[engine_name].time }}</p>{{- "" -}}
diff --git a/searx/templates/oscar/stats.html b/searx/templates/oscar/stats.html
index 94117b673..d9b2ab68e 100644
--- a/searx/templates/oscar/stats.html
+++ b/searx/templates/oscar/stats.html
@@ -52,11 +52,11 @@
                                 {%- endif -%}
                             </td>
                             <td class="response-time">
-                                {%- if engine_stat.total -%}
+                                {%- if engine_stat.total is not none -%}
                                 <div class="bar-chart-value">{{- engine_stat.total | round(1) -}}</div>{{- "" -}}
                                 <div class="bar-chart-graph" aria-labelledby="{{engine_stat.name}}_time" aria-hidden="true">{{- "" -}}
-                                    <div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{{- "" -}}
-                                    <div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{{- "" -}}
+                                    {% if engine_stat.http is not none and engine_stats.max_time %}<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{%- endif -%}
+                                    {% if engine_stat.processing is not none and engine_stats.max_time %}<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{%- endif -%}
                                 </div>
                                 <div class="engine-tooltip text-left" role="tooltip" id="{{engine_stat.name}}_time">{{- "" -}}
                                     <table class="table table-striped">
@@ -69,19 +69,19 @@
                                         <tr>
                                             <th scope="col">{{ _('Median') }}</th>
                                             <td>{{ engine_stat.total }}</td>
-                                            <td>{{ engine_stat.http }}</td>
+                                            <td>{{ engine_stat.http or '' }}</td>
                                             <td>{{ engine_stat.processing }}</td>
                                         </tr>
                                         <tr>
                                             <th scope="col">{{ _('P80') }}</th>
                                             <td>{{ engine_stat.total_p80 }}</td>
-                                            <td>{{ engine_stat.http_p80 }}</td>
+                                            <td>{{ engine_stat.http_p80 or '' }}</td>
                                             <td>{{ engine_stat.processing_p80 }}</td>
                                         </tr>
                                         <tr>
                                             <th scope="col">{{ _('P95') }}</th>
                                             <td>{{ engine_stat.total_p95 }}</td>
-                                            <td>{{ engine_stat.http_p95 }}</td>
+                                            <td>{{ engine_stat.http_p95 or '' }}</td>
                                             <td>{{ engine_stat.processing_p95 }}</td>
                                         </tr>
                                     </table>
diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html
index 6d33233c7..e3b1545b9 100644
--- a/searx/templates/simple/preferences.html
+++ b/searx/templates/simple/preferences.html
@@ -39,11 +39,13 @@
 <td class="{{ label }}" style="padding: 2px; width: 13rem;">{{- "" -}}
     {%- if stats[engine_name].time != None -%}
     <span class="stacked-bar-chart-value">{{- stats[engine_name].time -}}</span>{{- "" -}}
-    <span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">{{- "" -}}
+    <span class="stacked-bar-chart" aria-labelledby="{{engine_name}}_chart" aria-hidden="true">
+        {%- if max_rate95 is not none and max_rate95 > 0 -%}
         <span style="width: calc(max(2px, 100%*{{ (stats[engine_name].time / max_rate95)|round(3) }}))" class="stacked-bar-chart-median"></span>{{- "" -}}
         <span style="width: calc(100%*{{ ((stats[engine_name].rate80 - stats[engine_name].time) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate80"></span>{{- "" -}}
         <span style="width: calc(100%*{{ ((stats[engine_name].rate95 - stats[engine_name].rate80) / max_rate95)|round(3) }})" class="stacked-bar-chart-rate95"></span>{{- "" -}}
-        <span class="stacked-bar-chart-rate100"></span>{{- "" -}}
+        <span class="stacked-bar-chart-rate100"></span>
+        {%- endif -%}
     </span>{{- "" -}}
     <div class="engine-tooltip text-left" role="tooltip" id="{{engine_name}}_graph">{{- "" -}}
         <p>{{ _('Median') }}: {{ stats[engine_name].time }}</p>{{- "" -}}
diff --git a/searx/templates/simple/stats.html b/searx/templates/simple/stats.html
index 67c9c79cc..7058d04d4 100644
--- a/searx/templates/simple/stats.html
+++ b/searx/templates/simple/stats.html
@@ -52,12 +52,11 @@
             {%- endif -%}
         </td>
         <td class="response-time">
-            {%- if engine_stat.total -%}
-
+            {%- if engine_stat.total is not none -%}
             <div class="bar-chart-value">{{- engine_stat.total | round(1) -}}</div>{{- "" -}}
         <div class="bar-chart-graph" aria-labelledby="{{engine_stat.name}}_time" aria-hidden="true">
-              <div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{{- "" -}}
-              <div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{{- "" -}}
+              {% if engine_stat.http is not none and engine_stats.max_time %}<div class="bar-chart-serie1 bar{{ (100 * engine_stat.http / engine_stats.max_time)|round }}"></div>{%- endif -%}
+              {% if engine_stat.processing is not none and engine_stats.max_time %}<div class="bar-chart-serie2 bar{{ (100 * engine_stat.processing / engine_stats.max_time)|round }}"></div>{%- endif -%}
         </div>
             <div class="engine-tooltip" role="tooltip" id="{{engine_stat.name}}_time">{{- "" -}}
                 <table>
@@ -70,19 +69,19 @@
                     <tr>
                         <th scope="col">{{ _('Median') }}</th>
                         <td>{{ engine_stat.total }}</td>
-                        <td>{{ engine_stat.http }}</td>
+                        <td>{{ engine_stat.http or ''}}</td>
                         <td>{{ engine_stat.processing }}</td>
                     </tr>
                     <tr>
                         <th scope="col">{{ _('P80') }}</th>
                         <td>{{ engine_stat.total_p80 }}</td>
-                        <td>{{ engine_stat.http_p80 }}</td>
+                        <td>{{ engine_stat.http_p80 or '' }}</td>
                         <td>{{ engine_stat.processing_p80 }}</td>
                     </tr>
                     <tr>
                         <th scope="col">{{ _('P95') }}</th>
                         <td>{{ engine_stat.total_p95 }}</td>
-                        <td>{{ engine_stat.http_p95 }}</td>
+                        <td>{{ engine_stat.http_p95 or '' }}</td>
                         <td>{{ engine_stat.processing_p95 }}</td>
                     </tr>
                 </table>
diff --git a/searx/webapp.py b/searx/webapp.py
index 0ea82d098..7df12af18 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -568,7 +568,7 @@ def post_request(response):
         timings_total = ['total_' + str(i) + '_' + v['engine'] +
                          ';dur=' + str(round(v['total'] * 1000, 3)) for i, v in enumerate(timings)]
         timings_load = ['load_' + str(i) + '_' + v['engine'] +
-                        ';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings)]
+                        ';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings) if v.get('load')]
         timings_all = timings_all + timings_total + timings_load
     response.headers.add('Server-Timing', ', '.join(timings_all))
     return response
@@ -923,9 +923,9 @@ def preferences():
         result_count = int(result_count_sum / float(successful_count)) if successful_count else 0
 
         stats[e.name] = {
-            'time': median if median else None,
-            'rate80': rate80 if rate80 else None,
-            'rate95': rate95 if rate95 else None,
+            'time': median,
+            'rate80': rate80,
+            'rate95': rate95,
             'warn_timeout': e.timeout > settings['outgoing']['request_timeout'],
             'supports_selected_language': _is_selected_language_supported(e, request.preferences),
             'result_count': result_count,