Merge branch 'master' into docker/opencontainers

8 files changed, 119 insertions, 88 deletions

diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index ed0b87dbd..b193f7c60 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -89,8 +89,7 @@ def response(resp):
                         'content': content})
 
     try:
-        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))
-        result_len_container = utils.to_string(result_len_container)
+        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
         if "-" in result_len_container:
             # Remove the part "from-to" for paginated request ...
             result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
@@ -102,7 +101,7 @@ def response(resp):
         logger.debug('result error :\n%s', e)
         pass
 
-    if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
+    if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
         return []
 
     results.append({'number_of_results': result_len})
diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py
index 198ac2cff..c8ee34f7a 100644
--- a/searx/engines/flickr_noapi.py
+++ b/searx/engines/flickr_noapi.py
@@ -109,14 +109,22 @@ def response(resp):
         else:
             url = build_flickr_url(photo['ownerNsid'], photo['id'])
 
-        results.append({'url': url,
-                        'title': title,
-                        'img_src': img_src,
-                        'thumbnail_src': thumbnail_src,
-                        'content': content,
-                        'author': author,
-                        'source': source,
-                        'img_format': img_format,
-                        'template': 'images.html'})
+        result = {
+            'url': url,
+            'img_src': img_src,
+            'thumbnail_src': thumbnail_src,
+            'source': source,
+            'img_format': img_format,
+            'template': 'images.html'
+        }
+        try:
+            result['author'] = author
+            result['title'] = title
+            result['content'] = content
+        except:
+            result['author'] = ''
+            result['title'] = ''
+            result['content'] = ''
+        results.append(result)
 
     return results
diff --git a/searx/engines/ina.py b/searx/engines/ina.py
index 37a05f099..ea509649f 100644
--- a/searx/engines/ina.py
+++ b/searx/engines/ina.py
@@ -32,7 +32,7 @@ base_url = 'https://www.ina.fr'
 search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}'
 
 # specific xpath variables
-results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]'
+results_xpath = '//div[contains(@class,"search-results--list")]//div[@class="media-body"]'
 url_xpath = './/a/@href'
 title_xpath = './/h3[@class="h3--title media-heading"]'
 thumbnail_xpath = './/img/@src'
@@ -65,8 +65,11 @@ def response(resp):
         videoid = result.xpath(url_xpath)[0]
         url = base_url + videoid
         title = p.unescape(extract_text(result.xpath(title_xpath)))
-        thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
-        if thumbnail[0] == '/':
+        try:
+            thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
+        except:
+            thumbnail = ''
+        if thumbnail and thumbnail[0] == '/':
             thumbnail = base_url + thumbnail
         d = extract_text(result.xpath(publishedDate_xpath)[0])
         d = d.split('/')
diff --git a/searx/engines/microsoft_academic.py b/searx/engines/microsoft_academic.py
index 9387b08d0..9bac0069c 100644
--- a/searx/engines/microsoft_academic.py
+++ b/searx/engines/microsoft_academic.py
@@ -45,6 +45,8 @@ def request(query, params):
 def response(resp):
     results = []
     response_data = loads(resp.text)
+    if not response_data:
+        return results
 
     for result in response_data['results']:
         url = _get_url(result)
diff --git a/searx/engines/scanr_structures.py b/searx/engines/scanr_structures.py
index 72fd2b3c9..7208dcb70 100644
--- a/searx/engines/scanr_structures.py
+++ b/searx/engines/scanr_structures.py
@@ -29,7 +29,7 @@ def request(query, params):
     params['url'] = search_url
     params['method'] = 'POST'
     params['headers']['Content-type'] = "application/json"
-    params['data'] = dumps({"query": query,
+    params['data'] = dumps({"query": query.decode('utf-8'),
                             "searchField": "ALL",
                             "sortDirection": "ASC",
                             "sortOrder": "RELEVANCY",
diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py
index aed756be3..00c395706 100644
--- a/searx/engines/spotify.py
+++ b/searx/engines/spotify.py
@@ -12,10 +12,14 @@
 
 from json import loads
 from searx.url_utils import urlencode
+import requests
+import base64
 
 # engine dependent config
 categories = ['music']
 paging = True
+api_client_id = None
+api_client_secret = None
 
 # search-url
 url = 'https://api.spotify.com/'
@@ -31,6 +35,16 @@ def request(query, params):
 
     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
+    r = requests.post(
+        'https://accounts.spotify.com/api/token',
+        data={'grant_type': 'client_credentials'},
+        headers={'Authorization': 'Basic ' + base64.b64encode(
+            "{}:{}".format(api_client_id, api_client_secret).encode('utf-8')
+        ).decode('utf-8')}
+    )
+    j = loads(r.text)
+    params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))}
+
     return params
 
 
diff --git a/searx/settings.yml b/searx/settings.yml
index e41b84c13..2777f9caa 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -79,9 +79,10 @@ engines:
     categories : science
     timeout : 4.0
 
-  - name : base
-    engine : base
-    shortcut : bs
+#  tmp suspended:  dh key too small
+#  - name : base
+#    engine : base
+#    shortcut : bs
 
   - name : wikipedia
     engine : wikipedia
@@ -552,10 +553,11 @@ engines:
     timeout : 10.0
     disabled : True
 
-  - name : scanr structures
-    shortcut: scs
-    engine : scanr_structures
-    disabled : True
+# tmp suspended: bad certificate
+#  - name : scanr structures
+#    shortcut: scs
+#    engine : scanr_structures
+#    disabled : True
 
   - name : soundcloud
     engine : soundcloud
@@ -598,9 +600,12 @@ engines:
     shortcut : se
     categories : science
 
-  - name : spotify
-    engine : spotify
-    shortcut : stf
+# Spotify needs API credentials
+#  - name : spotify
+#    engine : spotify
+#    shortcut : stf
+#    api_client_id : *******
+#    api_client_secret : *******
 
   - name : startpage
     engine : startpage
diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html
index 9cf942695..43e3e26d9 100644
--- a/searx/templates/oscar/results.html
+++ b/searx/templates/oscar/results.html
@@ -15,7 +15,68 @@
     {% include 'oscar/search.html' %}
 
     <div class="row">
-        <div class="col-sm-8" id="main_results">
+        <div class="col-sm-4 col-sm-push-8" id="sidebar_results">
+            {% if number_of_results != '0' -%}
+                <p><small>{{ _('Number of results') }}: {{ number_of_results }}</small></p>
+            {%- endif %}
+
+            {% if unresponsive_engines and results|length >= 1 -%}
+            <div class="alert alert-danger fade in" role="alert">
+                <p>{{ _('Engines cannot retrieve results') }}:</p>
+                {%- for engine_name, error_type in unresponsive_engines -%}
+                {{- engine_name }} ({{ error_type }}){% if not loop.last %}, {% endif %}{{- "" -}}
+                {%- endfor -%}
+            </div>
+            {%- endif %}
+
+            {% if infoboxes -%}
+                {% for infobox in infoboxes %}
+                    {% include 'oscar/infobox.html' %}{{- "\n\n" -}}
+                {% endfor %}
+            {%- endif %}
+
+            {% if suggestions %}
+            <div class="panel panel-default">
+                <div class="panel-heading">
+                    <h4 class="panel-title">{{ _('Suggestions') }}</h4>
+                </div>
+                <div class="panel-body">
+                    {% for suggestion in suggestions %}
+                    <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} suggestion_item">
+                        <input type="hidden" name="q" value="{{ suggestion.url }}">
+                        <button type="submit" class="btn btn-default btn-xs">{{ suggestion.title }}</button>
+                    </form>
+                    {% endfor %}
+                </div>
+            </div>
+            {%- endif %}
+
+            <div class="panel panel-default">
+                <div class="panel-heading">{{- "" -}}
+                    <h4 class="panel-title">{{ _('Links') }}</h4>{{- "" -}}
+                </div>
+                <div class="panel-body">
+                    <form role="form">{{- "" -}}
+                        <div class="form-group">{{- "" -}}
+                            <label for="search_url">{{ _('Search URL') }}</label>{{- "" -}}
+                            <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ search_url() }}" readonly>{{- "" -}}
+                        </div>{{- "" -}}
+                    </form>
+                    <label>{{ _('Download results') }}</label>
+                    <div class="clearfix"></div>
+                    {% for output_type in ('csv', 'json', 'rss') %}
+                    <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
+                        {{- search_form_attrs(pageno) -}}
+                        <input type="hidden" name="format" value="{{ output_type }}">{{- "" -}}
+                        <button type="submit" class="btn btn-default">{{ output_type }}</button>{{- "" -}}
+                    </form>
+                    {% endfor %}
+                    <div class="clearfix"></div>
+                </div>
+            </div>
+        </div><!-- /#sidebar_results -->
+
+        <div class="col-sm-8 col-sm-pull-4" id="main_results">
             <h1 class="sr-only">{{ _('Search results') }}</h1>
 
             {% if corrections -%}
@@ -91,66 +152,5 @@
             {% endif %}
             {% endif %}
         </div><!-- /#main_results -->
-
-        <div class="col-sm-4" id="sidebar_results">
-            {% if number_of_results != '0' -%}
-                <p><small>{{ _('Number of results') }}: {{ number_of_results }}</small></p>
-            {%- endif %}
-
-            {% if unresponsive_engines and results|length >= 1 -%}
-            <div class="alert alert-danger fade in" role="alert">
-                <p>{{ _('Engines cannot retrieve results') }}:</p>
-                {%- for engine_name, error_type in unresponsive_engines -%}
-                {{- engine_name }} ({{ error_type }}){% if not loop.last %}, {% endif %}{{- "" -}}
-                {%- endfor -%}
-            </div>
-            {%- endif %}
-
-            {% if infoboxes -%}
-                {% for infobox in infoboxes %}
-                    {% include 'oscar/infobox.html' %}{{- "\n\n" -}}
-                {% endfor %}
-            {%- endif %}
-
-            {% if suggestions %}
-            <div class="panel panel-default">
-                <div class="panel-heading">
-                    <h4 class="panel-title">{{ _('Suggestions') }}</h4>
-                </div>
-                <div class="panel-body">
-                    {% for suggestion in suggestions %}
-                    <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} suggestion_item">
-                        <input type="hidden" name="q" value="{{ suggestion.url }}">
-                        <button type="submit" class="btn btn-default btn-xs">{{ suggestion.title }}</button>
-                    </form>
-                    {% endfor %}
-                </div>
-            </div>
-            {%- endif %}
-
-            <div class="panel panel-default">
-                <div class="panel-heading">{{- "" -}}
-                    <h4 class="panel-title">{{ _('Links') }}</h4>{{- "" -}}
-                </div>
-                <div class="panel-body">
-                    <form role="form">{{- "" -}}
-                        <div class="form-group">{{- "" -}}
-                            <label for="search_url">{{ _('Search URL') }}</label>{{- "" -}}
-                            <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ search_url() }}" readonly>{{- "" -}}
-                        </div>{{- "" -}}
-                    </form>
-                    <label>{{ _('Download results') }}</label>
-                    <div class="clearfix"></div>
-                    {% for output_type in ('csv', 'json', 'rss') %}
-                    <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
-                        {{- search_form_attrs(pageno) -}}
-                        <input type="hidden" name="format" value="{{ output_type }}">{{- "" -}}
-                        <button type="submit" class="btn btn-default">{{ output_type }}</button>{{- "" -}}
-                    </form>
-                    {% endfor %}
-                    <div class="clearfix"></div>
-                </div>
-            </div>
-        </div><!-- /#sidebar_results -->
     </div>
 {% endblock %}