From c329ea135ed8c7b56a16e08bf0ee8f6f82609406 Mon Sep 17 00:00:00 2001
From: 0xhtml <34682885+0xhtml@users.noreply.github.com>
Date: Wed, 31 Jul 2019 20:44:41 +0200
Subject: Fix spotify engine

---
 searx/engines/spotify.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

(limited to 'searx/engines')

diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py
index aed756be3..da32b334c 100644
--- a/searx/engines/spotify.py
+++ b/searx/engines/spotify.py
@@ -12,10 +12,14 @@
 
 from json import loads
 from searx.url_utils import urlencode
+import requests
+import base64
 
 # engine dependent config
 categories = ['music']
 paging = True
+api_client_id = None
+api_client_secret = None
 
 # search-url
 url = 'https://api.spotify.com/'
@@ -31,6 +35,16 @@ def request(query, params):
 
     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
+    r = requests.post(
+        'https://accounts.spotify.com/api/token',
+        data={'grant_type': 'client_credentials'},
+        headers={'Authorization': 'Basic ' + str(base64.b64encode(
+            (api_client_id + ":" + api_client_secret).encode('utf-8')
+        ), 'utf-8')}
+    )
+    j = loads(r.text)
+    params['headers'] = {'Authorization': 'Bearer ' + j['access_token']}
+
     return params
 
 
-- 
cgit v1.2.3


From 275b37cc7c87b562d08576be5268a4f8797b84ea Mon Sep 17 00:00:00 2001
From: 0xhtml <34682885+0xhtml@users.noreply.github.com>
Date: Wed, 31 Jul 2019 21:01:24 +0200
Subject: Fix error if the user hasn't set api credentials

---
 searx/engines/spotify.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py
index da32b334c..57b08a1e4 100644
--- a/searx/engines/spotify.py
+++ b/searx/engines/spotify.py
@@ -39,7 +39,7 @@ def request(query, params):
         'https://accounts.spotify.com/api/token',
         data={'grant_type': 'client_credentials'},
         headers={'Authorization': 'Basic ' + str(base64.b64encode(
-            (api_client_id + ":" + api_client_secret).encode('utf-8')
+            "{}:{}".format(api_client_id, api_client_secret).encode('utf-8')
         ), 'utf-8')}
     )
     j = loads(r.text)
-- 
cgit v1.2.3


From b2e1ee8d35050033b41765a2de49c0eea5f8b4b4 Mon Sep 17 00:00:00 2001
From: 0xhtml <34682885+0xhtml@users.noreply.github.com>
Date: Wed, 31 Jul 2019 21:09:02 +0200
Subject: Fix some more errors with none/wrong credentials

---
 searx/engines/spotify.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py
index 57b08a1e4..00c395706 100644
--- a/searx/engines/spotify.py
+++ b/searx/engines/spotify.py
@@ -38,12 +38,12 @@ def request(query, params):
     r = requests.post(
         'https://accounts.spotify.com/api/token',
         data={'grant_type': 'client_credentials'},
-        headers={'Authorization': 'Basic ' + str(base64.b64encode(
+        headers={'Authorization': 'Basic ' + base64.b64encode(
             "{}:{}".format(api_client_id, api_client_secret).encode('utf-8')
-        ), 'utf-8')}
+        ).decode('utf-8')}
     )
     j = loads(r.text)
-    params['headers'] = {'Authorization': 'Bearer ' + j['access_token']}
+    params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))}
 
     return params
 
-- 
cgit v1.2.3


From 8850036ded3af2ba7455cef53a8134022e1b544d Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 21 Dec 2019 20:25:39 +0100
Subject: [fix] add explicit useragent header to requests - closes #1459

---
 searx/engines/qwant.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'searx/engines')

diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index de12955c6..54e9dafad 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -50,6 +50,7 @@ def request(query, params):
         language = match_language(params['language'], supported_languages, language_aliases)
         params['url'] += '&locale=' + language.replace('-', '_').lower()
 
+    params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'
     return params
 
 
-- 
cgit v1.2.3


From e5305f886c0d7d5fb3f34d1fbd7f9a545c14c284 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 21 Dec 2019 20:51:30 +0100
Subject: [fix] fetch extra search param of gigablast - fixes #1293

---
 searx/engines/gigablast.py | 41 ++++++++++++++++++++++++++++-------------
 1 file changed, 28 insertions(+), 13 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py
index a84f3f69d..2a5067bc3 100644
--- a/searx/engines/gigablast.py
+++ b/searx/engines/gigablast.py
@@ -14,6 +14,7 @@ import random
 from json import loads
 from time import time
 from lxml.html import fromstring
+from searx.poolrequests import get
 from searx.url_utils import urlencode
 from searx.utils import eval_xpath
 
@@ -31,13 +32,9 @@ search_string = 'search?{query}'\
     '&c=main'\
     '&s={offset}'\
     '&format=json'\
-    '&qh=0'\
-    '&qlang={lang}'\
+    '&langcountry={lang}'\
     '&ff={safesearch}'\
-    '&rxiec={rxieu}'\
-    '&ulse={ulse}'\
-    '&rand={rxikd}'\
-    '&dbez={dbez}'
+    '&rand={rxikd}'
 # specific xpath variables
 results_xpath = '//response//result'
 url_xpath = './/url'
@@ -46,9 +43,26 @@ content_xpath = './/sum'
 
 supported_languages_url = 'https://gigablast.com/search?&rxikd=1'
 
+extra_param = '' # gigablast requires a random extra parameter
+# which can be extracted from the source code of the search page
+
+
+def parse_extra_param(text):
+    global extra_param
+    param_lines = [x for x in text.splitlines() if x.startswith('var url=') or x.startswith('url=url+')]
+    extra_param = ''
+    for l in param_lines:
+        extra_param += l.split("'")[1]
+    extra_param = extra_param.split('&')[-1]
+
+
+def init(engine_settings=None):
+    parse_extra_param(get('http://gigablast.com/search?c=main&qlangcountry=en-us&q=south&s=10').text)
+
 
 # do search-request
 def request(query, params):
+    print("EXTRAPARAM:", extra_param)
     offset = (params['pageno'] - 1) * number_of_results
 
     if params['language'] == 'all':
@@ -67,14 +81,11 @@ def request(query, params):
     search_path = search_string.format(query=urlencode({'q': query}),
                                        offset=offset,
                                        number_of_results=number_of_results,
-                                       rxikd=int(time() * 1000),
-                                       rxieu=random.randint(1000000000, 9999999999),
-                                       ulse=random.randint(100000000, 999999999),
                                        lang=language,
-                                       safesearch=safesearch,
-                                       dbez=random.randint(100000000, 999999999))
+                                       rxikd=int(time() * 1000),
+                                       safesearch=safesearch)
 
-    params['url'] = base_url + search_path
+    params['url'] = base_url + search_path + '&' + extra_param
 
     return params
 
@@ -84,7 +95,11 @@ def response(resp):
     results = []
 
     # parse results
-    response_json = loads(resp.text)
+    try:
+        response_json = loads(resp.text)
+    except:
+        parse_extra_param(resp.text)
+        return results
 
     for result in response_json['results']:
         # append result
-- 
cgit v1.2.3


From f8713512bedf19d4495e0b9a0fd86679daaf7f79 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 21 Dec 2019 20:56:38 +0100
Subject: [fix] convert byte query to string in osm engine - fixes #1220

---
 searx/engines/openstreetmap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py
index 733ba6203..cec10a3c7 100644
--- a/searx/engines/openstreetmap.py
+++ b/searx/engines/openstreetmap.py
@@ -24,7 +24,7 @@ result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'
 
 # do search-request
 def request(query, params):
-    params['url'] = base_url + search_string.format(query=query)
+    params['url'] = base_url + search_string.format(query=query.decode('utf-8'))
 
     return params
 
-- 
cgit v1.2.3


From 00512e36c133312eb74a82f6a2dec6d06214c42b Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 21 Dec 2019 21:01:08 +0100
Subject: [fix] handle empty response from wikipedia engine - closes #1114

---
 searx/engines/wikipedia.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'searx/engines')

diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index 4dae735d1..690da72fe 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -79,6 +79,9 @@ def response(resp):
 
     # wikipedia article's unique id
     # first valid id is assumed to be the requested article
+    if 'pages' not in search_result['query']:
+        return results
+
     for article_id in search_result['query']['pages']:
         page = search_result['query']['pages'][article_id]
         if int(article_id) > 0:
-- 
cgit v1.2.3


From fc457569f757dd10ff55393f472ea9ed49a42374 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 21 Dec 2019 21:13:43 +0100
Subject: [fix] pep8

---
 searx/engines/gigablast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py
index 2a5067bc3..5af593e36 100644
--- a/searx/engines/gigablast.py
+++ b/searx/engines/gigablast.py
@@ -43,7 +43,7 @@ content_xpath = './/sum'
 
 supported_languages_url = 'https://gigablast.com/search?&rxikd=1'
 
-extra_param = '' # gigablast requires a random extra parameter
+extra_param = ''  # gigablast requires a random extra parameter
 # which can be extracted from the source code of the search page
 
 
-- 
cgit v1.2.3


From 34ad3d6b34017523a9502f86b92c17fe389918eb Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Sat, 21 Dec 2019 21:25:50 +0100
Subject: [enh] display error message if gigablast extra param expired

---
 searx/engines/gigablast.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py
index 5af593e36..2bb29a9fe 100644
--- a/searx/engines/gigablast.py
+++ b/searx/engines/gigablast.py
@@ -99,7 +99,7 @@ def response(resp):
         response_json = loads(resp.text)
     except:
         parse_extra_param(resp.text)
-        return results
+        raise Exception('extra param expired, please reload')
 
     for result in response_json['results']:
         # append result
-- 
cgit v1.2.3


From c18048e0454f4e3dc75c778940903091fbeae06a Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sun, 25 Aug 2019 22:23:37 -0700
Subject: exclude disambiguation pages from wikipedia infobox

---
 searx/engines/wikipedia.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index 690da72fe..44dea56fa 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -21,7 +21,8 @@ search_url = base_url + u'w/api.php?'\
     'action=query'\
     '&format=json'\
     '&{query}'\
-    '&prop=extracts|pageimages'\
+    '&prop=extracts|pageimages|pageprops'\
+    '&ppprop=disambiguation'\
     '&exintro'\
     '&explaintext'\
     '&pithumbsize=300'\
@@ -87,7 +88,7 @@ def response(resp):
         if int(article_id) > 0:
             break
 
-    if int(article_id) < 0:
+    if int(article_id) < 0 or 'disambiguation' in page.get('pageprops', {}):
         return []
 
     title = page.get('title')
-- 
cgit v1.2.3


From 5706c12fba98e169c7c76a4d3c29aabf48242d63 Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sun, 25 Aug 2019 22:47:23 -0700
Subject: remove empty parenthesis in wikipedia's summary

They're usually IPA pronunciations which are removed
by the API.
---
 searx/engines/wikipedia.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'searx/engines')

diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index 44dea56fa..a216ba886 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -100,6 +100,7 @@ def response(resp):
     extract = page.get('extract')
 
     summary = extract_first_paragraph(extract, title, image)
+    summary = summary.replace('() ', '')
 
     # link to wikipedia article
     wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \
-- 
cgit v1.2.3


From 2292e6e130dca104cb324197b63611a012e4ef3c Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 2 Jan 2020 22:28:47 +0100
Subject: [fix] handle missing result size

---
 searx/engines/bing.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index ed0b87dbd..24776c400 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -63,6 +63,8 @@ def response(resp):
     results = []
     result_len = 0
 
+
+
     dom = html.fromstring(resp.text)
     # parse results
     for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
@@ -89,8 +91,7 @@ def response(resp):
                         'content': content})
 
     try:
-        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))
-        result_len_container = utils.to_string(result_len_container)
+        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
         if "-" in result_len_container:
             # Remove the part "from-to" for paginated request ...
             result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
@@ -102,7 +103,7 @@ def response(resp):
         logger.debug('result error :\n%s', e)
         pass
 
-    if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
+    if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
         return []
 
     results.append({'number_of_results': result_len})
-- 
cgit v1.2.3


From 2dc2e1e8f9c8ae0d28df56f42b2f4949d8611624 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 2 Jan 2020 22:29:10 +0100
Subject: [fix] skip invalid encoded attributes

---
 searx/engines/flickr_noapi.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py
index 198ac2cff..e1abb378f 100644
--- a/searx/engines/flickr_noapi.py
+++ b/searx/engines/flickr_noapi.py
@@ -109,14 +109,22 @@ def response(resp):
         else:
             url = build_flickr_url(photo['ownerNsid'], photo['id'])
 
-        results.append({'url': url,
-                        'title': title,
-                        'img_src': img_src,
-                        'thumbnail_src': thumbnail_src,
-                        'content': content,
-                        'author': author,
-                        'source': source,
-                        'img_format': img_format,
-                        'template': 'images.html'})
+        result = {
+            'url': url,
+            'img_src': img_src,
+            'thumbnail_src': thumbnail_src,
+            'source': source,
+            'img_format': img_format,
+            'template': 'images.html'
+        }
+        try:
+            result['author'] = author.encode('utf-8')
+            result['title'] = title.encode('utf-8')
+            result['content'] = content.encode('utf-8')
+        except:
+            result['author'] = ''
+            result['title'] = ''
+            result['content'] = ''
+        results.append(result)
 
     return results
-- 
cgit v1.2.3


From 86a378bd0109684bd45c917f94068e3c98441904 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 2 Jan 2020 22:29:28 +0100
Subject: [fix] handle missing thumbnail

---
 searx/engines/ina.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/ina.py b/searx/engines/ina.py
index 37a05f099..ea509649f 100644
--- a/searx/engines/ina.py
+++ b/searx/engines/ina.py
@@ -32,7 +32,7 @@ base_url = 'https://www.ina.fr'
 search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}'
 
 # specific xpath variables
-results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]'
+results_xpath = '//div[contains(@class,"search-results--list")]//div[@class="media-body"]'
 url_xpath = './/a/@href'
 title_xpath = './/h3[@class="h3--title media-heading"]'
 thumbnail_xpath = './/img/@src'
@@ -65,8 +65,11 @@ def response(resp):
         videoid = result.xpath(url_xpath)[0]
         url = base_url + videoid
         title = p.unescape(extract_text(result.xpath(title_xpath)))
-        thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
-        if thumbnail[0] == '/':
+        try:
+            thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
+        except:
+            thumbnail = ''
+        if thumbnail and thumbnail[0] == '/':
             thumbnail = base_url + thumbnail
         d = extract_text(result.xpath(publishedDate_xpath)[0])
         d = d.split('/')
-- 
cgit v1.2.3


From 1e6253ce16346fc6f439a07211b56770d06ba225 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 2 Jan 2020 22:29:55 +0100
Subject: [fix] handle empty response

---
 searx/engines/microsoft_academic.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'searx/engines')

diff --git a/searx/engines/microsoft_academic.py b/searx/engines/microsoft_academic.py
index 9387b08d0..9bac0069c 100644
--- a/searx/engines/microsoft_academic.py
+++ b/searx/engines/microsoft_academic.py
@@ -45,6 +45,8 @@ def request(query, params):
 def response(resp):
     results = []
     response_data = loads(resp.text)
+    if not response_data:
+        return results
 
     for result in response_data['results']:
         url = _get_url(result)
-- 
cgit v1.2.3


From ad5bb994b1cff56c4f021f88bfa62f38055f1416 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 2 Jan 2020 22:30:18 +0100
Subject: [fix] add py3 compatibility

---
 searx/engines/scanr_structures.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/scanr_structures.py b/searx/engines/scanr_structures.py
index 72fd2b3c9..7208dcb70 100644
--- a/searx/engines/scanr_structures.py
+++ b/searx/engines/scanr_structures.py
@@ -29,7 +29,7 @@ def request(query, params):
     params['url'] = search_url
     params['method'] = 'POST'
     params['headers']['Content-type'] = "application/json"
-    params['data'] = dumps({"query": query,
+    params['data'] = dumps({"query": query.decode('utf-8'),
                             "searchField": "ALL",
                             "sortDirection": "ASC",
                             "sortOrder": "RELEVANCY",
-- 
cgit v1.2.3


From 17b6faa4c3c1cf14a327f4a3538fc70dce08b756 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 2 Jan 2020 22:37:06 +0100
Subject: [fix] pep8

---
 searx/engines/bing.py | 2 --
 1 file changed, 2 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 24776c400..b193f7c60 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -63,8 +63,6 @@ def response(resp):
     results = []
     result_len = 0
 
-
-
     dom = html.fromstring(resp.text)
     # parse results
     for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
-- 
cgit v1.2.3


From db9d7d47bdce4e4e68e681748af01f84f993434c Mon Sep 17 00:00:00 2001
From: frankdelange <yetangitu-f@unternet.org>
Date: Tue, 7 Jan 2020 21:41:43 +0100
Subject: Fix double-encode error (fixes #1799)

---
 searx/engines/flickr_noapi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py
index e1abb378f..c8ee34f7a 100644
--- a/searx/engines/flickr_noapi.py
+++ b/searx/engines/flickr_noapi.py
@@ -118,9 +118,9 @@ def response(resp):
             'template': 'images.html'
         }
         try:
-            result['author'] = author.encode('utf-8')
-            result['title'] = title.encode('utf-8')
-            result['content'] = content.encode('utf-8')
+            result['author'] = author
+            result['title'] = title
+            result['content'] = content
         except:
             result['author'] = ''
             result['title'] = ''
-- 
cgit v1.2.3