From c329ea135ed8c7b56a16e08bf0ee8f6f82609406 Mon Sep 17 00:00:00 2001 From: 0xhtml <34682885+0xhtml@users.noreply.github.com> Date: Wed, 31 Jul 2019 20:44:41 +0200 Subject: Fix spotify engine --- searx/engines/spotify.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'searx') diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py index aed756be3..da32b334c 100644 --- a/searx/engines/spotify.py +++ b/searx/engines/spotify.py @@ -12,10 +12,14 @@ from json import loads from searx.url_utils import urlencode +import requests +import base64 # engine dependent config categories = ['music'] paging = True +api_client_id = None +api_client_secret = None # search-url url = 'https://api.spotify.com/' @@ -31,6 +35,16 @@ def request(query, params): params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset) + r = requests.post( + 'https://accounts.spotify.com/api/token', + data={'grant_type': 'client_credentials'}, + headers={'Authorization': 'Basic ' + str(base64.b64encode( + (api_client_id + ":" + api_client_secret).encode('utf-8') + ), 'utf-8')} + ) + j = loads(r.text) + params['headers'] = {'Authorization': 'Bearer ' + j['access_token']} + return params -- cgit v1.2.3 From ae3eeedb14b1f693872787ac67975728c5773b84 Mon Sep 17 00:00:00 2001 From: 0xhtml <34682885+0xhtml@users.noreply.github.com> Date: Wed, 31 Jul 2019 20:51:01 +0200 Subject: Require Spotify API credentials in settings --- searx/settings.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'searx') diff --git a/searx/settings.yml b/searx/settings.yml index 490b3af04..281aaceaf 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -590,9 +590,12 @@ engines: shortcut : se categories : science - - name : spotify - engine : spotify - shortcut : stf +# Spotify needs API credentials +# - name : spotify +# engine : spotify +# shortcut : stf +# api_client_id : ******* +# api_client_secret : ******* - name : startpage engine : startpage -- cgit v1.2.3 From 275b37cc7c87b562d08576be5268a4f8797b84ea Mon Sep 17 00:00:00 2001 From: 0xhtml <34682885+0xhtml@users.noreply.github.com> Date: Wed, 31 Jul 2019 21:01:24 +0200 Subject: Fix error if the user hasn't set api credentials --- searx/engines/spotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx') diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py index da32b334c..57b08a1e4 100644 --- a/searx/engines/spotify.py +++ b/searx/engines/spotify.py @@ -39,7 +39,7 @@ def request(query, params): 'https://accounts.spotify.com/api/token', data={'grant_type': 'client_credentials'}, headers={'Authorization': 'Basic ' + str(base64.b64encode( - (api_client_id + ":" + api_client_secret).encode('utf-8') + "{}:{}".format(api_client_id, api_client_secret).encode('utf-8') ), 'utf-8')} ) j = loads(r.text) -- cgit v1.2.3 From b2e1ee8d35050033b41765a2de49c0eea5f8b4b4 Mon Sep 17 00:00:00 2001 From: 0xhtml <34682885+0xhtml@users.noreply.github.com> Date: Wed, 31 Jul 2019 21:09:02 +0200 Subject: Fix some more errors with none/wrong credentials --- searx/engines/spotify.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'searx') diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py index 57b08a1e4..00c395706 100644 --- a/searx/engines/spotify.py +++ b/searx/engines/spotify.py @@ -38,12 +38,12 @@ def request(query, params): r = requests.post( 'https://accounts.spotify.com/api/token', data={'grant_type': 'client_credentials'}, - headers={'Authorization': 'Basic ' + str(base64.b64encode( + headers={'Authorization': 'Basic ' + base64.b64encode( "{}:{}".format(api_client_id, api_client_secret).encode('utf-8') - ), 'utf-8')} + ).decode('utf-8')} ) j = loads(r.text) - params['headers'] = {'Authorization': 'Bearer ' + j['access_token']} + params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))} return params -- cgit v1.2.3 From 0ae86cd1685d244c83a6080a7816365096ab06f8 Mon Sep 17 00:00:00 2001 From: Nick Espig Date: Tue, 4 Jun 2019 19:16:19 +0200 Subject: Fix not jumping to results loaded by infinite scroll Infinite scroll adds a `hr` tag to split up the sections loaded by it. The vim bindings `j` and `k`, which jump to the next and previous result respectively, search for a **direct** sibling with the class `result`. With the `hr` between results a direct sibling cannot be found. To fix this we remove the restriction of it having to be a direct sibling. --- searx/static/plugins/js/vim_hotkeys.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'searx') diff --git a/searx/static/plugins/js/vim_hotkeys.js b/searx/static/plugins/js/vim_hotkeys.js index 13bd070e0..b0f265cb5 100644 --- a/searx/static/plugins/js/vim_hotkeys.js +++ b/searx/static/plugins/js/vim_hotkeys.js @@ -125,6 +125,14 @@ $(document).ready(function() { } }); + function nextResult(current, direction) { + var next = current[direction](); + while (!next.is('.result') && next.length !== 0) { + next = next[direction](); + } + return next + } + function highlightResult(which) { return function() { var current = $('.result[data-vim-selected]'); @@ -157,13 +165,13 @@ $(document).ready(function() { } break; case 'down': - next = current.next('.result'); + next = nextResult(current, 'next'); if (next.length === 0) { next = $('.result:first'); } break; case 'up': - next = current.prev('.result'); + next = nextResult(current, 'prev'); if (next.length === 0) { next = $('.result:last'); } -- cgit v1.2.3 From ee6781d777f3a95f6e1c23499ecbc7257d5e35ec Mon Sep 17 00:00:00 2001 From: Vipul Date: Sat, 14 Sep 2019 12:37:39 +0000 Subject: [Fix] Libgen engine Libgen has switched to new domain (i.e https://libgen.is) with TLS support and older domain (i.e. http://libgen.io) is no longer accessible. See, https://en.wikipedia.org/wiki/Library_Genesis, for more information. Resolves: #1693 --- searx/settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx') diff --git a/searx/settings.yml b/searx/settings.yml index cf2b13e08..539049ea0 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -407,7 +407,7 @@ engines: - name : library genesis engine : xpath - search_url : http://libgen.io/search.php?req={query} + search_url : https://libgen.is/search.php?req={query} url_xpath : //a[contains(@href,"bookfi.net")]/@href title_xpath : //a[contains(@href,"book/")]/text()[1] content_xpath : //td/a[1][contains(@href,"=author")]/text() -- cgit v1.2.3 From f407dd8ef4e3f6c82bef31f678139d6db2a4d810 Mon Sep 17 00:00:00 2001 From: Vipul Date: Sat, 14 Sep 2019 12:45:02 +0000 Subject: Switch to https for some domains --- searx/settings.yml | 8 ++++---- searx/settings_robot.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'searx') diff --git a/searx/settings.yml b/searx/settings.yml index 539049ea0..d9a1f45f0 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -463,7 +463,7 @@ engines: - name : openairedatasets engine : json_engine paging : True - search_url : http://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} + search_url : https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query} results_query : response/results/result url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ title_query : metadata/oaf:entity/oaf:result/title/$ @@ -475,7 +475,7 @@ engines: - name : openairepublications engine : json_engine paging : True - search_url : http://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} + search_url : https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query} results_query : response/results/result url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$ title_query : metadata/oaf:entity/oaf:result/title/$ @@ -806,7 +806,7 @@ locales: doi_resolvers : oadoi.org : 'https://oadoi.org/' doi.org : 'https://doi.org/' - doai.io : 'http://doai.io/' - sci-hub.tw : 'http://sci-hub.tw/' + doai.io : 'https://doai.io/' + sci-hub.tw : 'https://sci-hub.tw/' default_doi_resolver : 'oadoi.org' diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml index 635809041..25f229e56 100644 --- a/searx/settings_robot.yml +++ b/searx/settings_robot.yml @@ -43,7 +43,7 @@ locales: doi_resolvers : oadoi.org : 'https://oadoi.org/' doi.org : 'https://doi.org/' - doai.io : 'http://doai.io/' - sci-hub.tw : 'http://sci-hub.tw/' + doai.io : 'https://doai.io/' + sci-hub.tw : 'https://sci-hub.tw/' default_doi_resolver : 'oadoi.org' -- cgit v1.2.3 From 8bea927bb02e02754834d6f9692942f621bd21c5 Mon Sep 17 00:00:00 2001 From: Vipul Date: Sun, 22 Dec 2019 01:21:22 +0000 Subject: [Fix] oscar: no HTML escaping prior to output When results are fetched from any programming related documentation site (like git-scm.com, docs.python.org etc), content in Info box is shown as raw HTML code. This change addresses the issue by using "safe" filter feature provided by Django. See, - https://docs.djangoproject.com/en/3.0/ref/templates/builtins/#safe - Searx issue tracker (issue #1649), for more information. Resolves: #1649 --- searx/templates/oscar/infobox.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx') diff --git a/searx/templates/oscar/infobox.html b/searx/templates/oscar/infobox.html index 9f5e58d2b..9802f11e2 100644 --- a/searx/templates/oscar/infobox.html +++ b/searx/templates/oscar/infobox.html @@ -6,7 +6,7 @@
{% if infobox.img_src %}{{ infobox.infobox }}{% endif %} - {% if infobox.content %}

{{ infobox.content }}

{% endif %} + {% if infobox.content %}

{{ infobox.content | safe }}

{% endif %} {% if infobox.attributes -%} -- cgit v1.2.3 From a1b85571a25d67b752bf6072255b928866be9c4f Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 2 Jan 2020 22:28:18 +0100 Subject: [fix] tmp suspend insecure engines --- searx/settings.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'searx') diff --git a/searx/settings.yml b/searx/settings.yml index 2a2d2bf87..2777f9caa 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -79,9 +79,10 @@ engines: categories : science timeout : 4.0 - - name : base - engine : base - shortcut : bs +# tmp suspended: dh key too small +# - name : base +# engine : base +# shortcut : bs - name : wikipedia engine : wikipedia @@ -552,10 +553,11 @@ engines: timeout : 10.0 disabled : True - - name : scanr structures - shortcut: scs - engine : scanr_structures - disabled : True +# tmp suspended: bad certificate +# - name : scanr structures +# shortcut: scs +# engine : scanr_structures +# disabled : True - name : soundcloud engine : soundcloud -- cgit v1.2.3 From 2292e6e130dca104cb324197b63611a012e4ef3c Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 2 Jan 2020 22:28:47 +0100 Subject: [fix] handle missing result size --- searx/engines/bing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'searx') diff --git a/searx/engines/bing.py b/searx/engines/bing.py index ed0b87dbd..24776c400 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -63,6 +63,8 @@ def response(resp): results = [] result_len = 0 + + dom = html.fromstring(resp.text) # parse results for result in eval_xpath(dom, '//div[@class="sa_cc"]'): @@ -89,8 +91,7 @@ def response(resp): 'content': content}) try: - result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()')) - result_len_container = utils.to_string(result_len_container) + result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()')) if "-" in result_len_container: # Remove the part "from-to" for paginated request ... result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:] @@ -102,7 +103,7 @@ def response(resp): logger.debug('result error :\n%s', e) pass - if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len: + if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len: return [] results.append({'number_of_results': result_len}) -- cgit v1.2.3 From 2dc2e1e8f9c8ae0d28df56f42b2f4949d8611624 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 2 Jan 2020 22:29:10 +0100 Subject: [fix] skip invalid encoded attributes --- searx/engines/flickr_noapi.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'searx') diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py index 198ac2cff..e1abb378f 100644 --- a/searx/engines/flickr_noapi.py +++ b/searx/engines/flickr_noapi.py @@ -109,14 +109,22 @@ def response(resp): else: url = build_flickr_url(photo['ownerNsid'], photo['id']) - results.append({'url': url, - 'title': title, - 'img_src': img_src, - 'thumbnail_src': thumbnail_src, - 'content': content, - 'author': author, - 'source': source, - 'img_format': img_format, - 'template': 'images.html'}) + result = { + 'url': url, + 'img_src': img_src, + 'thumbnail_src': thumbnail_src, + 'source': source, + 'img_format': img_format, + 'template': 'images.html' + } + try: + result['author'] = author.encode('utf-8') + result['title'] = title.encode('utf-8') + result['content'] = content.encode('utf-8') + except: + result['author'] = '' + result['title'] = '' + result['content'] = '' + results.append(result) return results -- cgit v1.2.3 From 86a378bd0109684bd45c917f94068e3c98441904 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 2 Jan 2020 22:29:28 +0100 Subject: [fix] handle missing thumbnail --- searx/engines/ina.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'searx') diff --git a/searx/engines/ina.py b/searx/engines/ina.py index 37a05f099..ea509649f 100644 --- a/searx/engines/ina.py +++ b/searx/engines/ina.py @@ -32,7 +32,7 @@ base_url = 'https://www.ina.fr' search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}' # specific xpath variables -results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]' +results_xpath = '//div[contains(@class,"search-results--list")]//div[@class="media-body"]' url_xpath = './/a/@href' title_xpath = './/h3[@class="h3--title media-heading"]' thumbnail_xpath = './/img/@src' @@ -65,8 +65,11 @@ def response(resp): videoid = result.xpath(url_xpath)[0] url = base_url + videoid title = p.unescape(extract_text(result.xpath(title_xpath))) - thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) - if thumbnail[0] == '/': + try: + thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) + except: + thumbnail = '' + if thumbnail and thumbnail[0] == '/': thumbnail = base_url + thumbnail d = extract_text(result.xpath(publishedDate_xpath)[0]) d = d.split('/') -- cgit v1.2.3 From 1e6253ce16346fc6f439a07211b56770d06ba225 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 2 Jan 2020 22:29:55 +0100 Subject: [fix] handle empty response --- searx/engines/microsoft_academic.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'searx') diff --git a/searx/engines/microsoft_academic.py b/searx/engines/microsoft_academic.py index 9387b08d0..9bac0069c 100644 --- a/searx/engines/microsoft_academic.py +++ b/searx/engines/microsoft_academic.py @@ -45,6 +45,8 @@ def request(query, params): def response(resp): results = [] response_data = loads(resp.text) + if not response_data: + return results for result in response_data['results']: url = _get_url(result) -- cgit v1.2.3 From ad5bb994b1cff56c4f021f88bfa62f38055f1416 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 2 Jan 2020 22:30:18 +0100 Subject: [fix] add py3 compatibility --- searx/engines/scanr_structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx') diff --git a/searx/engines/scanr_structures.py b/searx/engines/scanr_structures.py index 72fd2b3c9..7208dcb70 100644 --- a/searx/engines/scanr_structures.py +++ b/searx/engines/scanr_structures.py @@ -29,7 +29,7 @@ def request(query, params): params['url'] = search_url params['method'] = 'POST' params['headers']['Content-type'] = "application/json" - params['data'] = dumps({"query": query, + params['data'] = dumps({"query": query.decode('utf-8'), "searchField": "ALL", "sortDirection": "ASC", "sortOrder": "RELEVANCY", -- cgit v1.2.3 From 17b6faa4c3c1cf14a327f4a3538fc70dce08b756 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Thu, 2 Jan 2020 22:37:06 +0100 Subject: [fix] pep8 --- searx/engines/bing.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'searx') diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 24776c400..b193f7c60 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -63,8 +63,6 @@ def response(resp): results = [] result_len = 0 - - dom = html.fromstring(resp.text) # parse results for result in eval_xpath(dom, '//div[@class="sa_cc"]'): -- cgit v1.2.3