diff options
49 files changed, 268 insertions, 280 deletions
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 116eadc97..87b1b0eb4 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -90,6 +90,9 @@ def load_engine(engine_data): 'result_count': 0, 'search_count': 0, 'page_load_time': 0, + 'page_load_count': 0, + 'engine_time': 0, + 'engine_time_count': 0, 'score_count': 0, 'errors': 0 } @@ -106,32 +109,56 @@ def load_engine(engine_data): return engine +def to_percentage(stats, maxvalue): + for engine_stat in stats: + if maxvalue: + engine_stat['percentage'] = int(engine_stat['avg'] / maxvalue * 100) + else: + engine_stat['percentage'] = 0 + return stats + + def get_engines_stats(): # TODO refactor pageloads = [] + engine_times = [] results = [] scores = [] errors = [] scores_per_result = [] - max_pageload = max_results = max_score = max_errors = max_score_per_result = 0 # noqa + max_pageload = max_engine_times = max_results = max_score = max_errors = max_score_per_result = 0 # noqa for engine in engines.values(): if engine.stats['search_count'] == 0: continue results_num = \ engine.stats['result_count'] / float(engine.stats['search_count']) - load_times = engine.stats['page_load_time'] / float(engine.stats['search_count']) # noqa + + if engine.stats['page_load_count'] != 0: + load_times = engine.stats['page_load_time'] / float(engine.stats['page_load_count']) # noqa + else: + load_times = 0 + + if engine.stats['engine_time_count'] != 0: + this_engine_time = engine.stats['engine_time'] / float(engine.stats['engine_time_count']) # noqa + else: + this_engine_time = 0 + if results_num: score = engine.stats['score_count'] / float(engine.stats['search_count']) # noqa score_per_result = score / results_num else: score = score_per_result = 0.0 - max_results = max(results_num, max_results) + max_pageload = max(load_times, max_pageload) + max_engine_times = max(this_engine_time, max_engine_times) + max_results = max(results_num, max_results) max_score = max(score, max_score) max_score_per_result = max(score_per_result, max_score_per_result) max_errors = max(max_errors, engine.stats['errors']) + pageloads.append({'avg': load_times, 'name': engine.name}) + engine_times.append({'avg': this_engine_time, 'name': engine.name}) results.append({'avg': results_num, 'name': engine.name}) scores.append({'avg': score, 'name': engine.name}) errors.append({'avg': engine.stats['errors'], 'name': engine.name}) @@ -140,39 +167,19 @@ def get_engines_stats(): 'name': engine.name }) - for engine in pageloads: - if max_pageload: - engine['percentage'] = int(engine['avg'] / max_pageload * 100) - else: - engine['percentage'] = 0 - - for engine in results: - if max_results: - engine['percentage'] = int(engine['avg'] / max_results * 100) - else: - engine['percentage'] = 0 - - for engine in scores: - if max_score: - engine['percentage'] = int(engine['avg'] / max_score * 100) - else: - engine['percentage'] = 0 - - for engine in scores_per_result: - if max_score_per_result: - engine['percentage'] = int(engine['avg'] - / max_score_per_result * 100) - else: - engine['percentage'] = 0 - - for engine in errors: - if max_errors: - engine['percentage'] = int(float(engine['avg']) / max_errors * 100) - else: - engine['percentage'] = 0 + pageloads = to_percentage(pageloads, max_pageload) + engine_times = to_percentage(engine_times, max_engine_times) + results = to_percentage(results, max_results) + scores = to_percentage(scores, max_score) + scores_per_result = to_percentage(scores_per_result, max_score_per_result) + erros = to_percentage(errors, max_errors) return [ ( + gettext('Engine time (sec)'), + sorted(engine_times, key=itemgetter('avg')) + ), + ( gettext('Page loads (sec)'), sorted(pageloads, key=itemgetter('avg')) ), diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py index b846934f7..5ba512766 100644 --- a/searx/engines/archlinux.py +++ b/searx/engines/archlinux.py @@ -12,7 +12,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text @@ -135,7 +134,7 @@ def response(resp): for result in dom.xpath(xpath_results): link = result.xpath(xpath_link)[0] href = urljoin(base_url, link.attrib.get('href')) - title = escape(extract_text(link)) + title = extract_text(link) results.append({'url': href, 'title': title}) diff --git a/searx/engines/base.py b/searx/engines/base.py index 66491d395..a552453ce 100755 --- a/searx/engines/base.py +++ b/searx/engines/base.py @@ -16,7 +16,6 @@ from lxml import etree from urllib import urlencode from searx.utils import searx_useragent -from cgi import escape from datetime import datetime import re @@ -94,7 +93,7 @@ def response(resp): url = item.text elif item.attrib["name"] == "dcdescription": - content = escape(item.text[:300]) + content = item.text[:300] if len(item.text) > 300: content += "..." diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 540597162..58db61251 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -14,7 +14,6 @@ """ from urllib import urlencode -from cgi import escape from lxml import html from searx.engines.xpath import extract_text @@ -61,7 +60,7 @@ def response(resp): link = result.xpath('.//h3/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = escape(extract_text(result.xpath('.//p'))) + content = extract_text(result.xpath('.//p')) # append result results.append({'url': url, @@ -73,7 +72,7 @@ def response(resp): link = result.xpath('.//h2/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = escape(extract_text(result.xpath('.//p'))) + content = extract_text(result.xpath('.//p')) # append result results.append({'url': url, diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index ea6baf1c8..33c8355de 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -11,7 +11,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetter @@ -51,8 +50,8 @@ def response(resp): for result in search_res: link = result.xpath('.//td[@class="torrent_name"]//a')[0] href = urljoin(url, link.attrib.get('href')) - title = escape(extract_text(link)) - content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0])) + title = extract_text(link) + content = extract_text(result.xpath('.//pre[@class="snippet"]')[0]) content = "<br />".join(content.split("\n")) filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0] diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py index 4eb894725..317f34f59 100644 --- a/searx/engines/dailymotion.py +++ b/searx/engines/dailymotion.py @@ -14,7 +14,6 @@ from urllib import urlencode from json import loads -from cgi import escape from datetime import datetime # engine dependent config @@ -57,7 +56,7 @@ def response(resp): for res in search_res['list']: title = res['title'] url = res['url'] - content = escape(res['description']) + content = res['description'] thumbnail = res['thumbnail_360_url'] publishedDate = datetime.fromtimestamp(res['created_time'], None) embedded = embedded_url.format(videoid=res['id']) diff --git a/searx/engines/deezer.py b/searx/engines/deezer.py index 0530bc072..3db1af3d2 100644 --- a/searx/engines/deezer.py +++ b/searx/engines/deezer.py @@ -51,10 +51,11 @@ def response(resp): if url.startswith('http://'): url = 'https' + url[4:] - content = result['artist']['name'] +\ - " • " +\ - result['album']['title'] +\ - " • " + result['title'] + content = u'{} - {} - {}'.format( + result['artist']['name'], + result['album']['title'], + result['title']) + embedded = embedded_url.format(audioid=result['id']) # append result diff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py index 9765d5f60..20a9a8980 100644 --- a/searx/engines/dictzone.py +++ b/searx/engines/dictzone.py @@ -12,7 +12,6 @@ import re from urlparse import urljoin from lxml import html -from cgi import escape from searx.utils import is_valid_lang categories = ['general'] @@ -62,8 +61,8 @@ def response(resp): results.append({ 'url': urljoin(resp.url, '?%d' % k), - 'title': escape(from_result.text_content()), - 'content': escape('; '.join(to_results)) + 'title': from_result.text_content(), + 'content': '; '.join(to_results) }) return results diff --git a/searx/engines/digg.py b/searx/engines/digg.py index a10b38bb6..238b466a0 100644 --- a/searx/engines/digg.py +++ b/searx/engines/digg.py @@ -13,7 +13,6 @@ from urllib import quote_plus from json import loads from lxml import html -from cgi import escape from dateutil import parser # engine dependent config @@ -56,7 +55,7 @@ def response(resp): url = result.attrib.get('data-contenturl') thumbnail = result.xpath('.//img')[0].attrib.get('src') title = ''.join(result.xpath(title_xpath)) - content = escape(''.join(result.xpath(content_xpath))) + content = ''.join(result.xpath(content_xpath)) pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime') publishedDate = parser.parse(pubdate) diff --git a/searx/engines/fdroid.py b/searx/engines/fdroid.py index 0b16773e3..6d470a4eb 100644 --- a/searx/engines/fdroid.py +++ b/searx/engines/fdroid.py @@ -9,7 +9,6 @@ @parse url, title, content """ -from cgi import escape from urllib import urlencode from searx.engines.xpath import extract_text from lxml import html @@ -43,7 +42,7 @@ def response(resp): img_src = app.xpath('.//img/@src')[0] content = extract_text(app.xpath('./p')[0]) - content = escape(content.replace(title, '', 1).strip()) + content = content.replace(title, '', 1).strip() results.append({'url': url, 'title': title, diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py index 68d45bc17..5ce1160e9 100644 --- a/searx/engines/flickr.py +++ b/searx/engines/flickr.py @@ -77,21 +77,13 @@ def response(resp): url = build_flickr_url(photo['owner'], photo['id']) - title = photo['title'] - - content = '<span class="photo-author">' +\ - photo['ownername'] +\ - '</span><br />' +\ - '<span class="description">' +\ - photo['description']['_content'] +\ - '</span>' - # append result results.append({'url': url, - 'title': title, + 'title': photo['title'], 'img_src': img_src, 'thumbnail_src': thumbnail_src, - 'content': content, + 'content': photo['description']['_content'], + 'author': photo['ownername'], 'template': 'images.html'}) # return results diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py index 5c4193c11..68be139be 100644 --- a/searx/engines/flickr_noapi.py +++ b/searx/engines/flickr_noapi.py @@ -102,16 +102,15 @@ def response(resp): title = photo.get('title', '') - content = '<span class="photo-author">' +\ - photo['username'] +\ - '</span><br />' + author = photo['username'] # append result results.append({'url': url, 'title': title, 'img_src': img_src, 'thumbnail_src': thumbnail_src, - 'content': content, + 'content': '', + 'author': author, 'template': 'images.html'}) return results diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 6e4e24b68..5430eb3ba 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -10,7 +10,6 @@ @parse url, title, content """ -from cgi import escape from json import loads from random import randint from time import time @@ -78,8 +77,8 @@ def response(resp): for result in response_json['results']: # append result results.append({'url': result['url'], - 'title': escape(result['title']), - 'content': escape(result['sum'])}) + 'title': result['title'], + 'content': result['sum']}) # return results return results diff --git a/searx/engines/github.py b/searx/engines/github.py index cc1fc470c..7adef3be9 100644 --- a/searx/engines/github.py +++ b/searx/engines/github.py @@ -12,7 +12,6 @@ from urllib import urlencode from json import loads -from cgi import escape # engine dependent config categories = ['it'] @@ -48,7 +47,7 @@ def response(resp): url = res['html_url'] if res['description']: - content = escape(res['description'][:500]) + content = res['description'][:500] else: content = '' diff --git a/searx/engines/google.py b/searx/engines/google.py index ea93bc94f..0e2d522f4 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -9,7 +9,6 @@ # @parse url, title, content, suggestion import re -from cgi import escape from urllib import urlencode from urlparse import urlparse, parse_qsl from lxml import html, etree @@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname): def extract_text_from_dom(result, xpath): r = result.xpath(xpath) if len(r) > 0: - return escape(extract_text(r[0])) + return extract_text(r[0]) return None @@ -264,7 +263,7 @@ def response(resp): # parse suggestion for suggestion in dom.xpath(suggestion_xpath): # append suggestion - results.append({'suggestion': escape(extract_text(suggestion))}) + results.append({'suggestion': extract_text(suggestion)}) # return results return results diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py index 9cd8284da..059fa2a66 100644 --- a/searx/engines/kickass.py +++ b/searx/engines/kickass.py @@ -11,7 +11,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetter @@ -57,7 +56,7 @@ def response(resp): link = result.xpath('.//a[@class="cellMainLink"]')[0] href = urljoin(url, link.attrib['href']) title = extract_text(link) - content = escape(extract_text(result.xpath(content_xpath))) + content = extract_text(result.xpath(content_xpath)) seed = extract_text(result.xpath('.//td[contains(@class, "green")]')) leech = extract_text(result.xpath('.//td[contains(@class, "red")]')) filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]')) diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py index cda8231f7..4ca5b3171 100644 --- a/searx/engines/nyaa.py +++ b/searx/engines/nyaa.py @@ -9,7 +9,6 @@ @parse url, title, content, seed, leech, torrentfile """ -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text @@ -78,7 +77,7 @@ def response(resp): # torrent title page_a = result.xpath(xpath_title)[0] - title = escape(extract_text(page_a)) + title = extract_text(page_a) # link to the page href = page_a.attrib.get('href') @@ -90,7 +89,7 @@ def response(resp): try: file_size, suffix = result.xpath(xpath_filesize)[0].split(' ') file_size = int(float(file_size) * get_filesize_mul(suffix)) - except Exception as e: + except: file_size = None # seed count @@ -105,7 +104,6 @@ def response(resp): # content string contains all information not included into template content = 'Category: "{category}". Downloaded {downloads} times.' content = content.format(category=category, downloads=downloads) - content = escape(content) results.append({'url': href, 'title': title, diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py index 38baaada9..01ca7d42d 100644 --- a/searx/engines/openstreetmap.py +++ b/searx/engines/openstreetmap.py @@ -43,7 +43,7 @@ def response(resp): if 'display_name' not in r: continue - title = r['display_name'] + title = r['display_name'] or u'' osm_type = r.get('osm_type', r.get('type')) url = result_base_url.format(osm_type=osm_type, osm_id=r['osm_id']) diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py index 55446b410..ca21a3bb2 100644 --- a/searx/engines/piratebay.py +++ b/searx/engines/piratebay.py @@ -9,7 +9,6 @@ # @parse url, title, content, seed, leech, magnetlink from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetter @@ -62,7 +61,7 @@ def response(resp): link = result.xpath('.//div[@class="detName"]//a')[0] href = urljoin(url, link.attrib.get('href')) title = extract_text(link) - content = escape(extract_text(result.xpath(content_xpath))) + content = extract_text(result.xpath(content_xpath)) seed, leech = result.xpath('.//td[@align="right"]/text()')[:2] # convert seed to int if possible diff --git a/searx/engines/reddit.py b/searx/engines/reddit.py index 3ca7e44f6..b29792a3a 100644 --- a/searx/engines/reddit.py +++ b/searx/engines/reddit.py @@ -11,7 +11,6 @@ """ import json -from cgi import escape from urllib import urlencode from urlparse import urlparse, urljoin from datetime import datetime @@ -68,7 +67,7 @@ def response(resp): img_results.append(params) else: created = datetime.fromtimestamp(data['created_utc']) - content = escape(data['selftext']) + content = data['selftext'] if len(content) > 500: content = content[:500] + '...' params['content'] = content diff --git a/searx/engines/searchcode_doc.py b/searx/engines/searchcode_doc.py index f24fe6f90..6c1acdcdd 100644 --- a/searx/engines/searchcode_doc.py +++ b/searx/engines/searchcode_doc.py @@ -44,20 +44,12 @@ def response(resp): # parse results for result in search_results.get('results', []): href = result['url'] - title = "[" + result['type'] + "] " +\ - result['namespace'] +\ - " " + result['name'] - content = '<span class="highlight">[' +\ - result['type'] + "] " +\ - result['name'] + " " +\ - result['synopsis'] +\ - "</span><br />" +\ - result['description'] + title = "[{}] {} {}".format(result['type'], result['namespace'], result['name']) # append result results.append({'url': href, 'title': title, - 'content': content}) + 'content': result['description']}) # return results return results diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py index 854ebba03..e1309a9b5 100644 --- a/searx/engines/seedpeer.py +++ b/searx/engines/seedpeer.py @@ -9,7 +9,6 @@ # @parse url, title, content, seed, leech, magnetlink from urlparse import urljoin -from cgi import escape from urllib import quote from lxml import html from operator import itemgetter diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py index f75796e83..249ba91ef 100644 --- a/searx/engines/spotify.py +++ b/searx/engines/spotify.py @@ -46,10 +46,11 @@ def response(resp): if result['type'] == 'track': title = result['name'] url = result['external_urls']['spotify'] - content = result['artists'][0]['name'] +\ - " • " +\ - result['album']['name'] +\ - " • " + result['name'] + content = u'{} - {} - {}'.format( + result['artists'][0]['name'], + result['album']['name'], + result['name']) + embedded = embedded_url.format(audioid=result['id']) # append result diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py index fdd3711a9..5e7ab2901 100644 --- a/searx/engines/stackoverflow.py +++ b/searx/engines/stackoverflow.py @@ -11,7 +11,6 @@ """ from urlparse import urljoin -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text @@ -48,8 +47,8 @@ def response(resp): for result in dom.xpath(results_xpath): link = result.xpath(link_xpath)[0] href = urljoin(url, link.attrib.get('href')) - title = escape(extract_text(link)) - content = escape(extract_text(result.xpath(content_xpath))) + title = extract_text(link) + content = extract_text(result.xpath(content_xpath)) # append result results.append({'url': href, diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index d8b702c4d..6f6eae1cf 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -11,7 +11,6 @@ # @todo paging from lxml import html -from cgi import escape from dateutil import parser from datetime import datetime, timedelta import re @@ -79,10 +78,10 @@ def response(resp): if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url): continue - title = escape(extract_text(link)) + title = extract_text(link) if result.xpath('./p[@class="desc clk"]'): - content = escape(extract_text(result.xpath('./p[@class="desc clk"]'))) + content = extract_text(result.xpath('./p[@class="desc clk"]')) else: content = '' diff --git a/searx/engines/subtitleseeker.py b/searx/engines/subtitleseeker.py index 47d27d0b2..daba68be7 100644 --- a/searx/engines/subtitleseeker.py +++ b/searx/engines/subtitleseeker.py @@ -10,7 +10,6 @@ @parse url, title, content """ -from cgi import escape from urllib import quote_plus from lxml import html from searx.languages import language_codes @@ -59,7 +58,7 @@ def response(resp): elif search_lang: href = href + search_lang + '/' - title = escape(extract_text(link)) + title = extract_text(link) content = extract_text(result.xpath('.//div[contains(@class,"red")]')) content = content + " - " @@ -75,7 +74,7 @@ def response(resp): # append result results.append({'url': href, 'title': title, - 'content': escape(content)}) + 'content': content}) # return results return results diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py index 1a94ed64e..72184e428 100644 --- a/searx/engines/swisscows.py +++ b/searx/engines/swisscows.py @@ -10,7 +10,6 @@ @parse url, title, content """ -from cgi import escape from json import loads from urllib import urlencode, unquote import re @@ -78,7 +77,7 @@ def response(resp): # append result results.append({'url': result['SourceUrl'], - 'title': escape(result['Title']), + 'title': result['Title'], 'content': '', 'img_src': img_url, 'template': 'images.html'}) @@ -90,8 +89,8 @@ def response(resp): # append result results.append({'url': result_url, - 'title': escape(result_title), - 'content': escape(result_content)}) + 'title': result_title, + 'content': result_content}) # parse images for result in json.get('Images', []): @@ -100,7 +99,7 @@ def response(resp): # append result results.append({'url': result['SourceUrl'], - 'title': escape(result['Title']), + 'title': result['Title'], 'content': '', 'img_src': img_url, 'template': 'images.html'}) diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py index e2990e153..52b2cbe07 100644 --- a/searx/engines/tokyotoshokan.py +++ b/searx/engines/tokyotoshokan.py @@ -11,7 +11,6 @@ """ import re -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text diff --git a/searx/engines/torrentz.py b/searx/engines/torrentz.py index 92fbe7013..f9c832651 100644 --- a/searx/engines/torrentz.py +++ b/searx/engines/torrentz.py @@ -12,7 +12,6 @@ """ import re -from cgi import escape from urllib import urlencode from lxml import html from searx.engines.xpath import extract_text diff --git a/searx/engines/translated.py b/searx/engines/translated.py index 02047bc93..e78db0d8e 100644 --- a/searx/engines/translated.py +++ b/searx/engines/translated.py @@ -9,7 +9,6 @@ @parse url, title, content """ import re -from cgi import escape from searx.utils import is_valid_lang categories = ['general'] @@ -52,14 +51,14 @@ def request(query, params): def response(resp): results = [] results.append({ - 'url': escape(web_url.format( + 'url': web_url.format( from_lang=resp.search_params['from_lang'][2], to_lang=resp.search_params['to_lang'][2], - query=resp.search_params['query'])), - 'title': escape('[{0}-{1}] {2}'.format( + query=resp.search_params['query']), + 'title': '[{0}-{1}] {2}'.format( resp.search_params['from_lang'][1], resp.search_params['to_lang'][1], - resp.search_params['query'])), - 'content': escape(resp.json()['responseData']['translatedText']) + resp.search_params['query']), + 'content': resp.json()['responseData']['translatedText'] }) return results diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py index e318d93e6..1534501b3 100644 --- a/searx/engines/wolframalpha_noapi.py +++ b/searx/engines/wolframalpha_noapi.py @@ -8,7 +8,6 @@ # @stable no # @parse url, infobox -from cgi import escape from json import loads from time import time from urllib import urlencode diff --git a/searx/engines/yandex.py b/searx/engines/yandex.py index be3ec36ce..938fdd184 100644 --- a/searx/engines/yandex.py +++ b/searx/engines/yandex.py @@ -9,7 +9,6 @@ @parse url, title, content """ -from cgi import escape from urllib import urlencode from lxml import html from searx.search import logger @@ -52,8 +51,8 @@ def response(resp): for result in dom.xpath(results_xpath): try: res = {'url': result.xpath(url_xpath)[0], - 'title': escape(''.join(result.xpath(title_xpath))), - 'content': escape(''.join(result.xpath(content_xpath)))} + 'title': ''.join(result.xpath(title_xpath)), + 'content': ''.join(result.xpath(content_xpath))} except: logger.exception('yandex parse crash') continue diff --git a/searx/search.py b/searx/search.py index 0095de821..cef42d876 100644 --- a/searx/search.py +++ b/searx/search.py @@ -36,14 +36,53 @@ logger = logger.getChild('search') number_of_searches = 0 -def search_request_wrapper(fn, url, engine_name, **kwargs): - ret = None - engine = engines[engine_name] +def send_http_request(engine, request_params, timeout_limit): + response = None try: - ret = fn(url, **kwargs) + # create dictionary which contain all + # informations about the request + request_args = dict( + headers=request_params['headers'], + cookies=request_params['cookies'], + timeout=timeout_limit, + verify=request_params['verify'] + ) + # specific type of request (GET or POST) + if request_params['method'] == 'GET': + req = requests_lib.get + else: + req = requests_lib.post + request_args['data'] = request_params['data'] + + # for page_load_time stats + time_before_request = time() + + # send the request + response = req(request_params['url'], **request_args) + with threading.RLock(): + # no error : reset the suspend variables engine.continuous_errors = 0 engine.suspend_end_time = 0 + # update stats with current page-load-time + # only the HTTP request + engine.stats['page_load_time'] += time() - time_before_request + engine.stats['page_load_count'] += 1 + + # is there a timeout (no parsing in this case) + timeout_overhead = 0.2 # seconds + search_duration = time() - request_params['started'] + if search_duration > timeout_limit + timeout_overhead: + logger.exception('engine timeout on HTTP request:' + '{0} (search duration : {1} ms, time-out: {2} )' + .format(engine.name, search_duration, timeout_limit)) + with threading.RLock(): + engine.stats['errors'] += 1 + return False + + # everything is ok : return the response + return response + except: # increase errors stats with threading.RLock(): @@ -52,20 +91,65 @@ def search_request_wrapper(fn, url, engine_name, **kwargs): engine.suspend_end_time = time() + min(60, engine.continuous_errors) # print engine name and specific error message - logger.exception('engine crash: {0}'.format(engine_name)) - return ret + logger.exception('engine crash: {0}'.format(engine.name)) + return False + + +def search_one_request(engine_name, query, request_params, result_container, timeout_limit): + engine = engines[engine_name] + + # update request parameters dependent on + # search-engine (contained in engines folder) + engine.request(query, request_params) + + # TODO add support of offline engines + if request_params['url'] is None: + return False + + # ignoring empty urls + if not request_params['url']: + return False + + # send request + response = send_http_request(engine, request_params, timeout_limit) + + # parse response + success = None + if response: + # parse the response + response.search_params = request_params + try: + search_results = engine.response(response) + except: + logger.exception('engine crash: {0}'.format(engine.name)) + search_results = [] + + # add results + for result in search_results: + result['engine'] = engine.name + + result_container.extend(engine.name, search_results) + + success = True + else: + success = False + + with threading.RLock(): + # update stats : total time + engine.stats['engine_time'] += time() - request_params['started'] + engine.stats['engine_time_count'] += 1 + return success -def threaded_requests(requests): - timeout_limit = max(r[2]['timeout'] for r in requests) - search_start = time() + +def search_multiple_requests(requests, result_container, timeout_limit): + start_time = time() search_id = uuid4().__str__() - for fn, url, request_args, engine_name in requests: - request_args['timeout'] = timeout_limit + + for engine_name, query, request_params in requests: th = threading.Thread( - target=search_request_wrapper, - args=(fn, url, engine_name), - kwargs=request_args, + target=search_one_request, + args=(engine_name, query, request_params, result_container, timeout_limit), name=search_id, ) th._engine_name = engine_name @@ -73,7 +157,7 @@ def threaded_requests(requests): for th in threading.enumerate(): if th.name == search_id: - remaining_time = max(0.0, timeout_limit - (time() - search_start)) + remaining_time = max(0.0, timeout_limit - (time() - start_time)) th.join(remaining_time) if th.isAlive(): logger.warning('engine timeout: {0}'.format(th._engine_name)) @@ -91,49 +175,10 @@ def default_request_params(): } -# create a callback wrapper for the search engine results -def make_callback(engine_name, callback, params, result_container): - - # creating a callback wrapper for the search engine results - def process_callback(response, **kwargs): - # check if redirect comparing to the True value, - # because resp can be a Mock object, and any attribut name returns something. - if response.is_redirect is True: - logger.debug('{0} redirect on: {1}'.format(engine_name, response)) - return - - response.search_params = params - - search_duration = time() - params['started'] - # update stats with current page-load-time - with threading.RLock(): - engines[engine_name].stats['page_load_time'] += search_duration - - timeout_overhead = 0.2 # seconds - timeout_limit = engines[engine_name].timeout + timeout_overhead - - if search_duration > timeout_limit: - with threading.RLock(): - engines[engine_name].stats['errors'] += 1 - return - - # callback - search_results = callback(response) - - # add results - for result in search_results: - result['engine'] = engine_name - - result_container.extend(engine_name, search_results) - - return process_callback - - def get_search_query_from_webapp(preferences, form): query = None query_engines = [] query_categories = [] - query_paging = False query_pageno = 1 query_lang = 'all' query_time_range = None @@ -255,6 +300,10 @@ class Search(object): def search(self): global number_of_searches + # start time + start_time = time() + + # answeres ? answerers_results = ask(self.search_query) if answerers_results: @@ -274,6 +323,9 @@ class Search(object): search_query = self.search_query + # max of all selected engine timeout + timeout_limit = 0 + # start search-reqest for all selected engines for selected_engine in search_query.engines: if selected_engine['name'] not in engines: @@ -303,7 +355,7 @@ class Search(object): request_params = default_request_params() request_params['headers']['User-Agent'] = user_agent request_params['category'] = selected_engine['category'] - request_params['started'] = time() + request_params['started'] = start_time request_params['pageno'] = search_query.pageno if hasattr(engine, 'language') and engine.language: @@ -315,52 +367,16 @@ class Search(object): request_params['safesearch'] = search_query.safesearch request_params['time_range'] = search_query.time_range - # update request parameters dependent on - # search-engine (contained in engines folder) - engine.request(search_query.query.encode('utf-8'), request_params) - - if request_params['url'] is None: - # TODO add support of offline engines - pass - - # create a callback wrapper for the search engine results - callback = make_callback( - selected_engine['name'], - engine.response, - request_params, - self.result_container) - - # create dictionary which contain all - # informations about the request - request_args = dict( - headers=request_params['headers'], - hooks=dict(response=callback), - cookies=request_params['cookies'], - timeout=engine.timeout, - verify=request_params['verify'] - ) - - # specific type of request (GET or POST) - if request_params['method'] == 'GET': - req = requests_lib.get - else: - req = requests_lib.post - request_args['data'] = request_params['data'] - - # ignoring empty urls - if not request_params['url']: - continue - # append request to list - requests.append((req, request_params['url'], - request_args, - selected_engine['name'])) + requests.append((selected_engine['name'], search_query.query.encode('utf-8'), request_params)) - if not requests: - return self.result_container - # send all search-request - threaded_requests(requests) - start_new_thread(gc.collect, tuple()) + # update timeout_limit + timeout_limit = max(timeout_limit, engine.timeout) + + if requests: + # send all search-request + search_multiple_requests(requests, self.result_container, timeout_limit - (time() - start_time)) + start_new_thread(gc.collect, tuple()) # return results, suggestions, answers and infoboxes return self.result_container diff --git a/searx/static/plugins/js/infinite_scroll.js b/searx/static/plugins/js/infinite_scroll.js index 213f74b15..9cd582d7f 100644 --- a/searx/static/plugins/js/infinite_scroll.js +++ b/searx/static/plugins/js/infinite_scroll.js @@ -5,7 +5,7 @@ $(document).ready(function() { var formData = $('#pagination form:last').serialize(); if (formData) { $('#pagination').html('<div class="loading-spinner"></div>'); - $.post('/', formData, function (data) { + $.post('./', formData, function (data) { var body = $(data); $('#pagination').remove(); $('#main_results').append('<hr/>'); diff --git a/searx/templates/courgette/opensearch_response_rss.xml b/searx/templates/courgette/opensearch_response_rss.xml index 5673eb2e1..ddb60fa5e 100644 --- a/searx/templates/courgette/opensearch_response_rss.xml +++ b/searx/templates/courgette/opensearch_response_rss.xml @@ -3,14 +3,14 @@ xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:atom="http://www.w3.org/2005/Atom"> <channel> - <title>Searx search: {{ q }}</title> - <link>{{ base_url }}?q={{ q }}</link> - <description>Search results for "{{ q }}" - searx</description> + <title>Searx search: {{ q|e }}</title> + <link>{{ base_url }}?q={{ q|e }}</link> + <description>Search results for "{{ q|e }}" - searx</description> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:startIndex>1</opensearch:startIndex> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> - <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> + <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" /> {% for r in results %} <item> <title>{{ r.title }}</title> diff --git a/searx/templates/courgette/results.html b/searx/templates/courgette/results.html index 3ffbd5882..c72b7c3f7 100644 --- a/searx/templates/courgette/results.html +++ b/searx/templates/courgette/results.html @@ -1,6 +1,6 @@ {% extends "courgette/base.html" %} -{% block title %}{{ q }} - {% endblock %} -{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} +{% block title %}{{ q|e }} - {% endblock %} +{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} {% block content %} <div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div> <div class="small search center"> @@ -17,7 +17,7 @@ {% for output_type in ('csv', 'json', 'rss') %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="format" value="{{ output_type }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> @@ -62,7 +62,7 @@ {% if pageno > 1 %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} @@ -76,7 +76,7 @@ {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="submit" value="{{ _('next page') }} >>" /> </div> diff --git a/searx/templates/legacy/opensearch_response_rss.xml b/searx/templates/legacy/opensearch_response_rss.xml index 5673eb2e1..ddb60fa5e 100644 --- a/searx/templates/legacy/opensearch_response_rss.xml +++ b/searx/templates/legacy/opensearch_response_rss.xml @@ -3,14 +3,14 @@ xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:atom="http://www.w3.org/2005/Atom"> <channel> - <title>Searx search: {{ q }}</title> - <link>{{ base_url }}?q={{ q }}</link> - <description>Search results for "{{ q }}" - searx</description> + <title>Searx search: {{ q|e }}</title> + <link>{{ base_url }}?q={{ q|e }}</link> + <description>Search results for "{{ q|e }}" - searx</description> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:startIndex>1</opensearch:startIndex> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> - <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> + <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" /> {% for r in results %} <item> <title>{{ r.title }}</title> diff --git a/searx/templates/legacy/results.html b/searx/templates/legacy/results.html index f50700c6f..f0d78398d 100644 --- a/searx/templates/legacy/results.html +++ b/searx/templates/legacy/results.html @@ -1,6 +1,6 @@ {% extends "legacy/base.html" %} -{% block title %}{{ q }} - {% endblock %} -{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} +{% block title %}{{ q|e }} - {% endblock %} +{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %} {% block content %} <div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div> <div class="small search center"> @@ -18,7 +18,7 @@ {% for output_type in ('csv', 'json', 'rss') %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="left"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="format" value="{{ output_type }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> @@ -73,7 +73,7 @@ {% if pageno > 1 %} <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}"> <div class="{% if rtl %}right{% else %}left{% endif %}"> - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} @@ -87,7 +87,7 @@ {% for category in selected_categories %} <input type="hidden" name="category_{{ category }}" value="1"/> {% endfor %} - <input type="hidden" name="q" value="{{ q }}" /> + <input type="hidden" name="q" value="{{ q|e }}" /> <input type="hidden" name="pageno" value="{{ pageno+1 }}" /> <input type="submit" value="{{ _('next page') }} >>" /> </div> diff --git a/searx/templates/oscar/opensearch_response_rss.xml b/searx/templates/oscar/opensearch_response_rss.xml index 5673eb2e1..ddb60fa5e 100644 --- a/searx/templates/oscar/opensearch_response_rss.xml +++ b/searx/templates/oscar/opensearch_response_rss.xml @@ -3,14 +3,14 @@ xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/" xmlns:atom="http://www.w3.org/2005/Atom"> <channel> - <title>Searx search: {{ q }}</title> - <link>{{ base_url }}?q={{ q }}</link> - <description>Search results for "{{ q }}" - searx</description> + <title>Searx search: {{ q|e }}</title> + <link>{{ base_url }}?q={{ q|e }}</link> + <description>Search results for "{{ q|e }}" - searx</description> <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults> <opensearch:startIndex>1</opensearch:startIndex> <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage> <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/> - <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" /> + <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" /> {% for r in results %} <item> <title>{{ r.title }}</title> diff --git a/searx/templates/oscar/result_templates/images.html b/searx/templates/oscar/result_templates/images.html index f7bcf0786..b23f34915 100644 --- a/searx/templates/oscar/result_templates/images.html +++ b/searx/templates/oscar/result_templates/images.html @@ -13,7 +13,12 @@ </div>
<div class="modal-body">
<img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">
- {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %}
+ {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}
+ {% if result.content %}
+ <p class="result-content">
+ {{ result.content }}
+ </p>
+ {% endif %}
</div>
<div class="modal-footer">
<div class="clearfix"></div>
diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html index e71be325a..0ae83e74b 100644 --- a/searx/templates/oscar/results.html +++ b/searx/templates/oscar/results.html @@ -1,6 +1,6 @@ {% extends "oscar/base.html" %}
-{% block title %}{{ q }} - {% endblock %}
-{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}
+{% block title %}{{ q|e }} - {% endblock %}
+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}
{% block content %}
<div class="row">
<div class="col-sm-8" id="main_results">
@@ -37,9 +37,9 @@ <div id="pagination">
<div class="pull-left">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>
@@ -59,7 +59,7 @@ <div id="pagination">
<div class="pull-left">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno-1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
@@ -69,7 +69,7 @@ <div class="pull-right">
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}
- <input type="hidden" name="q" value="{{ q }}" />
+ <input type="hidden" name="q" value="{{ q|e }}" />
<input type="hidden" name="pageno" value="{{ pageno+1 }}" />
<input type="hidden" name="time_range" value="{{ time_range }}" />
<button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>
@@ -130,7 +130,7 @@ <div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">
- <input type="hidden" name="q" value="{{ q }}">
+ <input type="hidden" name="q" value="{{ q|e }}">
<input type="hidden" name="format" value="{{ output_type }}">
{% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %}
<input type="hidden" name="pageno" value="{{ pageno }}">
diff --git a/searx/templates/pix-art/results.html b/searx/templates/pix-art/results.html index f7d0e209b..8999e0513 100644 --- a/searx/templates/pix-art/results.html +++ b/searx/templates/pix-art/results.html @@ -5,7 +5,7 @@ {% endfor %} {% else %} {% extends "pix-art/base.html" %} -{% block title %}{{ q }} - {% endblock %} +{% block title %}{{ q|e }} - {% endblock %} {% block meta %}{% endblock %} {% block content %} <div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div> @@ -25,8 +25,8 @@ </span> <div id="pagination"> <br /> - <input type="button" onclick="load_more('{{ q }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" /> + <input type="button" onclick="load_more('{{ q|e }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" /> </div> </div> {% endblock %} -{% endif %}
\ No newline at end of file +{% endif %} diff --git a/searx/webapp.py b/searx/webapp.py index 8996aa2b9..7d4936f18 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -40,7 +40,7 @@ except: logger.critical("cannot import dependency: pygments") from sys import exit exit(1) - +from cgi import escape from datetime import datetime, timedelta from urllib import urlencode from urlparse import urlparse, urljoin @@ -433,8 +433,10 @@ def index(): for result in results: if output_format == 'html': if 'content' in result and result['content']: - result['content'] = highlight_content(result['content'][:1024], search_query.query.encode('utf-8')) - result['title'] = highlight_content(result['title'], search_query.query.encode('utf-8')) + result['content'] = highlight_content(escape(result['content'][:1024]), + search_query.query.encode('utf-8')) + result['title'] = highlight_content(escape(result['title'] or u''), + search_query.query.encode('utf-8')) else: if result.get('content'): result['content'] = html_to_text(result['content']).strip() @@ -599,6 +601,8 @@ def preferences(): if e.timeout > settings['outgoing']['request_timeout']: stats[e.name]['warn_timeout'] = True + # get first element [0], the engine time, + # and then the second element [1] : the time (the first one is the label) for engine_stat in get_engines_stats()[0][1]: stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3) if engine_stat.get('avg') > settings['outgoing']['request_timeout']: diff --git a/tests/unit/engines/test_deezer.py b/tests/unit/engines/test_deezer.py index cfef852af..5b9f55c33 100644 --- a/tests/unit/engines/test_deezer.py +++ b/tests/unit/engines/test_deezer.py @@ -42,7 +42,7 @@ class TestDeezerEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'Title of track') self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042') - self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') + self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track') self.assertTrue('100' in results[0]['embedded']) json = r""" diff --git a/tests/unit/engines/test_flickr.py b/tests/unit/engines/test_flickr.py index 2d7472a92..be97647ce 100644 --- a/tests/unit/engines/test_flickr.py +++ b/tests/unit/engines/test_flickr.py @@ -52,7 +52,7 @@ class TestFlickrEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertTrue('o.jpg' in results[0]['img_src']) self.assertTrue('n.jpg' in results[0]['thumbnail_src']) - self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Owner' in results[0]['author']) self.assertTrue('Description' in results[0]['content']) json = r""" @@ -76,7 +76,7 @@ class TestFlickrEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertTrue('z.jpg' in results[0]['img_src']) self.assertTrue('z.jpg' in results[0]['thumbnail_src']) - self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Owner' in results[0]['author']) self.assertTrue('Description' in results[0]['content']) json = r""" @@ -100,7 +100,7 @@ class TestFlickrEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054') self.assertTrue('o.jpg' in results[0]['img_src']) self.assertTrue('o.jpg' in results[0]['thumbnail_src']) - self.assertTrue('Owner' in results[0]['content']) + self.assertTrue('Owner' in results[0]['author']) self.assertTrue('Description' in results[0]['content']) json = r""" diff --git a/tests/unit/engines/test_flickr_noapi.py b/tests/unit/engines/test_flickr_noapi.py index 6d09071bd..5f8b069e3 100644 --- a/tests/unit/engines/test_flickr_noapi.py +++ b/tests/unit/engines/test_flickr_noapi.py @@ -145,7 +145,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertIn('k.jpg', results[0]['img_src']) self.assertIn('n.jpg', results[0]['thumbnail_src']) - self.assertIn('Owner', results[0]['content']) + self.assertIn('Owner', results[0]['author']) # no n size, only the z size json = """ @@ -188,7 +188,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertIn('z.jpg', results[0]['img_src']) self.assertIn('z.jpg', results[0]['thumbnail_src']) - self.assertIn('Owner', results[0]['content']) + self.assertIn('Owner', results[0]['author']) # no z or n size json = """ @@ -231,7 +231,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434') self.assertIn('o.jpg', results[0]['img_src']) self.assertIn('o.jpg', results[0]['thumbnail_src']) - self.assertIn('Owner', results[0]['content']) + self.assertIn('Owner', results[0]['author']) # no image test json = """ diff --git a/tests/unit/engines/test_kickass.py b/tests/unit/engines/test_kickass.py index 96c17911c..3a75c6697 100644 --- a/tests/unit/engines/test_kickass.py +++ b/tests/unit/engines/test_kickass.py @@ -98,7 +98,7 @@ class TestKickassEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') - self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') + self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['leech'], 1) self.assertEqual(results[0]['filesize'], 449) @@ -381,7 +381,7 @@ class TestKickassEngine(SearxTestCase): self.assertEqual(len(results), 5) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html') - self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') + self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted') self.assertEqual(results[0]['seed'], 10) self.assertEqual(results[0]['leech'], 1) self.assertEqual(results[0]['files'], 4) diff --git a/tests/unit/engines/test_searchcode_doc.py b/tests/unit/engines/test_searchcode_doc.py index 7228613ed..d02bb7a44 100644 --- a/tests/unit/engines/test_searchcode_doc.py +++ b/tests/unit/engines/test_searchcode_doc.py @@ -56,9 +56,6 @@ class TestSearchcodeDocEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], '[Type] Namespace test') self.assertEqual(results[0]['url'], 'http://url') - self.assertIn('Synopsis', results[0]['content']) - self.assertIn('Type', results[0]['content']) - self.assertIn('test', results[0]['content']) self.assertIn('Description', results[0]['content']) json = r""" diff --git a/tests/unit/engines/test_spotify.py b/tests/unit/engines/test_spotify.py index fd274abbd..e37c344d2 100644 --- a/tests/unit/engines/test_spotify.py +++ b/tests/unit/engines/test_spotify.py @@ -90,7 +90,7 @@ class TestSpotifyEngine(SearxTestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'Title of track') self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa') - self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') + self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track') self.assertIn('1000', results[0]['embedded']) json = """ |