diff options
56 files changed, 693 insertions, 605 deletions
diff --git a/Dockerfile b/Dockerfile index f336d4b3d..b0b5a609d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,9 @@ ARG TIMESTAMP_UWSGI=0 ARG LABEL_VCS_REF= ARG LABEL_VCS_URL= -ENV BASE_URL= \ +ENV INSTANCE_NAME=searx \ + AUTOCOMPLETE= \ + BASE_URL= \ MORTY_KEY= \ MORTY_URL= diff --git a/dockerfiles/docker-entrypoint.sh b/dockerfiles/docker-entrypoint.sh index c731e0a89..8b4c34860 100755 --- a/dockerfiles/docker-entrypoint.sh +++ b/dockerfiles/docker-entrypoint.sh @@ -29,6 +29,8 @@ do printf " -f Always update on the configuration files (existing files are renamed with the .old suffix)\n" printf " Without this option, new configuration files are copied with the .new suffix\n" printf "\nEnvironment variables:\n\n" + printf " INSTANCE_NAME settings.yml : general.instance_name\n" + printf " AUTOCOMPLETE settings.yml : search.autocomplete\n" printf " BASE_URL settings.yml : server.base_url\n" printf " MORTY_URL settings.yml : result_proxy.url\n" printf " MORTY_KEY settings.yml : result_proxy.key\n" @@ -53,6 +55,8 @@ patch_searx_settings() { # update settings.yml sed -i -e "s|base_url : False|base_url : ${BASE_URL}|g" \ + -e "s/instance_name : \"searx\"/instance_name : \"${INSTANCE_NAME}\"/g" \ + -e "s/autocomplete : \"\"/autocomplete : \"${AUTOCOMPLETE}\"/g" \ -e "s/ultrasecretkey/$(openssl rand -hex 32)/g" \ "${CONF}" diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index a10b1ccd9..2393f52b6 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -27,7 +27,7 @@ from json import loads from requests import get from searx import settings from searx import logger -from searx.utils import load_module, match_language +from searx.utils import load_module, match_language, get_engine_from_settings logger = logger.getChild('engines') @@ -53,7 +53,8 @@ engine_default_args = {'paging': False, 'disabled': False, 'suspend_end_time': 0, 'continuous_errors': 0, - 'time_range_support': False} + 'time_range_support': False, + 'offline': False} def load_engine(engine_data): @@ -128,14 +129,16 @@ def load_engine(engine_data): engine.stats = { 'result_count': 0, 'search_count': 0, - 'page_load_time': 0, - 'page_load_count': 0, 'engine_time': 0, 'engine_time_count': 0, 'score_count': 0, 'errors': 0 } + if not engine.offline: + engine.stats['page_load_time'] = 0 + engine.stats['page_load_count'] = 0 + for category_name in engine.categories: categories.setdefault(category_name, []).append(engine) @@ -173,11 +176,6 @@ def get_engines_stats(): results_num = \ engine.stats['result_count'] / float(engine.stats['search_count']) - if engine.stats['page_load_count'] != 0: - load_times = engine.stats['page_load_time'] / float(engine.stats['page_load_count']) # noqa - else: - load_times = 0 - if engine.stats['engine_time_count'] != 0: this_engine_time = engine.stats['engine_time'] / float(engine.stats['engine_time_count']) # noqa else: @@ -189,14 +187,19 @@ def get_engines_stats(): else: score = score_per_result = 0.0 - max_pageload = max(load_times, max_pageload) + if not engine.offline: + load_times = 0 + if engine.stats['page_load_count'] != 0: + load_times = engine.stats['page_load_time'] / float(engine.stats['page_load_count']) # noqa + max_pageload = max(load_times, max_pageload) + pageloads.append({'avg': load_times, 'name': engine.name}) + max_engine_times = max(this_engine_time, max_engine_times) max_results = max(results_num, max_results) max_score = max(score, max_score) max_score_per_result = max(score_per_result, max_score_per_result) max_errors = max(max_errors, engine.stats['errors']) - pageloads.append({'avg': load_times, 'name': engine.name}) engine_times.append({'avg': this_engine_time, 'name': engine.name}) results.append({'avg': results_num, 'name': engine.name}) scores.append({'avg': score, 'name': engine.name}) @@ -255,7 +258,7 @@ def initialize_engines(engine_list): load_engines(engine_list) def engine_init(engine_name, init_fn): - init_fn() + init_fn(get_engine_from_settings(engine_name)) logger.debug('%s engine: Initialized', engine_name) for engine_name, engine in engines.items(): diff --git a/searx/engines/arxiv.py b/searx/engines/arxiv.py index 182861892..e3c871d17 100644 --- a/searx/engines/arxiv.py +++ b/searx/engines/arxiv.py @@ -17,6 +17,7 @@ from searx.url_utils import urlencode categories = ['science'] +paging = True base_url = 'http://export.arxiv.org/api/query?search_query=all:'\ + '{query}&start={offset}&max_results={number_of_results}' diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 1e614867b..ed0b87dbd 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -18,7 +18,7 @@ from lxml import html from searx import logger, utils from searx.engines.xpath import extract_text from searx.url_utils import urlencode -from searx.utils import match_language, gen_useragent +from searx.utils import match_language, gen_useragent, eval_xpath logger = logger.getChild('bing engine') @@ -65,11 +65,11 @@ def response(resp): dom = html.fromstring(resp.text) # parse results - for result in dom.xpath('//div[@class="sa_cc"]'): - link = result.xpath('.//h3/a')[0] + for result in eval_xpath(dom, '//div[@class="sa_cc"]'): + link = eval_xpath(result, './/h3/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = extract_text(result.xpath('.//p')) + content = extract_text(eval_xpath(result, './/p')) # append result results.append({'url': url, @@ -77,11 +77,11 @@ def response(resp): 'content': content}) # parse results again if nothing is found yet - for result in dom.xpath('//li[@class="b_algo"]'): - link = result.xpath('.//h2/a')[0] + for result in eval_xpath(dom, '//li[@class="b_algo"]'): + link = eval_xpath(result, './/h2/a')[0] url = link.attrib.get('href') title = extract_text(link) - content = extract_text(result.xpath('.//p')) + content = extract_text(eval_xpath(result, './/p')) # append result results.append({'url': url, @@ -89,7 +89,7 @@ def response(resp): 'content': content}) try: - result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()')) + result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()')) result_len_container = utils.to_string(result_len_container) if "-" in result_len_container: # Remove the part "from-to" for paginated request ... @@ -113,9 +113,9 @@ def response(resp): def _fetch_supported_languages(resp): supported_languages = [] dom = html.fromstring(resp.text) - options = dom.xpath('//div[@id="limit-languages"]//input') + options = eval_xpath(dom, '//div[@id="limit-languages"]//input') for option in options: - code = option.xpath('./@id')[0].replace('_', '-') + code = eval_xpath(option, './@id')[0].replace('_', '-') if code == 'nb': code = 'no' supported_languages.append(code) diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py index bb85c6dc5..a0e27e622 100644 --- a/searx/engines/deviantart.py +++ b/searx/engines/deviantart.py @@ -24,7 +24,7 @@ time_range_support = True # search-url base_url = 'https://www.deviantart.com/' -search_url = base_url + 'browse/all/?offset={offset}&{query}' +search_url = base_url + 'search?page={page}&{query}' time_range_url = '&order={range}' time_range_dict = {'day': 11, @@ -37,9 +37,7 @@ def request(query, params): if params['time_range'] and params['time_range'] not in time_range_dict: return params - offset = (params['pageno'] - 1) * 24 - - params['url'] = search_url.format(offset=offset, + params['url'] = search_url.format(page=params['pageno'], query=urlencode({'q': query})) if params['time_range'] in time_range_dict: params['url'] += time_range_url.format(range=time_range_dict[params['time_range']]) @@ -57,28 +55,27 @@ def response(resp): dom = html.fromstring(resp.text) - regex = re.compile(r'\/200H\/') - # parse results - for result in dom.xpath('.//span[@class="thumb wide"]'): - link = result.xpath('.//a[@class="torpedo-thumb-link"]')[0] - url = link.attrib.get('href') - title = extract_text(result.xpath('.//span[@class="title"]')) - thumbnail_src = link.xpath('.//img')[0].attrib.get('src') - img_src = regex.sub('/', thumbnail_src) - - # http to https, remove domain sharding - thumbnail_src = re.sub(r"https?://(th|fc)\d+.", "https://th01.", thumbnail_src) - thumbnail_src = re.sub(r"http://", "https://", thumbnail_src) - - url = re.sub(r"http://(.*)\.deviantart\.com/", "https://\\1.deviantart.com/", url) - - # append result - results.append({'url': url, - 'title': title, - 'img_src': img_src, - 'thumbnail_src': thumbnail_src, - 'template': 'images.html'}) + for row in dom.xpath('//div[contains(@data-hook, "content_row")]'): + for result in row.xpath('./div'): + link = result.xpath('.//a[@data-hook="deviation_link"]')[0] + url = link.attrib.get('href') + title = link.attrib.get('title') + thumbnail_src = result.xpath('.//img')[0].attrib.get('src') + img_src = thumbnail_src + + # http to https, remove domain sharding + thumbnail_src = re.sub(r"https?://(th|fc)\d+.", "https://th01.", thumbnail_src) + thumbnail_src = re.sub(r"http://", "https://", thumbnail_src) + + url = re.sub(r"http://(.*)\.deviantart\.com/", "https://\\1.deviantart.com/", url) + + # append result + results.append({'url': url, + 'title': title, + 'img_src': img_src, + 'thumbnail_src': thumbnail_src, + 'template': 'images.html'}) # return results return results diff --git a/searx/engines/dictzone.py b/searx/engines/dictzone.py index 09db048cc..423af0971 100644 --- a/searx/engines/dictzone.py +++ b/searx/engines/dictzone.py @@ -11,7 +11,7 @@ import re from lxml import html -from searx.utils import is_valid_lang +from searx.utils import is_valid_lang, eval_xpath from searx.url_utils import urljoin categories = ['general'] @@ -47,14 +47,14 @@ def response(resp): dom = html.fromstring(resp.text) - for k, result in enumerate(dom.xpath(results_xpath)[1:]): + for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]): try: - from_result, to_results_raw = result.xpath('./td') + from_result, to_results_raw = eval_xpath(result, './td') except: continue to_results = [] - for to_result in to_results_raw.xpath('./p/a'): + for to_result in eval_xpath(to_results_raw, './p/a'): t = to_result.text_content() if t.strip(): to_results.append(to_result.text_content()) diff --git a/searx/engines/digg.py b/searx/engines/digg.py index 4369ccb84..073410eb0 100644 --- a/searx/engines/digg.py +++ b/searx/engines/digg.py @@ -15,7 +15,8 @@ import string from dateutil import parser from json import loads from lxml import html -from searx.url_utils import quote_plus +from searx.url_utils import urlencode +from datetime import datetime # engine dependent config categories = ['news', 'social media'] @@ -23,7 +24,7 @@ paging = True # search-url base_url = 'https://digg.com/' -search_url = base_url + 'api/search/{query}.json?position={position}&format=html' +search_url = base_url + 'api/search/?{query}&from={position}&size=20&format=html' # specific xpath variables results_xpath = '//article' @@ -38,9 +39,9 @@ digg_cookie_chars = string.ascii_uppercase + string.ascii_lowercase +\ # do search-request def request(query, params): - offset = (params['pageno'] - 1) * 10 + offset = (params['pageno'] - 1) * 20 params['url'] = search_url.format(position=offset, - query=quote_plus(query)) + query=urlencode({'q': query})) params['cookies']['frontend.auid'] = ''.join(random.choice( digg_cookie_chars) for _ in range(22)) return params @@ -52,30 +53,17 @@ def response(resp): search_result = loads(resp.text) - if 'html' not in search_result or search_result['html'] == '': - return results - - dom = html.fromstring(search_result['html']) - # parse results - for result in dom.xpath(results_xpath): - url = result.attrib.get('data-contenturl') - thumbnail = result.xpath('.//img')[0].attrib.get('src') - title = ''.join(result.xpath(title_xpath)) - content = ''.join(result.xpath(content_xpath)) - pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime') - publishedDate = parser.parse(pubdate) - - # http to https - thumbnail = thumbnail.replace("http://static.digg.com", "https://static.digg.com") + for result in search_result['mapped']: + published = datetime.strptime(result['created']['ISO'], "%Y-%m-%d %H:%M:%S") # append result - results.append({'url': url, - 'title': title, - 'content': content, + results.append({'url': result['url'], + 'title': result['title'], + 'content': result['excerpt'], 'template': 'videos.html', - 'publishedDate': publishedDate, - 'thumbnail': thumbnail}) + 'publishedDate': published, + 'thumbnail': result['images']['thumbImage']}) # return results return results diff --git a/searx/engines/doku.py b/searx/engines/doku.py index a391be444..d20e66026 100644 --- a/searx/engines/doku.py +++ b/searx/engines/doku.py @@ -11,6 +11,7 @@ from lxml.html import fromstring from searx.engines.xpath import extract_text +from searx.utils import eval_xpath from searx.url_utils import urlencode # engine dependent config @@ -45,16 +46,16 @@ def response(resp): # parse results # Quickhits - for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'): + for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'): try: - res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1] + res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1] except: continue if not res_url: continue - title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title')) + title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title')) # append result results.append({'title': title, @@ -62,13 +63,13 @@ def response(resp): 'url': base_url + res_url}) # Search results - for r in doc.xpath('//dl[@class="search_results"]/*'): + for r in eval_xpath(doc, '//dl[@class="search_results"]/*'): try: if r.tag == "dt": - res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1] - title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title')) + res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1] + title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title')) elif r.tag == "dd": - content = extract_text(r.xpath('.')) + content = extract_text(eval_xpath(r, '.')) # append result results.append({'title': title, diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index fb8f523ac..0d2c0af2d 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -18,7 +18,7 @@ from json import loads from searx.engines.xpath import extract_text from searx.poolrequests import get from searx.url_utils import urlencode -from searx.utils import match_language +from searx.utils import match_language, eval_xpath # engine dependent config categories = ['general'] @@ -65,21 +65,36 @@ def get_region_code(lang, lang_list=[]): def request(query, params): - if params['time_range'] and params['time_range'] not in time_range_dict: + if params['time_range'] not in (None, 'None', '') and params['time_range'] not in time_range_dict: return params offset = (params['pageno'] - 1) * 30 region_code = get_region_code(params['language'], supported_languages) - if region_code: - params['url'] = url.format( - query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset) + params['url'] = 'https://duckduckgo.com/html/' + if params['pageno'] > 1: + params['method'] = 'POST' + params['data']['q'] = query + params['data']['s'] = offset + params['data']['dc'] = 30 + params['data']['nextParams'] = '' + params['data']['v'] = 'l' + params['data']['o'] = 'json' + params['data']['api'] = '/d.js' + if params['time_range'] in time_range_dict: + params['data']['df'] = time_range_dict[params['time_range']] + if region_code: + params['data']['kl'] = region_code else: - params['url'] = url.format( - query=urlencode({'q': query}), offset=offset, dc_param=offset) + if region_code: + params['url'] = url.format( + query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset) + else: + params['url'] = url.format( + query=urlencode({'q': query}), offset=offset, dc_param=offset) - if params['time_range'] in time_range_dict: - params['url'] += time_range_url.format(range=time_range_dict[params['time_range']]) + if params['time_range'] in time_range_dict: + params['url'] += time_range_url.format(range=time_range_dict[params['time_range']]) return params @@ -91,17 +106,19 @@ def response(resp): doc = fromstring(resp.text) # parse results - for r in doc.xpath(result_xpath): + for i, r in enumerate(eval_xpath(doc, result_xpath)): + if i >= 30: + break try: - res_url = r.xpath(url_xpath)[-1] + res_url = eval_xpath(r, url_xpath)[-1] except: continue if not res_url: continue - title = extract_text(r.xpath(title_xpath)) - content = extract_text(r.xpath(content_xpath)) + title = extract_text(eval_xpath(r, title_xpath)) + content = extract_text(eval_xpath(r, content_xpath)) # append result results.append({'title': title, diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py index 957a13ea6..79d10c303 100644 --- a/searx/engines/duckduckgo_definitions.py +++ b/searx/engines/duckduckgo_definitions.py @@ -1,3 +1,14 @@ +""" +DuckDuckGo (definitions) + +- `Instant Answer API`_ +- `DuckDuckGo query`_ + +.. _Instant Answer API: https://duckduckgo.com/api +.. _DuckDuckGo query: https://api.duckduckgo.com/?q=DuckDuckGo&format=json&pretty=1 + +""" + import json from lxml import html from re import compile @@ -25,7 +36,8 @@ def result_to_text(url, text, htmlResult): def request(query, params): params['url'] = url.format(query=urlencode({'q': query})) language = match_language(params['language'], supported_languages, language_aliases) - params['headers']['Accept-Language'] = language.split('-')[0] + language = language.split('-')[0] + params['headers']['Accept-Language'] = language return params @@ -43,8 +55,9 @@ def response(resp): # add answer if there is one answer = search_res.get('Answer', '') - if answer != '': - results.append({'answer': html_to_text(answer)}) + if answer: + if search_res.get('AnswerType', '') not in ['calc']: + results.append({'answer': html_to_text(answer)}) # add infobox if 'Definition' in search_res: diff --git a/searx/engines/duden.py b/searx/engines/duden.py index 444f18c1f..cf2f1a278 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -11,6 +11,7 @@ from lxml import html, etree import re from searx.engines.xpath import extract_text +from searx.utils import eval_xpath from searx.url_utils import quote, urljoin from searx import logger @@ -52,9 +53,9 @@ def response(resp): dom = html.fromstring(resp.text) try: - number_of_results_string = re.sub('[^0-9]', '', dom.xpath( - '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0] - ) + number_of_results_string =\ + re.sub('[^0-9]', '', + eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]) results.append({'number_of_results': int(number_of_results_string)}) @@ -62,12 +63,12 @@ def response(resp): logger.debug("Couldn't read number of results.") pass - for result in dom.xpath('//section[not(contains(@class, "essay"))]'): + for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'): try: - url = result.xpath('.//h2/a')[0].get('href') + url = eval_xpath(result, './/h2/a')[0].get('href') url = urljoin(base_url, url) - title = result.xpath('string(.//h2/a)').strip() - content = extract_text(result.xpath('.//p')) + title = eval_xpath(result, 'string(.//h2/a)').strip() + content = extract_text(eval_xpath(result, './/p')) # append result results.append({'url': url, 'title': title, diff --git a/searx/engines/framalibre.py b/searx/engines/framalibre.py index 146cdaeec..f3441fa5f 100644 --- a/searx/engines/framalibre.py +++ b/searx/engines/framalibre.py @@ -10,7 +10,10 @@ @parse url, title, content, thumbnail, img_src """ -from cgi import escape +try: + from cgi import escape +except: + from html import escape from lxml import html from searx.engines.xpath import extract_text from searx.url_utils import urljoin, urlencode diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index a6aa5d718..a84f3f69d 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -15,6 +15,7 @@ from json import loads from time import time from lxml.html import fromstring from searx.url_utils import urlencode +from searx.utils import eval_xpath # engine dependent config categories = ['general'] @@ -35,8 +36,8 @@ search_string = 'search?{query}'\ '&ff={safesearch}'\ '&rxiec={rxieu}'\ '&ulse={ulse}'\ - '&rand={rxikd}' # current unix timestamp - + '&rand={rxikd}'\ + '&dbez={dbez}' # specific xpath variables results_xpath = '//response//result' url_xpath = './/url' @@ -70,7 +71,8 @@ def request(query, params): rxieu=random.randint(1000000000, 9999999999), ulse=random.randint(100000000, 999999999), lang=language, - safesearch=safesearch) + safesearch=safesearch, + dbez=random.randint(100000000, 999999999)) params['url'] = base_url + search_path @@ -98,9 +100,9 @@ def response(resp): def _fetch_supported_languages(resp): supported_languages = [] dom = fromstring(resp.text) - links = dom.xpath('//span[@id="menu2"]/a') + links = eval_xpath(dom, '//span[@id="menu2"]/a') for link in links: - href = link.xpath('./@href')[0].split('lang%3A') + href = eval_xpath(link, './@href')[0].split('lang%3A') if len(href) == 2: code = href[1].split('_') if len(code) == 2: diff --git a/searx/engines/google.py b/searx/engines/google.py index 03f0523e7..19bde710d 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -14,7 +14,7 @@ from lxml import html, etree from searx.engines.xpath import extract_text, extract_url from searx import logger from searx.url_utils import urlencode, urlparse, parse_qsl -from searx.utils import match_language +from searx.utils import match_language, eval_xpath logger = logger.getChild('google engine') @@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname): # returns extract_text on the first result selected by the xpath or None def extract_text_from_dom(result, xpath): - r = result.xpath(xpath) + r = eval_xpath(result, xpath) if len(r) > 0: return extract_text(r[0]) return None @@ -199,8 +199,9 @@ def request(query, params): params['headers']['Accept-Language'] = language + ',' + language + '-' + country params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - # Force Internet Explorer 12 user agent to avoid loading the new UI that Searx can't parse - params['headers']['User-Agent'] = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)" + # Force Safari 3.1 on Mac OS X (Leopard) user agent to avoid loading the new UI that Searx can't parse + params['headers']['User-Agent'] = ("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4)" + "AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1") params['google_hostname'] = google_hostname @@ -226,21 +227,21 @@ def response(resp): # convert the text to dom dom = html.fromstring(resp.text) - instant_answer = dom.xpath('//div[@id="_vBb"]//text()') + instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()') if instant_answer: results.append({'answer': u' '.join(instant_answer)}) try: - results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0] + results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0] .split()[1].replace(',', '')) results.append({'number_of_results': results_num}) except: pass # parse results - for result in dom.xpath(results_xpath): + for result in eval_xpath(dom, results_xpath): try: - title = extract_text(result.xpath(title_xpath)[0]) - url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname) + title = extract_text(eval_xpath(result, title_xpath)[0]) + url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname) parsed_url = urlparse(url, google_hostname) # map result @@ -249,7 +250,7 @@ def response(resp): continue # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start): # print "yooooo"*30 - # x = result.xpath(map_near) + # x = eval_xpath(result, map_near) # if len(x) > 0: # # map : near the location # results = results + parse_map_near(parsed_url, x, google_hostname) @@ -286,11 +287,11 @@ def response(resp): continue # parse suggestion - for suggestion in dom.xpath(suggestion_xpath): + for suggestion in eval_xpath(dom, suggestion_xpath): # append suggestion results.append({'suggestion': extract_text(suggestion)}) - for correction in dom.xpath(spelling_suggestion_xpath): + for correction in eval_xpath(dom, spelling_suggestion_xpath): results.append({'correction': extract_text(correction)}) # return results @@ -299,9 +300,9 @@ def response(resp): def parse_images(result, google_hostname): results = [] - for image in result.xpath(images_xpath): - url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname) - img_src = extract_text(image.xpath(image_img_src_xpath)[0]) + for image in eval_xpath(result, images_xpath): + url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname) + img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0]) # append result results.append({'url': url, @@ -388,10 +389,10 @@ def attributes_to_html(attributes): def _fetch_supported_languages(resp): supported_languages = {} dom = html.fromstring(resp.text) - options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]') + options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]') for option in options: - code = option.xpath('./@value')[0].split('_')[-1] - name = option.xpath('./@data-name')[0].title() + code = eval_xpath(option, './@value')[0].split('_')[-1] + name = eval_xpath(option, './@data-name')[0].title() supported_languages[code] = {"name": name} return supported_languages diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index d9a49e9cc..636913114 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -70,11 +70,21 @@ def response(resp): try: metadata = loads(result) - img_format = "{0} {1}x{2}".format(metadata['ity'], str(metadata['ow']), str(metadata['oh'])) - source = "{0} ({1})".format(metadata['st'], metadata['isu']) + + img_format = metadata.get('ity', '') + img_width = metadata.get('ow', '') + img_height = metadata.get('oh', '') + if img_width and img_height: + img_format += " {0}x{1}".format(img_width, img_height) + + source = metadata.get('st', '') + source_url = metadata.get('isu', '') + if source_url: + source += " ({0})".format(source_url) + results.append({'url': metadata['ru'], 'title': metadata['pt'], - 'content': metadata['s'], + 'content': metadata.get('s', ''), 'source': source, 'img_format': img_format, 'thumbnail_src': metadata['tu'], diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py new file mode 100644 index 000000000..f9b1f99c8 --- /dev/null +++ b/searx/engines/seedpeer.py @@ -0,0 +1,78 @@ +# Seedpeer (Videos, Music, Files) +# +# @website https://seedpeer.me +# @provide-api no (nothing found) +# +# @using-api no +# @results HTML (using search portal) +# @stable yes (HTML can change) +# @parse url, title, content, seed, leech, magnetlink + +from lxml import html +from json import loads +from operator import itemgetter +from searx.url_utils import quote, urljoin +from searx.engines.xpath import extract_text + + +url = 'https://seedpeer.me/' +search_url = url + 'search/{search_term}?page={page_no}' +torrent_file_url = url + 'torrent/{torrent_hash}' + +# specific xpath variables +script_xpath = '//script[@type="text/javascript"][not(@src)]' +torrent_xpath = '(//table)[2]/tbody/tr' +link_xpath = '(./td)[1]/a/@href' +age_xpath = '(./td)[2]' +size_xpath = '(./td)[3]' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(search_term=quote(query), + page_no=params['pageno']) + return params + + +# get response from search-request +def response(resp): + results = [] + dom = html.fromstring(resp.text) + result_rows = dom.xpath(torrent_xpath) + + try: + script_element = dom.xpath(script_xpath)[0] + json_string = script_element.text[script_element.text.find('{'):] + torrents_json = loads(json_string) + except: + return [] + + # parse results + for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']): + title = torrent_json['name'] + seed = int(torrent_json['seeds']) + leech = int(torrent_json['peers']) + size = int(torrent_json['size']) + torrent_hash = torrent_json['hash'] + + torrentfile = torrent_file_url.format(torrent_hash=torrent_hash) + magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash) + + age = extract_text(torrent_row.xpath(age_xpath)) + link = torrent_row.xpath(link_xpath)[0] + + href = urljoin(url, link) + + # append result + results.append({'url': href, + 'title': title, + 'content': age, + 'seed': seed, + 'leech': leech, + 'filesize': size, + 'torrentfile': torrentfile, + 'magnetlink': magnetlink, + 'template': 'torrent.html'}) + + # return results sorted by seeder + return sorted(results, key=itemgetter('seed'), reverse=True) diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py index 3ba9a7f39..284689bf6 100644 --- a/searx/engines/soundcloud.py +++ b/searx/engines/soundcloud.py @@ -51,7 +51,9 @@ def get_client_id(): if response.ok: tree = html.fromstring(response.content) - script_tags = tree.xpath("//script[contains(@src, '/assets/app')]") + # script_tags has been moved from /assets/app/ to /assets/ path. I + # found client_id in https://a-v2.sndcdn.com/assets/49-a0c01933-3.js + script_tags = tree.xpath("//script[contains(@src, '/assets/')]") app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None] # extracts valid app_js urls from soundcloud.com content @@ -66,7 +68,7 @@ def get_client_id(): return "" -def init(): +def init(engine_settings=None): global guest_client_id # api-key guest_client_id = get_client_id() diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 6638f3d83..76567396f 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -15,6 +15,8 @@ from dateutil import parser from datetime import datetime, timedelta import re from searx.engines.xpath import extract_text +from searx.languages import language_codes +from searx.utils import eval_xpath # engine dependent config categories = ['general'] @@ -22,7 +24,7 @@ categories = ['general'] # (probably the parameter qid), require # storing of qid's between mulitble search-calls -# paging = False +paging = True language_support = True # search-url @@ -32,23 +34,32 @@ search_url = base_url + 'do/search' # specific xpath variables # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"] # not ads: div[@class="result"] are the direct childs of div[@id="results"] -results_xpath = '//li[contains(@class, "search-result") and contains(@class, "search-item")]' -link_xpath = './/h3/a' -content_xpath = './p[@class="search-item__body"]' +results_xpath = '//div[@class="w-gl__result"]' +link_xpath = './/a[@class="w-gl__result-title"]' +content_xpath = './/p[@class="w-gl__description"]' # do search-request def request(query, params): - offset = (params['pageno'] - 1) * 10 params['url'] = search_url params['method'] = 'POST' - params['data'] = {'query': query, - 'startat': offset} + params['data'] = { + 'query': query, + 'page': params['pageno'], + 'cat': 'web', + 'cmd': 'process_search', + 'engine0': 'v1all', + } # set language if specified if params['language'] != 'all': - params['data']['with_language'] = ('lang_' + params['language'].split('-')[0]) + language = 'english' + for lc, _, _, lang in language_codes: + if lc == params['language']: + language = lang + params['data']['language'] = language + params['data']['lui'] = language return params @@ -60,8 +71,8 @@ def response(resp): dom = html.fromstring(resp.text) # parse results - for result in dom.xpath(results_xpath): - links = result.xpath(link_xpath) + for result in eval_xpath(dom, results_xpath): + links = eval_xpath(result, link_xpath) if not links: continue link = links[0] @@ -77,8 +88,8 @@ def response(resp): title = extract_text(link) - if result.xpath(content_xpath): - content = extract_text(result.xpath(content_xpath)) + if eval_xpath(result, content_xpath): + content = extract_text(eval_xpath(result, content_xpath)) else: content = '' diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py index 5ea2b9958..e913b3915 100644 --- a/searx/engines/wikidata.py +++ b/searx/engines/wikidata.py @@ -16,7 +16,7 @@ from searx.poolrequests import get from searx.engines.xpath import extract_text from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url from searx.url_utils import urlencode -from searx.utils import match_language +from searx.utils import match_language, eval_xpath from json import loads from lxml.html import fromstring @@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]' media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a' -# xpath_cache -xpath_cache = {} - - -def get_xpath(xpath_str): - result = xpath_cache.get(xpath_str, None) - if not result: - result = etree.XPath(xpath_str) - xpath_cache[xpath_str] = result - return result - - -def eval_xpath(element, xpath_str): - xpath = get_xpath(xpath_str) - return xpath(element) - def get_id_cache(result): id_cache = {} diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py index 2cbbc5adc..387c9fa17 100644 --- a/searx/engines/wolframalpha_noapi.py +++ b/searx/engines/wolframalpha_noapi.py @@ -55,7 +55,7 @@ def obtain_token(): return token -def init(): +def init(engine_settings=None): obtain_token() diff --git a/searx/engines/www1x.py b/searx/engines/www1x.py index 508803240..f1154b16d 100644 --- a/searx/engines/www1x.py +++ b/searx/engines/www1x.py @@ -11,8 +11,8 @@ """ from lxml import html -import re from searx.url_utils import urlencode, urljoin +from searx.engines.xpath import extract_text # engine dependent config categories = ['images'] @@ -34,41 +34,18 @@ def request(query, params): def response(resp): results = [] - # get links from result-text - regex = re.compile('(</a>|<a)') - results_parts = re.split(regex, resp.text) - - cur_element = '' - - # iterate over link parts - for result_part in results_parts: + dom = html.fromstring(resp.text) + for res in dom.xpath('//div[@class="List-item MainListing"]'): # processed start and end of link - if result_part == '<a': - cur_element = result_part - continue - elif result_part != '</a>': - cur_element += result_part - continue - - cur_element += result_part - - # fix xml-error - cur_element = cur_element.replace('"></a>', '"/></a>') - - dom = html.fromstring(cur_element) - link = dom.xpath('//a')[0] + link = res.xpath('//a')[0] url = urljoin(base_url, link.attrib.get('href')) - title = link.attrib.get('title', '') + title = extract_text(link) - thumbnail_src = urljoin(base_url, link.xpath('.//img')[0].attrib['src']) + thumbnail_src = urljoin(base_url, res.xpath('.//img')[0].attrib['src']) # TODO: get image with higher resolution img_src = thumbnail_src - # check if url is showing to a photo - if '/photo/' not in url: - continue - # append result results.append({'url': url, 'title': title, diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index 61494ce4e..b75896cc7 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -1,6 +1,6 @@ from lxml import html from lxml.etree import _ElementStringResult, _ElementUnicodeResult -from searx.utils import html_to_text +from searx.utils import html_to_text, eval_xpath from searx.url_utils import unquote, urlencode, urljoin, urlparse search_url = None @@ -104,15 +104,15 @@ def response(resp): results = [] dom = html.fromstring(resp.text) if results_xpath: - for result in dom.xpath(results_xpath): - url = extract_url(result.xpath(url_xpath), search_url) - title = extract_text(result.xpath(title_xpath)) - content = extract_text(result.xpath(content_xpath)) + for result in eval_xpath(dom, results_xpath): + url = extract_url(eval_xpath(result, url_xpath), search_url) + title = extract_text(eval_xpath(result, title_xpath)) + content = extract_text(eval_xpath(result, content_xpath)) tmp_result = {'url': url, 'title': title, 'content': content} # add thumbnail if available if thumbnail_xpath: - thumbnail_xpath_result = result.xpath(thumbnail_xpath) + thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath) if len(thumbnail_xpath_result) > 0: tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url) @@ -120,14 +120,14 @@ def response(resp): else: for url, title, content in zip( (extract_url(x, search_url) for - x in dom.xpath(url_xpath)), - map(extract_text, dom.xpath(title_xpath)), - map(extract_text, dom.xpath(content_xpath)) + x in eval_xpath(dom, url_xpath)), + map(extract_text, eval_xpath(dom, title_xpath)), + map(extract_text, eval_xpath(dom, content_xpath)) ): results.append({'url': url, 'title': title, 'content': content}) if not suggestion_xpath: return results - for suggestion in dom.xpath(suggestion_xpath): + for suggestion in eval_xpath(dom, suggestion_xpath): results.append({'suggestion': extract_text(suggestion)}) return results diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 73b78bcf7..36c1a11f8 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -14,7 +14,7 @@ from lxml import html from searx.engines.xpath import extract_text, extract_url from searx.url_utils import unquote, urlencode -from searx.utils import match_language +from searx.utils import match_language, eval_xpath # engine dependent config categories = ['general'] @@ -109,21 +109,21 @@ def response(resp): dom = html.fromstring(resp.text) try: - results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0] + results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0] .split()[0].replace(',', '')) results.append({'number_of_results': results_num}) except: pass # parse results - for result in dom.xpath(results_xpath): + for result in eval_xpath(dom, results_xpath): try: - url = parse_url(extract_url(result.xpath(url_xpath), search_url)) - title = extract_text(result.xpath(title_xpath)[0]) + url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url)) + title = extract_text(eval_xpath(result, title_xpath)[0]) except: continue - content = extract_text(result.xpath(content_xpath)[0]) + content = extract_text(eval_xpath(result, content_xpath)[0]) # append result results.append({'url': url, @@ -131,7 +131,7 @@ def response(resp): 'content': content}) # if no suggestion found, return results - suggestions = dom.xpath(suggestion_xpath) + suggestions = eval_xpath(dom, suggestion_xpath) if not suggestions: return results @@ -148,9 +148,9 @@ def response(resp): def _fetch_supported_languages(resp): supported_languages = [] dom = html.fromstring(resp.text) - options = dom.xpath('//div[@id="yschlang"]/span/label/input') + options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input') for option in options: - code_parts = option.xpath('./@value')[0][5:].split('_') + code_parts = eval_xpath(option, './@value')[0][5:].split('_') if len(code_parts) == 2: code = code_parts[0] + '-' + code_parts[1].upper() else: diff --git a/searx/exceptions.py b/searx/exceptions.py index c605ddcab..0175acfa3 100644 --- a/searx/exceptions.py +++ b/searx/exceptions.py @@ -28,5 +28,6 @@ class SearxParameterException(SearxException): else: message = 'Invalid value "' + value + '" for parameter ' + name super(SearxParameterException, self).__init__(message) + self.message = message self.parameter_name = name self.parameter_value = value diff --git a/searx/plugins/https_rewrite.py b/searx/plugins/https_rewrite.py index 3d986770e..82556017e 100644 --- a/searx/plugins/https_rewrite.py +++ b/searx/plugins/https_rewrite.py @@ -225,6 +225,9 @@ def https_url_rewrite(result): def on_result(request, search, result): + if 'parsed_url' not in result: + return True + if result['parsed_url'].scheme == 'http': https_url_rewrite(result) return True diff --git a/searx/plugins/oa_doi_rewrite.py b/searx/plugins/oa_doi_rewrite.py index d4942498f..be80beb26 100644 --- a/searx/plugins/oa_doi_rewrite.py +++ b/searx/plugins/oa_doi_rewrite.py @@ -35,6 +35,9 @@ def get_doi_resolver(args, preference_doi_resolver): def on_result(request, search, result): + if 'parsed_url' not in result: + return True + doi = extract_doi(result['parsed_url']) if doi and len(doi) < 50: for suffix in ('/', '.pdf', '/full', '/meta', '/abstract'): diff --git a/searx/plugins/tracker_url_remover.py b/searx/plugins/tracker_url_remover.py index 630c8a638..33dd621e1 100644 --- a/searx/plugins/tracker_url_remover.py +++ b/searx/plugins/tracker_url_remover.py @@ -17,10 +17,10 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from flask_babel import gettext import re -from searx.url_utils import urlunparse +from searx.url_utils import urlunparse, parse_qsl, urlencode -regexes = {re.compile(r'utm_[^&]+&?'), - re.compile(r'(wkey|wemail)[^&]+&?'), +regexes = {re.compile(r'utm_[^&]+'), + re.compile(r'(wkey|wemail)[^&]*'), re.compile(r'&$')} name = gettext('Tracker URL remover') @@ -30,16 +30,23 @@ preference_section = 'privacy' def on_result(request, search, result): + if 'parsed_url' not in result: + return True + query = result['parsed_url'].query if query == "": return True - - for reg in regexes: - query = reg.sub('', query) - - if query != result['parsed_url'].query: - result['parsed_url'] = result['parsed_url']._replace(query=query) - result['url'] = urlunparse(result['parsed_url']) + parsed_query = parse_qsl(query) + + changes = 0 + for i, (param_name, _) in enumerate(list(parsed_query)): + for reg in regexes: + if reg.match(param_name): + parsed_query.pop(i - changes) + changes += 1 + result['parsed_url'] = result['parsed_url']._replace(query=urlencode(parsed_query)) + result['url'] = urlunparse(result['parsed_url']) + break return True diff --git a/searx/query.py b/searx/query.py index 382aed871..c4002bd31 100644 --- a/searx/query.py +++ b/searx/query.py @@ -184,7 +184,7 @@ class SearchQuery(object): self.lang = lang self.safesearch = safesearch self.pageno = pageno - self.time_range = time_range + self.time_range = None if time_range in ('', 'None', None) else time_range self.timeout_limit = timeout_limit def __str__(self): diff --git a/searx/results.py b/searx/results.py index be74a836b..3b1e4bd62 100644 --- a/searx/results.py +++ b/searx/results.py @@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2): for url2 in infobox2.get('urls', []): unique_url = True - for url1 in infobox1.get('urls', []): - if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))): + parsed_url2 = urlparse(url2.get('url', '')) + for url1 in urls1: + if compare_urls(urlparse(url1.get('url', '')), parsed_url2): unique_url = False break if unique_url: @@ -188,8 +189,9 @@ class ResultContainer(object): add_infobox = True infobox_id = infobox.get('id', None) if infobox_id is not None: + parsed_url_infobox_id = urlparse(infobox_id) for existingIndex in self.infoboxes: - if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)): + if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id): merge_two_infoboxes(existingIndex, infobox) add_infobox = False @@ -197,6 +199,13 @@ class ResultContainer(object): self.infoboxes.append(infobox) def _merge_result(self, result, position): + if 'url' in result: + self.__merge_url_result(result, position) + return + + self.__merge_result_no_url(result, position) + + def __merge_url_result(self, result, position): result['parsed_url'] = urlparse(result['url']) # if the result has no scheme, use http as default @@ -210,51 +219,60 @@ class ResultContainer(object): if result.get('content'): result['content'] = WHITESPACE_REGEX.sub(' ', result['content']) - # check for duplicates - duplicated = False + duplicated = self.__find_duplicated_http_result(result) + if duplicated: + self.__merge_duplicated_http_result(duplicated, result, position) + return + + # if there is no duplicate found, append result + result['positions'] = [position] + with RLock(): + self._merged_results.append(result) + + def __find_duplicated_http_result(self, result): result_template = result.get('template') for merged_result in self._merged_results: + if 'parsed_url' not in merged_result: + continue if compare_urls(result['parsed_url'], merged_result['parsed_url'])\ and result_template == merged_result.get('template'): if result_template != 'images.html': # not an image, same template, same url : it's a duplicate - duplicated = merged_result - break + return merged_result else: # it's an image # it's a duplicate if the parsed_url, template and img_src are differents if result.get('img_src', '') == merged_result.get('img_src', ''): - duplicated = merged_result - break + return merged_result + return None - # merge duplicates together - if duplicated: - # using content with more text - if result_content_len(result.get('content', '')) >\ - result_content_len(duplicated.get('content', '')): - duplicated['content'] = result['content'] + def __merge_duplicated_http_result(self, duplicated, result, position): + # using content with more text + if result_content_len(result.get('content', '')) >\ + result_content_len(duplicated.get('content', '')): + duplicated['content'] = result['content'] - # merge all result's parameters not found in duplicate - for key in result.keys(): - if not duplicated.get(key): - duplicated[key] = result.get(key) + # merge all result's parameters not found in duplicate + for key in result.keys(): + if not duplicated.get(key): + duplicated[key] = result.get(key) - # add the new position - duplicated['positions'].append(position) + # add the new position + duplicated['positions'].append(position) - # add engine to list of result-engines - duplicated['engines'].add(result['engine']) + # add engine to list of result-engines + duplicated['engines'].add(result['engine']) - # using https if possible - if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https': - duplicated['url'] = result['parsed_url'].geturl() - duplicated['parsed_url'] = result['parsed_url'] + # using https if possible + if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https': + duplicated['url'] = result['parsed_url'].geturl() + duplicated['parsed_url'] = result['parsed_url'] - # if there is no duplicate found, append result - else: - result['positions'] = [position] - with RLock(): - self._merged_results.append(result) + def __merge_result_no_url(self, result, position): + result['engines'] = set([result['engine']]) + result['positions'] = [position] + with RLock(): + self._merged_results.append(result) def order_results(self): for result in self._merged_results: diff --git a/searx/search.py b/searx/search.py index 9c7142c74..5c268cc5d 100644 --- a/searx/search.py +++ b/searx/search.py @@ -77,7 +77,7 @@ def send_http_request(engine, request_params): return req(request_params['url'], **request_args) -def search_one_request(engine, query, request_params): +def search_one_http_request(engine, query, request_params): # update request parameters dependent on # search-engine (contained in engines folder) engine.request(query, request_params) @@ -97,7 +97,53 @@ def search_one_request(engine, query, request_params): return engine.response(response) +def search_one_offline_request(engine, query, request_params): + return engine.search(query, request_params) + + def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit): + if engines[engine_name].offline: + return search_one_offline_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit) # noqa + return search_one_http_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit) + + +def search_one_offline_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit): + engine = engines[engine_name] + + try: + search_results = search_one_offline_request(engine, query, request_params) + + if search_results: + result_container.extend(engine_name, search_results) + + engine_time = time() - start_time + result_container.add_timing(engine_name, engine_time, engine_time) + with threading.RLock(): + engine.stats['engine_time'] += engine_time + engine.stats['engine_time_count'] += 1 + + except ValueError as e: + record_offline_engine_stats_on_error(engine, result_container, start_time) + logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e)) + except Exception as e: + record_offline_engine_stats_on_error(engine, result_container, start_time) + + result_container.add_unresponsive_engine(( + engine_name, + u'{0}: {1}'.format(gettext('unexpected crash'), e), + )) + logger.exception('engine {0} : exception : {1}'.format(engine_name, e)) + + +def record_offline_engine_stats_on_error(engine, result_container, start_time): + engine_time = time() - start_time + result_container.add_timing(engine.name, engine_time, engine_time) + + with threading.RLock(): + engine.stats['errors'] += 1 + + +def search_one_http_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit): # set timeout for all HTTP requests requests_lib.set_timeout_for_thread(timeout_limit, start_time=start_time) # reset the HTTP total time @@ -111,7 +157,7 @@ def search_one_request_safe(engine_name, query, request_params, result_container try: # send requests and parse the results - search_results = search_one_request(engine, query, request_params) + search_results = search_one_http_request(engine, query, request_params) # check if the engine accepted the request if search_results is not None: @@ -427,19 +473,21 @@ class Search(object): continue # set default request parameters - request_params = default_request_params() - request_params['headers']['User-Agent'] = user_agent - request_params['category'] = selected_engine['category'] - request_params['pageno'] = search_query.pageno + request_params = {} + if not engine.offline: + request_params = default_request_params() + request_params['headers']['User-Agent'] = user_agent - if hasattr(engine, 'language') and engine.language: - request_params['language'] = engine.language - else: - request_params['language'] = search_query.lang + if hasattr(engine, 'language') and engine.language: + request_params['language'] = engine.language + else: + request_params['language'] = search_query.lang - # 0 = None, 1 = Moderate, 2 = Strict - request_params['safesearch'] = search_query.safesearch - request_params['time_range'] = search_query.time_range + request_params['safesearch'] = search_query.safesearch + request_params['time_range'] = search_query.time_range + + request_params['category'] = selected_engine['category'] + request_params['pageno'] = search_query.pageno # append request to list requests.append((selected_engine['name'], search_query.query, request_params)) diff --git a/searx/settings.yml b/searx/settings.yml index cf2b13e08..c6f805331 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -161,11 +161,12 @@ engines: weight : 2 disabled : True - - name : digbt - engine : digbt - shortcut : dbt - timeout : 6.0 - disabled : True +# cloudflare protected +# - name : digbt +# engine : digbt +# shortcut : dbt +# timeout : 6.0 +# disabled : True - name : digg engine : digg @@ -703,9 +704,9 @@ engines: shortcut: vo categories: social media search_url : https://searchvoat.co/?t={query} - url_xpath : //div[@class="entry"]/p/a[contains(@class, "title")]/@href - title_xpath : //div[@class="entry"]/p/a[contains(@class, "title")] - content_xpath : //div[@class="entry"]/p/span[@class="domain"]/a/text() + url_xpath : //div[@class="entry"]//p[@class="title"]/a/@href + title_xpath : //div[@class="entry"]//p[@class="title"]/a/text() + content_xpath : //div[@class="entry"]//span[@class="domain"]/a/text() timeout : 10.0 disabled : True @@ -743,10 +744,15 @@ engines: title_xpath : ./h2 content_xpath : ./p[@class="s"] suggestion_xpath : /html/body//div[@class="top-info"]/p[@class="top-info spell"]/a - first_page_num : 1 + first_page_num : 0 page_size : 10 disabled : True + - name : seedpeer + shortcut : speu + engine : seedpeer + categories: files, music, videos + # - name : yacy # engine : yacy # shortcut : ya diff --git a/searx/static/themes/courgette/css/style.css b/searx/static/themes/courgette/css/style.css index ad5d233ff..508c4b605 100644 --- a/searx/static/themes/courgette/css/style.css +++ b/searx/static/themes/courgette/css/style.css @@ -1 +1 @@ -a,h2{color:#666}.center,html{position:relative}#categories_container>div,.top_margin a{display:inline-block}#categories,.center{text-align:center}#categories .hidden,.cache_link,.highlight .c,.highlight .cm,.highlight .ge,.highlight .sd{font-style:italic}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=search]{-webkit-appearance:textfield}h2{text-transform:uppercase}body{font-family:sans-serif;line-height:1.5;margin:0;background:#EEE}html{min-height:100%}.title h1{font-size:7em;color:#3498DB;margin:-20px auto 0;line-height:100px;padding-bottom:20px}.center{max-width:70em;background:rgba(255,255,255,.6);padding:2em;margin:7% auto 0}.center.search{position:static;width:auto;background:0 0;margin:auto;padding-top:1.8em}@media screen and (min-width:1001px){.center:after{content:"";z-index:-1;background:url(../img/bg-body-index.jpg) no-repeat;background-size:cover;width:100%;height:100%;top:0;left:0;position:fixed}.center.search:after{content:none}}.autocompleter-choices{position:absolute;margin:0;padding:0;background:#FFF}.autocompleter-choices li{padding:.5em 1em}.autocompleter-choices li:hover{background:#3498DB;color:#FFF;cursor:pointer}.top_margin{position:absolute;bottom:-3.5em;width:100%;left:0}.top_margin a{margin-right:1em;color:#FFF;text-decoration:none}.top_margin a:focus,.top_margin a:hover{text-decoration:underline}@media screen and (max-width:1000px){.center{background:0 0}.top_margin a{color:#333}}.checkbox_container{margin-top:1.5em}.checkbox_container label{padding:.5em 1em;color:#333;cursor:pointer;font-size:.9em}.checkbox_container input[type=checkbox]:checked+label,.checkbox_container label:hover{background:#3498DB;color:#FFF}.checkbox_container input[type=checkbox]{position:absolute;top:-9999px}#categories .hidden{display:none;position:absolute;bottom:1em;left:0;text-align:center;width:100%;font-size:.9em;color:#333}#categories:hover .hidden,.right a{display:block}@media screen and (max-width:900px){#categories_container{letter-spacing:-5px}#categories_container>div{letter-spacing:normal;margin-top:1em}.checkbox_container{margin:0}.checkbox_container label{display:block;background:#CCC;padding:1em;border:1px solid #FFF}.top_margin{position:static}#categories .hidden{position:static;display:block}}@media screen and (max-width:900px) and (min-width:501px){#categories_container>div{width:31%;margin-left:2.333%}#categories_container>div:nth-child(3n+1){margin-left:0}}@media screen and (max-width:500px){#categories_container>div{width:48%;margin-left:2%;font-size:.9em}#categories_container>div:nth-child(2n+1){margin-left:0}.title h1{background:url(../img/searx-mobile.png) no-repeat;width:200px;height:39px}}#search_wrapper{position:relative}.q{padding:.5em 3em .5em 1em;width:100%;font-size:1.5em;border:0;color:#666}.cache_link,.result p{font-size:.9em}#search_submit{position:absolute;top:0;right:0;border:0;background:url(../img/search-icon.png) center center/65% auto no-repeat #3498DB;text-indent:-9999px;width:5em;height:100%;cursor:pointer}#sidebar,.right{position:fixed;width:15em;right:0;text-align:right}#search_submit:focus,#search_submit:hover{background-color:#0665A2}#sidebar{background:#3498DB;top:0;height:100%;padding:1.5em}.right{bottom:1.5em;z-index:1;padding:0 1.5em}.right a{color:#FFF;text-decoration:none}#sidebar form,#suggestions form,.row fieldset{display:inline-block}.right a:focus,.right a:hover{text-decoration:underline}#preferences{background:url(../img/preference-icon.png) right center/12% auto no-repeat;padding-right:1.8em}#search_url input{border:0;padding:.5em}#sidebar>div{margin-bottom:1em;color:#FFF}#sidebar input[type=submit]{background:#CCC;border:0;padding:.5em 1em;cursor:pointer;margin-top:.5em}#sidebar input[type=submit]:focus,#sidebar input[type=submit]:hover{color:#FFF;background-color:#0665A2}#results{padding:0 17em 0 2em}.result .content{margin:0;color:#666}.result .url{margin-top:0;color:#FF6530}.result .favicon{float:left;position:relative;top:.5em;margin-right:.5em}.definition_result{background:#CCC;padding:1em}.definition_result .result_title,.definition_result p{margin:0}.result_title{margin-bottom:0;font-weight:400}.result_title a{color:#3498DB;text-decoration:none}#answers,#suggestions span{color:#666}.result_title a:focus,.result_title a:hover{text-decoration:underline}.cache_link{color:#666}.search.center{padding-right:17em}#answers{border:2px solid #3498DB;padding:20px;text-align:center;max-width:70em;margin:0 auto 20px}#suggestions{margin-bottom:1em}#suggestions form{vertical-align:top;margin-bottom:.5em}#suggestions input[type=submit]{color:#333;padding:.5em 1em;border:0;background:#CCC;cursor:pointer}#suggestions input[type=submit]:focus,#suggestions input[type=submit]:hover{background:#3498DB;color:#FFF}#pagination{margin:1.5em 0 2em}#pagination form+form{float:right;margin-top:-2em}input[type=submit]{display:inline-block;background:#3498DB;color:#FFF;border:0;padding:.6em 1em;cursor:pointer}input[type=submit]:focus,input[type=submit]:hover{background:#0665A2}.row{max-width:60em;margin:auto}.row a{color:#3498DB}.row form{letter-spacing:-5px}.row form>*{letter-spacing:normal}.row p{margin:0}.row fieldset{width:48%;vertical-align:top}.row fieldset:last-of-type{display:block;width:auto;background:0 0;padding:0}fieldset,table tr:nth-child(odd){background:#CCC}.row fieldset:nth-child(odd){margin-right:2%}.row fieldset:nth-child(2){min-height:10.5em}@media screen and (max-width:900px){.row{margin:0 1em}.row fieldset{width:49%}.row fieldset,.row fieldset:nth-child(odd){margin-right:0}.row fieldset:first-child{width:100%;margin-right:0}.row fieldset:nth-child(even){margin-right:2%}}@media screen and (max-width:800px){.row fieldset,select{width:100%}table{font-size:.8em}#sidebar,.right{display:none}#results{padding:0 2em}.search.center{padding-right:2em}}@media screen and (max-width:400px){.row #categories_container>div{width:100%;margin-left:0}}fieldset{border:0;margin:1em 0;padding:1.5em}table{width:100%;text-align:left;border:1px solid #CCC;border-collapse:collapse}table th{background:#999;color:#FFF}table td,table th{padding:.5em 1em;border:1px solid #FFF}.engine_checkbox label{padding:.5em;background:#3498DB;color:#FFF;cursor:pointer}.engine_checkbox .deny{background:#3498DB}.engine_checkbox .allow{display:none;background:#666}.engine_checkbox input{display:none}.engine_checkbox input:checked+.allow{display:inline}.engine_checkbox input:checked+.allow+.deny{display:none}.row input[type=submit]{font-size:1em;margin:1em 0 2em}.row .right{position:static;display:inline-block}.row .right a{color:#333;width:auto;text-align:left;padding:0}.small_font{font-size:.8em}table th{padding:1em}legend{background:#EEE;padding:0 1em;position:relative}select{border:1px solid #DDD;padding:.5em .8em;font-size:1em}.highlight .hll{background-color:#ffc}.highlight{font-weight:700;background:#f8f8f8}.highlight .c{color:#408080}.highlight .err{border:1px solid red}.highlight .k{color:green;font-weight:700}.highlight .o{color:#666}.highlight .cm{color:#408080}.highlight .cp{color:#BC7A00}.highlight .c1,.highlight .cs{color:#408080;font-style:italic}.highlight .gd{color:#A00000}.highlight .gr{color:red}.highlight .gh{color:navy;font-weight:700}.highlight .gi{color:#00A000}.highlight .go{color:#888}.highlight .gp{color:navy;font-weight:700}.highlight .gs{font-weight:700}.highlight .gu{color:purple;font-weight:700}.highlight .gt{color:#04D}.highlight .kc,.highlight .kd,.highlight .kn{color:green;font-weight:700}.highlight .kp{color:green}.highlight .kr{color:green;font-weight:700}.highlight .kt{color:#B00040}.highlight .m{color:#666}.highlight .s{color:#BA2121}.highlight .na{color:#7D9029}.highlight .nb{color:green}.highlight .nc{color:#00F;font-weight:700}.highlight .no{color:#800}.highlight .nd{color:#A2F}.highlight .ni{color:#999;font-weight:700}.highlight .ne{color:#D2413A;font-weight:700}.highlight .nf{color:#00F}.highlight .nl{color:#A0A000}.highlight .nn{color:#00F;font-weight:700}.highlight .nt{color:green;font-weight:700}.highlight .nv{color:#19177C}.highlight .ow{color:#A2F;font-weight:700}.highlight .w{color:#bbb}.highlight .mf,.highlight .mh,.highlight .mi,.highlight .mo{color:#666}.highlight .s2,.highlight .sb,.highlight .sc{color:#BA2121}.highlight .sd{color:#BA2121}.highlight .se{color:#B62;font-weight:700}.highlight .sh{color:#BA2121}.highlight .si{color:#B68;font-weight:700}.highlight .sx{color:green}.highlight .sr{color:#B68}.highlight .s1{color:#BA2121}.highlight .ss{color:#19177C}.highlight .bp{color:green}.highlight .vc,.highlight .vg,.highlight .vi{color:#19177C}.highlight .il{color:#666}.highlight pre{overflow:auto}.highlight .lineno{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;cursor:default}.highlight .lineno::selection{background:0 0}.highlight .lineno::-moz-selection{background:0 0}
\ No newline at end of file +a,h2{color:#666}.center,html{position:relative}#categories_container>div,.top_margin a{display:inline-block}#categories,.center{text-align:center}#categories .hidden,.cache_link,.highlight .c,.highlight .cm,.highlight .ge,.highlight .sd{font-style:italic}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}input[type=search]{-webkit-appearance:textfield}h2{text-transform:uppercase}body{font-family:sans-serif;line-height:1.5;margin:0;background:#EEE}html{min-height:100%}.title h1{font-size:7em;color:#3498DB;margin:-20px auto 0;line-height:100px;padding-bottom:20px}.center{max-width:70em;background:rgba(255,255,255,.6);padding:2em;margin:7% auto 0}.center.search{position:static;width:auto;background:0 0;margin:auto;padding-top:1.8em}@media screen and (min-width:1001px){.center:after{content:"";z-index:-1;background:url(../img/bg-body-index.jpg) no-repeat;background-size:cover;width:100%;height:100%;top:0;left:0;position:fixed}.center.search:after{content:none}}.autocompleter-choices{position:absolute;margin:0;padding:0;background:#FFF}.autocompleter-choices li{padding:.5em 1em}.autocompleter-choices li:hover{background:#3498DB;color:#FFF;cursor:pointer}.top_margin{position:absolute;bottom:-3.5em;width:100%;left:0}.top_margin a{margin-right:1em;color:#FFF;text-decoration:none}.top_margin a:focus,.top_margin a:hover{text-decoration:underline}@media screen and (max-width:1000px){.center{background:0 0}.top_margin a{color:#333}}.checkbox_container{margin-top:1.5em}.checkbox_container label{padding:.5em 1em;color:#333;cursor:pointer;font-size:.9em}.checkbox_container input[type=checkbox]:checked+label,.checkbox_container label:hover{background:#3498DB;color:#FFF}.checkbox_container input[type=checkbox]{position:absolute;top:-9999px}#categories .hidden{display:none;position:absolute;bottom:1em;left:0;text-align:center;width:100%;font-size:.9em;color:#333}#categories:hover .hidden,.right a{display:block}@media screen and (max-width:900px){#categories_container{letter-spacing:-5px}#categories_container>div{letter-spacing:normal;margin-top:1em}.checkbox_container{margin:0}.checkbox_container label{display:block;background:#CCC;padding:1em;border:1px solid #FFF}.top_margin{position:static}#categories .hidden{position:static;display:block}}@media screen and (max-width:900px) and (min-width:501px){#categories_container>div{width:31%;margin-left:2.333%}#categories_container>div:nth-child(3n+1){margin-left:0}}@media screen and (max-width:500px){#categories_container>div{width:48%;margin-left:2%;font-size:.9em}#categories_container>div:nth-child(2n+1){margin-left:0}.title h1{background:url(../img/searx-mobile.png) no-repeat;width:200px;height:39px}}#search_wrapper{position:relative}.q{padding:.5em 3em .5em 1em;width:100%;font-size:1.5em;border:0;color:#666}.cache_link,.result p{font-size:.9em}#search_submit{position:absolute;top:0;right:0;border:0;background:url(../img/search-icon.png) center center/65% auto no-repeat #3498DB;text-indent:-9999px;width:5em;height:100%;cursor:pointer}#sidebar,.right{position:fixed;width:15em;right:0;text-align:right}#search_submit:focus,#search_submit:hover{background-color:#0665A2}#sidebar{background:#3498DB;top:0;height:100%;padding:1.5em}.right{bottom:1.5em;z-index:1;padding:0 1.5em}.right a{color:#FFF;text-decoration:none}#sidebar form,#suggestions form,.row fieldset{display:inline-block}.right a:focus,.right a:hover{text-decoration:underline}#preferences{background:url(../img/preference-icon.png) right center/12% auto no-repeat;padding-right:1.8em}#search_url input{border:0;padding:.5em}#sidebar>div{margin-bottom:1em;color:#FFF}#sidebar input[type=submit]{background:#CCC;border:0;padding:.5em 1em;cursor:pointer;margin-top:.5em}#sidebar input[type=submit]:focus,#sidebar input[type=submit]:hover{color:#FFF;background-color:#0665A2}#results{padding:0 17em 0 2em}.result .engines{text-align:right}.result .content{margin:0;color:#666}.result .url{margin-top:0;color:#FF6530}.result .favicon{float:left;position:relative;top:.5em;margin-right:.5em}.definition_result{background:#CCC;padding:1em}.definition_result .result_title,.definition_result p{margin:0}.result_title{margin-bottom:0;font-weight:400}.result_title a{color:#3498DB;text-decoration:none}#answers,#suggestions span{color:#666}.result_title a:focus,.result_title a:hover{text-decoration:underline}.cache_link{color:#666}.search.center{padding-right:17em}#answers{border:2px solid #3498DB;padding:20px;text-align:center;max-width:70em;margin:0 auto 20px}#suggestions{margin-bottom:1em}#suggestions form{vertical-align:top;margin-bottom:.5em}#suggestions input[type=submit]{color:#333;padding:.5em 1em;border:0;background:#CCC;cursor:pointer}#suggestions input[type=submit]:focus,#suggestions input[type=submit]:hover{background:#3498DB;color:#FFF}#pagination{margin:1.5em 0 2em}#pagination form+form{float:right;margin-top:-2em}input[type=submit]{display:inline-block;background:#3498DB;color:#FFF;border:0;padding:.6em 1em;cursor:pointer}input[type=submit]:focus,input[type=submit]:hover{background:#0665A2}.row{max-width:60em;margin:auto}.row a{color:#3498DB}.row form{letter-spacing:-5px}.row form>*{letter-spacing:normal}.row p{margin:0}.row fieldset{width:48%;vertical-align:top}.row fieldset:last-of-type{display:block;width:auto;background:0 0;padding:0}fieldset,table tr:nth-child(odd){background:#CCC}.row fieldset:nth-child(odd){margin-right:2%}.row fieldset:nth-child(2){min-height:10.5em}@media screen and (max-width:900px){.row{margin:0 1em}.row fieldset{width:49%}.row fieldset,.row fieldset:nth-child(odd){margin-right:0}.row fieldset:first-child{width:100%;margin-right:0}.row fieldset:nth-child(even){margin-right:2%}}@media screen and (max-width:800px){.row fieldset,select{width:100%}table{font-size:.8em}#sidebar,.right{display:none}#results{padding:0 2em}.search.center{padding-right:2em}}@media screen and (max-width:400px){.row #categories_container>div{width:100%;margin-left:0}}fieldset{border:0;margin:1em 0;padding:1.5em}table{width:100%;text-align:left;border:1px solid #CCC;border-collapse:collapse}table th{background:#999;color:#FFF}table td,table th{padding:.5em 1em;border:1px solid #FFF}.engine_checkbox label{padding:.5em;background:#3498DB;color:#FFF;cursor:pointer}.engine_checkbox .deny{background:#3498DB}.engine_checkbox .allow{display:none;background:#666}.engine_checkbox input{display:none}.engine_checkbox input:checked+.allow{display:inline}.engine_checkbox input:checked+.allow+.deny{display:none}.row input[type=submit]{font-size:1em;margin:1em 0 2em}.row .right{position:static;display:inline-block}.row .right a{color:#333;width:auto;text-align:left;padding:0}.small_font{font-size:.8em}table th{padding:1em}legend{background:#EEE;padding:0 1em;position:relative}select{border:1px solid #DDD;padding:.5em .8em;font-size:1em}.highlight .hll{background-color:#ffc}.highlight{font-weight:700;background:#f8f8f8}.highlight .c{color:#408080}.highlight .err{border:1px solid red}.highlight .k{color:green;font-weight:700}.highlight .o{color:#666}.highlight .cm{color:#408080}.highlight .cp{color:#BC7A00}.highlight .c1,.highlight .cs{color:#408080;font-style:italic}.highlight .gd{color:#A00000}.highlight .gr{color:red}.highlight .gh{color:navy;font-weight:700}.highlight .gi{color:#00A000}.highlight .go{color:#888}.highlight .gp{color:navy;font-weight:700}.highlight .gs{font-weight:700}.highlight .gu{color:purple;font-weight:700}.highlight .gt{color:#04D}.highlight .kc,.highlight .kd,.highlight .kn{color:green;font-weight:700}.highlight .kp{color:green}.highlight .kr{color:green;font-weight:700}.highlight .kt{color:#B00040}.highlight .m{color:#666}.highlight .s{color:#BA2121}.highlight .na{color:#7D9029}.highlight .nb{color:green}.highlight .nc{color:#00F;font-weight:700}.highlight .no{color:#800}.highlight .nd{color:#A2F}.highlight .ni{color:#999;font-weight:700}.highlight .ne{color:#D2413A;font-weight:700}.highlight .nf{color:#00F}.highlight .nl{color:#A0A000}.highlight .nn{color:#00F;font-weight:700}.highlight .nt{color:green;font-weight:700}.highlight .nv{color:#19177C}.highlight .ow{color:#A2F;font-weight:700}.highlight .w{color:#bbb}.highlight .mf,.highlight .mh,.highlight .mi,.highlight .mo{color:#666}.highlight .s2,.highlight .sb,.highlight .sc{color:#BA2121}.highlight .sd{color:#BA2121}.highlight .se{color:#B62;font-weight:700}.highlight .sh{color:#BA2121}.highlight .si{color:#B68;font-weight:700}.highlight .sx{color:green}.highlight .sr{color:#B68}.highlight .s1{color:#BA2121}.highlight .ss{color:#19177C}.highlight .bp{color:green}.highlight .vc,.highlight .vg,.highlight .vi{color:#19177C}.highlight .il{color:#666}.highlight pre{overflow:auto}.highlight .lineno{-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;cursor:default}.highlight .lineno::selection{background:0 0}.highlight .lineno::-moz-selection{background:0 0}
\ No newline at end of file diff --git a/searx/static/themes/courgette/less/style.less b/searx/static/themes/courgette/less/style.less index 0387af5c0..26da72812 100644 --- a/searx/static/themes/courgette/less/style.less +++ b/searx/static/themes/courgette/less/style.less @@ -325,6 +325,10 @@ a { font-size: 0.9em; } +.result .engines { + text-align: right; +} + .result .content { margin: 0; color: #666; diff --git a/searx/static/themes/legacy/css/style.css b/searx/static/themes/legacy/css/style.css index f434148bd..ca746a369 100644 --- a/searx/static/themes/legacy/css/style.css +++ b/searx/static/themes/legacy/css/style.css @@ -1 +1 @@ -.highlight .c,.highlight .cm,.highlight .ge,.highlight .sd{font-style:italic}#categories,.highlight .lineno{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none}#container,.search,body,html{padding:0;margin:0}div.title h1,input[type=checkbox]{visibility:hidden}#categories,.checkbox_container label,.engine_checkbox label,.highlight .lineno{-webkit-touch-callout:none;-khtml-user-select:none}#answers input[type=submit],#infoboxes input[type=submit],#sidebar input[type=submit],#suggestions input[type=submit],.result_title a:hover,.torrent_result a:hover{text-decoration:underline}#infoboxes,.result .content,.result .url,.result h3{word-wrap:break-word}#apis,#infoboxes .infobox br,#pagination,#pagination br,.result,.result .content br.last{clear:both}.highlight .hll{background-color:#ffc}.highlight{background:#f8f8f8}.highlight .c{color:#408080}.highlight .err{border:1px solid red}.highlight .k{color:green;font-weight:700}.highlight .o{color:#666}.highlight .cm{color:#408080}.highlight .cp{color:#BC7A00}.highlight .c1,.highlight .cs{color:#408080;font-style:italic}.highlight .gd{color:#A00000}.highlight .gr{color:red}.highlight .gh{color:navy;font-weight:700}.highlight .gi{color:#00A000}.highlight .go{color:#888}.highlight .gp{color:navy;font-weight:700}.highlight .gs{font-weight:700}.highlight .gu{color:purple;font-weight:700}.highlight .gt{color:#04D}.highlight .kc,.highlight .kd,.highlight .kn{color:green;font-weight:700}.highlight .kp{color:green}.highlight .kr{color:green;font-weight:700}.highlight .kt{color:#B00040}.highlight .m{color:#666}.highlight .s{color:#BA2121}.highlight .na{color:#7D9029}.highlight .nb{color:green}.highlight .nc{color:#00F;font-weight:700}.highlight .no{color:#800}.highlight .nd{color:#A2F}.highlight .ni{color:#999;font-weight:700}.highlight .ne{color:#D2413A;font-weight:700}.highlight .nf{color:#00F}.highlight .nl{color:#A0A000}.highlight .nn{color:#00F;font-weight:700}.highlight .nt{color:green;font-weight:700}.highlight .nv{color:#19177C}.highlight .ow{color:#A2F;font-weight:700}.highlight .w{color:#bbb}.highlight .mf,.highlight .mh,.highlight .mi,.highlight .mo{color:#666}.highlight .s2,.highlight .sb,.highlight .sc{color:#BA2121}.highlight .sd{color:#BA2121}.highlight .se{color:#B62;font-weight:700}.highlight .sh{color:#BA2121}.highlight .si{color:#B68;font-weight:700}.highlight .sx{color:green}.highlight .sr{color:#B68}.highlight .s1{color:#BA2121}.highlight .ss{color:#19177C}.highlight .bp{color:green}.highlight .vc,.highlight .vg,.highlight .vi{color:#19177C}.highlight .il{color:#666}.highlight pre{overflow:auto}.highlight .lineno{user-select:none;cursor:default}.highlight .lineno::selection{background:0 0}.highlight .lineno::-moz-selection{background:0 0}html{font-family:sans-serif;font-size:.9em;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-moz-text-size-adjust:100%;text-size-adjust:100%;color:#444}#container{width:100%;position:absolute;top:0}.search .checkbox_container label{font-size:.9em;border-bottom:2px solid #E8E7E6}.search .checkbox_container label:hover{border-bottom:2px solid #3498DB}.search .checkbox_container input[type=checkbox]:checked+label{border-bottom:2px solid #2980B9}#search_wrapper{position:relative;width:50em;padding:10px}.center #search_wrapper{margin-left:auto;margin-right:auto}.q,ul.autocompleter-choices{margin:0;border:1px solid #3498DB}.q{background:#FFF;color:#222;font-size:16px;height:28px;outline:0;padding:2px 2px 2px 8px;padding-right:0!important;width:100%;z-index:2}#search_submit{position:absolute;top:13px;right:1px;padding:0;border:0;background:url(../img/search-icon.png) no-repeat;background-size:24px 24px;opacity:.8;width:24px;height:30px;font-size:0}@media screen and (max-width:50em){#search_wrapper{width:90%;clear:both;overflow:hidden}}ul.autocompleter-choices{position:absolute;padding:0;list-style:none;border-left-color:#3498DB;border-right-color:#3498DB;border-bottom-color:#3498DB;text-align:left;font-family:Verdana,Geneva,Arial,Helvetica,sans-serif;z-index:50;background-color:#FFF;color:#444}ul.autocompleter-choices li{position:relative;margin:-2px 0 0;padding:.2em 1.5em .2em 1em;display:block;float:none!important;cursor:pointer;font-weight:400;white-space:nowrap;font-size:1em;line-height:1.5em}ul.autocompleter-choices li.autocompleter-selected{background-color:#444;color:#FFF}ul.autocompleter-choices li.autocompleter-selected span.autocompleter-queried{color:#9FCFFF}ul.autocompleter-choices span.autocompleter-queried{display:inline;float:none;font-weight:700;margin:0;padding:0}.row{max-width:800px;margin:20px auto;text-align:justify}.row h1{font-size:3em;margin-top:50px}.row p{padding:0 10px;max-width:700px}.row h3,.row ul{margin:4px 8px}.hmarg{margin:0 20px;border:1px solid #3498DB;padding:4px 10px}a:active.hmarg,a:hover.hmarg,a:link.hmarg,a:visited.hmarg{color:#3498DB}.top_margin{margin-top:60px}.center{text-align:center}h1{font-size:5em}div.title{background:url(../img/searx.png) center no-repeat;width:100%;min-height:80px}input[type=submit]{padding:2px 6px;margin:2px 4px;display:inline-block;background:#3498DB;color:#FFF;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;border:0;cursor:pointer}fieldset{margin:8px;border:1px solid #3498DB}#categories{margin:0 10px;user-select:none}.checkbox_container{display:inline-block;position:relative;margin:0 3px;padding:0}.checkbox_container input{display:none}.checkbox_container label,.engine_checkbox label{cursor:pointer;padding:4px 10px;margin:0;display:block;text-transform:capitalize;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.checkbox_container input[type=checkbox]:checked+label{background:#3498DB;color:#FFF}.engine_checkbox{padding:4px}label.allow{background:#E74C3C;padding:4px 8px;color:#FFF;display:none}label.deny{background:#2ECC71;padding:4px 8px;color:#444;display:inline}.engine_checkbox input[type=checkbox]:checked+label:nth-child(2)+label{display:none}.engine_checkbox input[type=checkbox]:checked+label.allow{display:inline}a{text-decoration:none;color:#1a11be}a:visited{color:#8E44AD}.result{margin:19px 0 18px;padding:0}.result_title{margin-bottom:0}.result_title a{color:#2980B9;font-weight:400;font-size:1.1em}.result_title a:visited{color:#8E44AD}.cache_link{font-size:10px!important}.result h3{font-size:1em;margin:5px 0 1px;padding:0}.result .content,.result .url,.small_font{font-size:.8em}.result .content{margin:0;padding:0;max-width:54em;line-height:1.24}.result .content img{float:left;margin-right:5px;max-width:200px;max-height:100px}.result .url{margin:0 0 3px;padding:0;max-width:54em;color:#C0392B}.result .published_date{font-size:.8em;color:#888;Margin:5px 20px}.result .thumbnail{width:400px}.engines{color:#888}.small p{margin:2px 0}.right{float:right}.invisible{display:none}.left{float:left}.highlight{color:#094089}.content .highlight{color:#000}.image_result{display:inline-block;margin:10px;position:relative;max-height:160px}.image_result img{border:0;max-height:160px}.image_result p{margin:0;padding:0}.image_result p span a{display:none;color:#FFF}.image_result p:hover span a{display:block;position:absolute;bottom:0;right:0;padding:4px;background-color:rgba(0,0,0,.6);font-size:.7em}#categories_container,.percentage{position:relative}.torrent_result{border-left:10px solid #d3d3d3;padding-left:3px}.torrent_result p{margin:3px;font-size:.8em}.torrent_result a{color:#2980B9}.torrent_result a:visited{color:#8E44AD}.definition_result{border-left:10px solid gray;padding-left:3px}#infoboxes,#sidebar{margin:0 2px 5px 5px;padding:0 2px 2px}.percentage{width:300px}.percentage div{background:#444}table{width:100%}td{padding:0 4px}tr:hover{background:#DDD}#results{margin:auto auto 20px;padding:0;width:50em}#sidebar{position:fixed;bottom:10px;left:10px;width:14em}#answers input,#infoboxes input,#sidebar input,#suggestions input{padding:0;margin:3px;font-size:.8em;display:inline-block;background:0 0;color:#444;cursor:pointer}#suggestions form{display:inline}#answers,#suggestions{margin-top:20px;max-width:45em}#suggestions-title{color:#888}#answers{border:2px solid #2980B9;padding:20px}#answers form,#infoboxes form{min-width:210px}#infoboxes{position:absolute;top:100px;right:20px;max-width:21em}#infoboxes .infobox{margin:10px 0;border:1px solid #ddd;padding:5px;font-size:.8em}#infoboxes .infobox img{max-width:90%;max-heigt:12em;display:block;margin:5px;padding:5px}#infoboxes .infobox h2{margin:0}#apis,#search_url{margin-top:8px}#infoboxes .infobox table{table-layout:fixed}#infoboxes .infobox table td{vertical-align:top}#infoboxes .infobox input{font-size:1em}#search_url input{border:1px solid #888;padding:4px;color:#444;width:14em;display:block;margin:4px;font-size:.8em}#preferences{top:10px;padding:0;border:0;background:url(../img/preference-icon.png) no-repeat;background-size:28px 28px;opacity:.8;width:28px;height:30px;display:block}#preferences *{display:none}@media screen and (max-width:50em){#results{margin:auto;padding:0;width:90%}.github{display:none}.checkbox_container{display:block;width:90%}.checkbox_container label{border-bottom:0}.preferences_container{display:none;postion:fixed!important;top:100px;right:0}}@media screen and (max-width:75em){div.title h1{font-size:1em}html.touch #categories{width:95%;height:30px;text-align:left;overflow-x:scroll;overflow-y:hidden;-webkit-overflow-scrolling:touch}html.touch #categories #categories_container{width:1000px;width:-moz-max-content;width:-webkit-max-content;width:max-content}html.touch #categories #categories_container .checkbox_container{display:inline-block;width:auto}#answers,#suggestions{margin-top:5px}#infoboxes{position:inherit;max-width:inherit}#infoboxes .infobox{clear:both}#infoboxes .infobox img{float:left;max-width:10em}#categories{font-size:90%;clear:both}#categories .checkbox_container{margin:auto}#sidebar{position:static;max-width:50em;margin:0 0 2px;padding:0;float:none;border:none;width:auto}#sidebar input{border:0}#apis,#search_url{display:none}.result{border-top:1px solid #E8E7E6;margin:8px 0}.image_result,.image_result img,.result .thumbnail{max-width:98%}}.favicon{float:left;margin-right:4px;margin-top:2px}.preferences_back{background:#3498DB;border:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;cursor:pointer;display:inline-block;margin:2px 4px;padding:4px 6px}.preferences_back a{color:#FFF}.hidden{opacity:0;overflow:hidden;font-size:.8em;position:absolute;bottom:-20px;width:100%;text-position:center;background:#fff;transition:opacity 1s ease}#categories_container:hover .hidden{transition:opacity 1s ease;opacity:.8}
\ No newline at end of file +.highlight .c,.highlight .cm,.highlight .ge,.highlight .sd{font-style:italic}#categories,.highlight .lineno{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none}#container,.search,body,html{padding:0;margin:0}div.title h1,input[type=checkbox]{visibility:hidden}#categories,.checkbox_container label,.engine_checkbox label,.highlight .lineno{-webkit-touch-callout:none;-khtml-user-select:none}#answers input[type=submit],#infoboxes input[type=submit],#sidebar input[type=submit],#suggestions input[type=submit],.result_title a:hover,.torrent_result a:hover{text-decoration:underline}#infoboxes,.result .content,.result .url,.result h3{word-wrap:break-word}#apis,#infoboxes .infobox br,#pagination,#pagination br,.result,.result .content br.last{clear:both}.highlight .hll{background-color:#ffc}.highlight{background:#f8f8f8}.highlight .c{color:#408080}.highlight .err{border:1px solid red}.highlight .k{color:green;font-weight:700}.highlight .o{color:#666}.highlight .cm{color:#408080}.highlight .cp{color:#BC7A00}.highlight .c1,.highlight .cs{color:#408080;font-style:italic}.highlight .gd{color:#A00000}.highlight .gr{color:red}.highlight .gh{color:navy;font-weight:700}.highlight .gi{color:#00A000}.highlight .go{color:#888}.highlight .gp{color:navy;font-weight:700}.highlight .gs{font-weight:700}.highlight .gu{color:purple;font-weight:700}.highlight .gt{color:#04D}.highlight .kc,.highlight .kd,.highlight .kn{color:green;font-weight:700}.highlight .kp{color:green}.highlight .kr{color:green;font-weight:700}.highlight .kt{color:#B00040}.highlight .m{color:#666}.highlight .s{color:#BA2121}.highlight .na{color:#7D9029}.highlight .nb{color:green}.highlight .nc{color:#00F;font-weight:700}.highlight .no{color:#800}.highlight .nd{color:#A2F}.highlight .ni{color:#999;font-weight:700}.highlight .ne{color:#D2413A;font-weight:700}.highlight .nf{color:#00F}.highlight .nl{color:#A0A000}.highlight .nn{color:#00F;font-weight:700}.highlight .nt{color:green;font-weight:700}.highlight .nv{color:#19177C}.highlight .ow{color:#A2F;font-weight:700}.highlight .w{color:#bbb}.highlight .mf,.highlight .mh,.highlight .mi,.highlight .mo{color:#666}.highlight .s2,.highlight .sb,.highlight .sc{color:#BA2121}.highlight .sd{color:#BA2121}.highlight .se{color:#B62;font-weight:700}.highlight .sh{color:#BA2121}.highlight .si{color:#B68;font-weight:700}.highlight .sx{color:green}.highlight .sr{color:#B68}.highlight .s1{color:#BA2121}.highlight .ss{color:#19177C}.highlight .bp{color:green}.highlight .vc,.highlight .vg,.highlight .vi{color:#19177C}.highlight .il{color:#666}.highlight pre{overflow:auto}.highlight .lineno{user-select:none;cursor:default}.highlight .lineno::selection{background:0 0}.highlight .lineno::-moz-selection{background:0 0}html{font-family:sans-serif;font-size:.9em;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-moz-text-size-adjust:100%;text-size-adjust:100%;color:#444}#container{width:100%;position:absolute;top:0}.search .checkbox_container label{font-size:.9em;border-bottom:2px solid #E8E7E6}.search .checkbox_container label:hover{border-bottom:2px solid #3498DB}.search .checkbox_container input[type=checkbox]:checked+label{border-bottom:2px solid #2980B9}#search_wrapper{position:relative;width:50em;padding:10px}.center #search_wrapper{margin-left:auto;margin-right:auto}.q,ul.autocompleter-choices{margin:0;border:1px solid #3498DB}.q{background:#FFF;color:#222;font-size:16px;height:28px;outline:0;padding:2px 2px 2px 8px;padding-right:0!important;width:100%;z-index:2}#search_submit{position:absolute;top:13px;right:1px;padding:0;border:0;background:url(../img/search-icon.png) no-repeat;background-size:24px 24px;opacity:.8;width:24px;height:30px;font-size:0}@media screen and (max-width:50em){#search_wrapper{width:90%;clear:both;overflow:hidden}}ul.autocompleter-choices{position:absolute;padding:0;list-style:none;border-left-color:#3498DB;border-right-color:#3498DB;border-bottom-color:#3498DB;text-align:left;font-family:Verdana,Geneva,Arial,Helvetica,sans-serif;z-index:50;background-color:#FFF;color:#444}ul.autocompleter-choices li{position:relative;margin:-2px 0 0;padding:.2em 1.5em .2em 1em;display:block;float:none!important;cursor:pointer;font-weight:400;white-space:nowrap;font-size:1em;line-height:1.5em}ul.autocompleter-choices li.autocompleter-selected{background-color:#444;color:#FFF}ul.autocompleter-choices li.autocompleter-selected span.autocompleter-queried{color:#9FCFFF}ul.autocompleter-choices span.autocompleter-queried{display:inline;float:none;font-weight:700;margin:0;padding:0}.row{max-width:800px;margin:20px auto;text-align:justify}.row h1{font-size:3em;margin-top:50px}.row p{padding:0 10px;max-width:700px}.row h3,.row ul{margin:4px 8px}.hmarg{margin:0 20px;border:1px solid #3498DB;padding:4px 10px}a:active.hmarg,a:hover.hmarg,a:link.hmarg,a:visited.hmarg{color:#3498DB}.top_margin{margin-top:60px}.center{text-align:center}h1{font-size:5em}div.title{background:url(../img/searx.png) center no-repeat;width:100%;min-height:80px}input[type=submit]{padding:2px 6px;margin:2px 4px;display:inline-block;background:#3498DB;color:#FFF;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;border:0;cursor:pointer}fieldset{margin:8px;border:1px solid #3498DB}#categories{margin:0 10px;user-select:none}.checkbox_container{display:inline-block;position:relative;margin:0 3px;padding:0}.checkbox_container input{display:none}.checkbox_container label,.engine_checkbox label{cursor:pointer;padding:4px 10px;margin:0;display:block;text-transform:capitalize;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.checkbox_container input[type=checkbox]:checked+label{background:#3498DB;color:#FFF}.engine_checkbox{padding:4px}label.allow{background:#E74C3C;padding:4px 8px;color:#FFF;display:none}label.deny{background:#2ECC71;padding:4px 8px;color:#444;display:inline}.engine_checkbox input[type=checkbox]:checked+label:nth-child(2)+label{display:none}.engine_checkbox input[type=checkbox]:checked+label.allow{display:inline}a{text-decoration:none;color:#1a11be}a:visited{color:#8E44AD}.result{margin:19px 0 18px;padding:0}.result_title{margin-bottom:0}.result_title a{color:#2980B9;font-weight:400;font-size:1.1em}.result_title a:visited{color:#8E44AD}.cache_link{font-size:10px!important}.result h3{font-size:1em;margin:5px 0 1px;padding:0}.result .content,.result .url,.small_font{font-size:.8em}.result .content{margin:0;padding:0;max-width:54em;line-height:1.24}.result .content img{float:left;margin-right:5px;max-width:200px;max-height:100px}.result .url{margin:0 0 3px;padding:0;max-width:54em;color:#C0392B}.result .published_date{font-size:.8em;color:#888;Margin:5px 20px}.result .thumbnail{width:400px}.engines{color:#888}.small p{margin:2px 0}.right{float:right}.invisible{display:none}.left{float:left}.highlight{color:#094089}.content .highlight{color:#000}.image_result{display:inline-block;margin:10px;position:relative;max-height:160px}.image_result img{border:0;max-height:160px}.image_result p{margin:0;padding:0}.image_result p span a{display:none;color:#FFF}.image_result p:hover span a{display:block;position:absolute;bottom:0;right:0;padding:4px;background-color:rgba(0,0,0,.6);font-size:.7em}#categories_container,.percentage{position:relative}.torrent_result{border-left:10px solid #d3d3d3;padding-left:3px}.torrent_result p{margin:3px;font-size:.8em}.torrent_result a{color:#2980B9}.torrent_result a:visited{color:#8E44AD}.definition_result{border-left:10px solid gray;padding-left:3px}.percentage{width:300px}.percentage div{background:#444}table{width:100%}.result-table{margin-bottom:10px}#infoboxes,#sidebar{margin:0 2px 5px 5px;padding:0 2px 2px}td{padding:0 4px}tr:hover{background:#DDD}#results{margin:auto auto 20px;padding:0;width:50em}#sidebar{position:fixed;bottom:10px;left:10px;width:14em}#answers input,#infoboxes input,#sidebar input,#suggestions input{padding:0;margin:3px;font-size:.8em;display:inline-block;background:0 0;color:#444;cursor:pointer}#suggestions form{display:inline}#answers,#suggestions{margin-top:20px;max-width:45em}#suggestions-title{color:#888}#answers{border:2px solid #2980B9;padding:20px}#answers form,#infoboxes form{min-width:210px}#infoboxes{position:absolute;top:100px;right:20px;max-width:21em}#infoboxes .infobox{margin:10px 0;border:1px solid #ddd;padding:5px;font-size:.8em}#infoboxes .infobox img{max-width:90%;max-heigt:12em;display:block;margin:5px;padding:5px}#infoboxes .infobox h2{margin:0}#apis,#search_url{margin-top:8px}#infoboxes .infobox table{table-layout:fixed}#infoboxes .infobox table td{vertical-align:top}#infoboxes .infobox input{font-size:1em}#search_url input{border:1px solid #888;padding:4px;color:#444;width:14em;display:block;margin:4px;font-size:.8em}#preferences{top:10px;padding:0;border:0;background:url(../img/preference-icon.png) no-repeat;background-size:28px 28px;opacity:.8;width:28px;height:30px;display:block}#preferences *{display:none}@media screen and (max-width:50em){#results{margin:auto;padding:0;width:90%}.github{display:none}.checkbox_container{display:block;width:90%}.checkbox_container label{border-bottom:0}.preferences_container{display:none;postion:fixed!important;top:100px;right:0}}@media screen and (max-width:75em){div.title h1{font-size:1em}html.touch #categories{width:95%;height:30px;text-align:left;overflow-x:scroll;overflow-y:hidden;-webkit-overflow-scrolling:touch}html.touch #categories #categories_container{width:1000px;width:-moz-max-content;width:-webkit-max-content;width:max-content}html.touch #categories #categories_container .checkbox_container{display:inline-block;width:auto}#answers,#suggestions{margin-top:5px}#infoboxes{position:inherit;max-width:inherit}#infoboxes .infobox{clear:both}#infoboxes .infobox img{float:left;max-width:10em}#categories{font-size:90%;clear:both}#categories .checkbox_container{margin:auto}#sidebar{position:static;max-width:50em;margin:0 0 2px;padding:0;float:none;border:none;width:auto}#sidebar input{border:0}#apis,#search_url{display:none}.result{border-top:1px solid #E8E7E6;margin:8px 0}.image_result,.image_result img,.result .thumbnail{max-width:98%}}.favicon{float:left;margin-right:4px;margin-top:2px}.preferences_back{background:#3498DB;border:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;cursor:pointer;display:inline-block;margin:2px 4px;padding:4px 6px}.preferences_back a{color:#FFF}.hidden{opacity:0;overflow:hidden;font-size:.8em;position:absolute;bottom:-20px;width:100%;text-position:center;background:#fff;transition:opacity 1s ease}#categories_container:hover .hidden{transition:opacity 1s ease;opacity:.8}
\ No newline at end of file diff --git a/searx/static/themes/legacy/less/style.less b/searx/static/themes/legacy/less/style.less index 4374f7d68..bbeaf105e 100644 --- a/searx/static/themes/legacy/less/style.less +++ b/searx/static/themes/legacy/less/style.less @@ -376,6 +376,10 @@ table { width: 100%; } +.result-table { + margin-bottom: 10px; +} + td { padding: 0 4px; } diff --git a/searx/templates/courgette/result_templates/key-value.html b/searx/templates/courgette/result_templates/key-value.html new file mode 100644 index 000000000..789e8de92 --- /dev/null +++ b/searx/templates/courgette/result_templates/key-value.html @@ -0,0 +1,13 @@ +<div class="result"> +<table> + {% for key, value in result.items() %} + {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions'] %} + {% continue %} + {% endif %} + <tr> + <td><b>{{ key|upper }}</b>: {{ value|safe }}</td> + </tr> + {% endfor %} +</table> +<p class="engines">{{ result.engines|join(', ') }}</p> +</div> diff --git a/searx/templates/courgette/result_templates/torrent.html b/searx/templates/courgette/result_templates/torrent.html index d659064d9..7f94a221e 100644 --- a/searx/templates/courgette/result_templates/torrent.html +++ b/searx/templates/courgette/result_templates/torrent.html @@ -4,7 +4,7 @@ {% endif %} <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3> {% if result.content %}<span class="content">{{ result.content|safe }}</span><br />{% endif %} - {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %} + {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %} <span> {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} {% if result.torrentfile %}<a href="{{ result.torrentfile }}" class="torrentfile" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('torrent file') }}</a>{% endif %} diff --git a/searx/templates/legacy/result_templates/key-value.html b/searx/templates/legacy/result_templates/key-value.html new file mode 100644 index 000000000..a5bb509d9 --- /dev/null +++ b/searx/templates/legacy/result_templates/key-value.html @@ -0,0 +1,13 @@ +<table class="result-table"> + {% for key, value in result.items() %} + {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions'] %} + {% continue %} + {% endif %} + <tr> + <td><b>{{ key|upper }}</b>: {{ value|safe }}</td> + </tr> + {% endfor %} + <tr> + <td><b>ENGINES</b>: {{ result.engines|join(', ') }}</td> + </tr> +</table> diff --git a/searx/templates/legacy/result_templates/torrent.html b/searx/templates/legacy/result_templates/torrent.html index 7a8ac33de..068e05373 100644 --- a/searx/templates/legacy/result_templates/torrent.html +++ b/searx/templates/legacy/result_templates/torrent.html @@ -8,6 +8,6 @@ <p> {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} {% if result.torrentfile %}<a href="{{ result.torrentfile }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="torrentfile">{{ _('torrent file') }}</a>{% endif %} - - {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %} + {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %} </p> </div> diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html index 0ff957521..5f6463642 100644 --- a/searx/templates/oscar/macros.html +++ b/searx/templates/oscar/macros.html @@ -14,7 +14,7 @@ <!-- Draw result header --> {% macro result_header(result, favicons) -%} -<h4 class="result_header">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{{ result_link(result.url, result.title|safe) }}</h4> +<h4 class="result_header">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{% if result.url %}{{ result_link(result.url, result.title|safe) }}{% else %}{{ result.title|safe}}{% endif %}</h4> {%- endmacro %} <!-- Draw result sub header --> @@ -31,12 +31,16 @@ {% for engine in result.engines %} <span class="label label-default">{{ engine }}</span> {% endfor %} + {% if result.url %} <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small> + {% endif %} {% if proxify %} <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small> {% endif %} </div> +{% if result.pretty_url %} <div class="external-link">{{ result.pretty_url }}</div> +{% endif %} {%- endmacro %} <!-- Draw result footer --> @@ -45,11 +49,15 @@ {% for engine in result.engines %} <span class="label label-default">{{ engine }}</span> {% endfor %} + {% if result.url %} <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small> + {% endif %} {% if proxify %} <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small> {% endif %} + {% if result.pretty_url %} <div class="external-link">{{ result.pretty_url }}</div> + {% endif %} {%- endmacro %} {% macro preferences_item_header(info, label, rtl) -%} diff --git a/searx/templates/oscar/result_templates/key-value.html b/searx/templates/oscar/result_templates/key-value.html new file mode 100644 index 000000000..67c748e7f --- /dev/null +++ b/searx/templates/oscar/result_templates/key-value.html @@ -0,0 +1,19 @@ +{% from 'oscar/macros.html' import result_footer, result_footer_rtl with context %} +<div class="panel panel-default"> +<table class="table table-responsive table-bordered table-condensed"> + {% for key, value in result.items() %} + {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions'] %} + {% continue %} + {% endif %} + <tr> + <td><b>{{ key|upper }}</b>: {{ value }}</td> + </tr> + {% endfor %} +</table> + +{% if rtl %} +{{ result_footer_rtl(result) }} +{% else %} +{{ result_footer(result) }} +{% endif %} +</div> diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html index f5ea415e2..089367e36 100644 --- a/searx/templates/oscar/result_templates/torrent.html +++ b/searx/templates/oscar/result_templates/torrent.html @@ -3,7 +3,7 @@ {{ result_header(result, favicons) }} {{ result_sub_header(result) }} -{% if result.seed %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %} +{% if result.seed is defined %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %} {% if result.filesize %}<br />{{ icon('floppy-disk') }} {{ _('Filesize') }} <span class="badge"> {% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }} diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html index 9a95265b9..fce7f97d6 100644 --- a/searx/templates/oscar/results.html +++ b/searx/templates/oscar/results.html @@ -22,8 +22,8 @@ <span class="result_header text-muted form-inline pull-left suggestion_item">{{ _('Try searching for:') }}</span>
{% for correction in corrections %}
<form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation" class="form-inline pull-left suggestion_item">
- <input type="hidden" name="q" value="{{ query_prefix + correction }}">
- <button type="submit" class="btn btn-default btn-xs">{{ correction }}</button>
+ <input type="hidden" name="q" value="{{ correction.url }}">
+ <button type="submit" class="btn btn-default btn-xs">{{ correction.title }}</button>
</form>
{% endfor %}
</div>
diff --git a/searx/templates/oscar/search.html b/searx/templates/oscar/search.html index 59ee4688d..bf9a9af5f 100644 --- a/searx/templates/oscar/search.html +++ b/searx/templates/oscar/search.html @@ -3,9 +3,9 @@ <div class="row">
<div class="col-xs-12 col-md-8">
<div class="input-group search-margin">
- <input type="search" name="q" class="form-control" id="q" placeholder="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}">
+ <input type="search" name="q" class="form-control" id="q" placeholder="{{ _('Search for...') }}" aria-label="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}">
<span class="input-group-btn">
- <button type="submit" class="btn btn-default"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
+ <button type="submit" class="btn btn-default" aria-label="{{ _('Start search') }}"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
</span>
</div>
</div>
diff --git a/searx/templates/oscar/search_full.html b/searx/templates/oscar/search_full.html index 6fdae4028..fd8a9a393 100644 --- a/searx/templates/oscar/search_full.html +++ b/searx/templates/oscar/search_full.html @@ -6,9 +6,9 @@ {% else %}
<div class="input-group col-md-8 col-md-offset-2">
{% endif %}
- <input type="search" name="q" class="form-control input-lg autofocus" id="q" placeholder="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}">
+ <input type="search" name="q" class="form-control input-lg autofocus" id="q" placeholder="{{ _('Search for...') }}" aria-label="{{ _('Search for...') }}" autocomplete="off" value="{{ q }}">
<span class="input-group-btn">
- <button type="submit" class="btn btn-default input-lg"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
+ <button type="submit" class="btn btn-default input-lg" aria-label="{{ _('Start search') }}"><span class="hide_if_nojs">{{ icon('search') }}</span><span class="hidden active_if_nojs">{{ _('Start search') }}</span></button>
</span>
</div>
<div class="col-md-8 col-md-offset-2 advanced">
diff --git a/searx/templates/simple/result_templates/key-value.html b/searx/templates/simple/result_templates/key-value.html new file mode 100644 index 000000000..eebaa2c85 --- /dev/null +++ b/searx/templates/simple/result_templates/key-value.html @@ -0,0 +1,11 @@ +<table> + {% for key, value in result.items() %} + {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions'] %} + {% continue %} + {% endif %} + <tr> + <td><b>{{ key|upper }}</b>: {{ value }}</td> + </tr> + {% endfor %} +</table> +<div class="engines">{% for engine in result.engines %}<span>{{ engine }}</span>{% endfor %}</div>{{- '' -}} diff --git a/searx/templates/simple/result_templates/torrent.html b/searx/templates/simple/result_templates/torrent.html index 3c7fd15e8..71c775bc9 100644 --- a/searx/templates/simple/result_templates/torrent.html +++ b/searx/templates/simple/result_templates/torrent.html @@ -6,7 +6,7 @@ {% if result.magnetlink %}<p class="altlink"> • {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}</p>{% endif %} {% if result.torrentfile %}<p class="altlink"> • {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}</p>{% endif %} -{% if result.seed %}<p class="stat"> • {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %} +{% if result.seed is defined %}<p class="stat"> • {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %} {%- if result.filesize %}<p class="stat">{{ icon('floppy-disk') }} {{ _('Filesize') }}<span class="badge"> {%- if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }} diff --git a/searx/templates/simple/results.html b/searx/templates/simple/results.html index 770eebe81..8885abc30 100644 --- a/searx/templates/simple/results.html +++ b/searx/templates/simple/results.html @@ -95,13 +95,13 @@ {% for correction in corrections %} <div class="left"> <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" role="navigation"> - <input type="hidden" name="q" value="{{ correction }}"> + <input type="hidden" name="q" value="{{ correction.url }}"> <input type="hidden" name="time_range" value="{{ time_range }}"> <input type="hidden" name="language" value="{{ current_language }}"> <input type="hidden" name="safesearch" value="{{ safesearch }}"> <input type="hidden" name="theme" value="{{ theme }}"> {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit }}" >{% endif %} - <input type="submit" value="{{ correction }}"> + <input type="submit" value="{{ correction.title }}"> </form> </div> {% endfor %} diff --git a/searx/utils.py b/searx/utils.py index d88bc9897..5ea9dc89c 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -13,6 +13,7 @@ from numbers import Number from os.path import splitext, join from io import open from random import choice +from lxml.etree import XPath import sys import json @@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE) useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__)) + "/data/useragents.json", 'r', encoding='utf-8').read()) +xpath_cache = dict() lang_to_lc_cache = dict() @@ -308,14 +310,15 @@ def int_or_zero(num): def is_valid_lang(lang): is_abbr = (len(lang) == 2) + lang = lang.lower().decode('utf-8') if is_abbr: for l in language_codes: - if l[0][:2] == lang.lower(): + if l[0][:2] == lang: return (True, l[0][:2], l[3].lower()) return False else: for l in language_codes: - if l[1].lower() == lang.lower(): + if l[1].lower() == lang or l[3].lower() == lang: return (True, l[0][:2], l[3].lower()) return False @@ -434,3 +437,31 @@ def ecma_unescape(s): # "%20" becomes " ", "%F3" becomes "รณ" s = ecma_unescape2_re.sub(lambda e: unichr(int(e.group(1), 16)), s) return s + + +def get_engine_from_settings(name): + """Return engine configuration from settings.yml of a given engine name""" + + if 'engines' not in settings: + return {} + + for engine in settings['engines']: + if 'name' not in engine: + continue + if name == engine['name']: + return engine + + return {} + + +def get_xpath(xpath_str): + result = xpath_cache.get(xpath_str, None) + if result is None: + result = XPath(xpath_str) + xpath_cache[xpath_str] = result + return result + + +def eval_xpath(element, xpath_str): + xpath = get_xpath(xpath_str) + return xpath(element) diff --git a/searx/webapp.py b/searx/webapp.py index ffe9b4da9..7cf4106d3 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -41,7 +41,10 @@ except: logger.critical("cannot import dependency: pygments") from sys import exit exit(1) -from cgi import escape +try: + from cgi import escape +except: + from html import escape from datetime import datetime, timedelta from time import time from werkzeug.contrib.fixers import ProxyFix @@ -124,6 +127,7 @@ app = Flask( app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True +app.jinja_env.add_extension('jinja2.ext.loopcontrols') app.secret_key = settings['server']['secret_key'] if not searx_debug \ @@ -153,20 +157,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None @babel.localeselector def get_locale(): - locale = request.accept_languages.best_match(settings['locales'].keys()) - - if request.preferences.get_value('locale') != '': - locale = request.preferences.get_value('locale') + if 'locale' in request.form\ + and request.form['locale'] in settings['locales']: + return request.form['locale'] if 'locale' in request.args\ and request.args['locale'] in settings['locales']: - locale = request.args['locale'] + return request.args['locale'] - if 'locale' in request.form\ - and request.form['locale'] in settings['locales']: - locale = request.form['locale'] + if request.preferences.get_value('locale') != '': + return request.preferences.get_value('locale') - return locale + return request.accept_languages.best_match(settings['locales'].keys()) # code-highlighter @@ -538,14 +540,16 @@ def index(): if output_format == 'html': if 'content' in result and result['content']: result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query) - result['title'] = highlight_content(escape(result['title'] or u''), search_query.query) + if 'title' in result and result['title']: + result['title'] = highlight_content(escape(result['title'] or u''), search_query.query) else: if result.get('content'): result['content'] = html_to_text(result['content']).strip() # removing html content and whitespace duplications result['title'] = ' '.join(html_to_text(result['title']).strip().split()) - result['pretty_url'] = prettify_url(result['url']) + if 'url' in result: + result['pretty_url'] = prettify_url(result['url']) # TODO, check if timezone is calculated right if 'publishedDate' in result: @@ -607,6 +611,12 @@ def index(): 'title': suggestion }, result_container.suggestions) + + correction_urls = list(map(lambda correction: { + 'url': raw_text_query.changeSearchQuery(correction).getFullQuery(), + 'title': correction + }, + result_container.corrections)) # return render( 'results.html', @@ -619,7 +629,7 @@ def index(): advanced_search=advanced_search, suggestions=suggestion_urls, answers=result_container.answers, - corrections=result_container.corrections, + corrections=correction_urls, infoboxes=result_container.infoboxes, paging=result_container.paging, unresponsive_engines=result_container.unresponsive_engines, diff --git a/tests/unit/engines/test_deviantart.py b/tests/unit/engines/test_deviantart.py index bd2cf182f..a31151037 100644 --- a/tests/unit/engines/test_deviantart.py +++ b/tests/unit/engines/test_deviantart.py @@ -22,74 +22,3 @@ class TestDeviantartEngine(SearxTestCase): dicto['time_range'] = 'year' params = deviantart.request(query, dicto) self.assertEqual({}, params['url']) - - def test_response(self): - self.assertRaises(AttributeError, deviantart.response, None) - self.assertRaises(AttributeError, deviantart.response, []) - self.assertRaises(AttributeError, deviantart.response, '') - self.assertRaises(AttributeError, deviantart.response, '[]') - - response = mock.Mock(text='<html></html>') - self.assertEqual(deviantart.response(response), []) - - response = mock.Mock(status_code=302) - self.assertEqual(deviantart.response(response), []) - - html = """ - <div id="page-1-results" class="page-results results-page-thumb torpedo-container"> - <span class="thumb wide" href="http://amai911.deviantart.com/art/Horse-195212845" - data-super-full-width="900" data-super-full-height="600"> - <a class="torpedo-thumb-link" href="https://url.of.image"> - <img data-sigil="torpedo-img" src="https://url.of.thumbnail" /> - </a> - <span class="info"><span class="title-wrap"><span class="title">Title of image</span></span> - </div> - """ - response = mock.Mock(text=html) - results = deviantart.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Title of image') - self.assertEqual(results[0]['url'], 'https://url.of.image') - self.assertNotIn('content', results[0]) - self.assertEqual(results[0]['thumbnail_src'], 'https://url.of.thumbnail') - - html = """ - <span class="tt-fh-tc" style="width: 202px;"> - <span class="tt-bb" style="width: 202px;"> - </span> - <span class="shadow"> - <a class="thumb" href="http://url.of.result/2nd.part.of.url" - title="Behoimi BE Animation Test by test-0, Jan 4, - 2010 in Digital Art > Animation"> <i></i> - <img width="200" height="200" alt="Test" - src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg"> - </a> - </span> - <!-- ^TTT --> - </span> - <span class="details"> - <a href="http://test-0.deviantart.com/art/Test" class="t" - title="Behoimi BE Animation Test by test-0, Jan 4, 2010"> - <span class="tt-fh-oe">Title of image</span> </a> - <small> - <span class="category"> - <span class="age"> - 5 years ago - </span> - in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010" - href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a> - </span> - <div class="commentcount"> - <a href="http://test-0.deviantart.com/art/Test#comments"> - <span class="iconcommentsstats"></span>9 Comments</a> - </div> - <a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425"> - <span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a> - </span> - </small> <!-- TTT$ --> - """ - response = mock.Mock(text=html) - results = deviantart.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_digg.py b/tests/unit/engines/test_digg.py index 6e7c9cc99..8bc4c67c2 100644 --- a/tests/unit/engines/test_digg.py +++ b/tests/unit/engines/test_digg.py @@ -14,88 +14,3 @@ class TestDiggEngine(SearxTestCase): self.assertIn('url', params) self.assertIn(query, params['url']) self.assertIn('digg.com', params['url']) - - def test_response(self): - self.assertRaises(AttributeError, digg.response, None) - self.assertRaises(AttributeError, digg.response, []) - self.assertRaises(AttributeError, digg.response, '') - self.assertRaises(AttributeError, digg.response, '[]') - - response = mock.Mock(text='{}') - self.assertEqual(digg.response(response), []) - - response = mock.Mock(text='{"data": []}') - self.assertEqual(digg.response(response), []) - - json = """ - { - "status": "ok", - "num": 10, - "next_position": 20, - "html": "<article itemscope itemtype=\\"http://schema.org/Article\\" - class=\\"story-container digg-story-el hentry entry story-1sRANah col-1\\" - data-content-id=\\"1sRANah\\" data-contenturl=\\"http://url.of.link\\" - data-position=\\"0\\" data-diggs=\\"24\\" data-tweets=\\"69\\" - data-digg-score=\\"1190\\"> <div class=\\"story-image story-image-thumb\\"> - <a data-position=\\"0\\" data-content-id=\\"1sRANah\\" - class=\\"story-link\\" href=\\"http://www.thedailybeast.com/\\" - target=\\"_blank\\"><img class=\\"story-image-img\\" - src=\\"http://url.of.image.jpeg\\" width=\\"312\\" height=\\"170\\" - alt=\\"\\" /> </a> </div> <div class=\\"story-content\\"><header - class=\\"story-header\\"> <div itemprop=\\"alternativeHeadline\\" - class=\\"story-kicker\\" >Kicker</div> <h2 itemprop=\\"headline\\" - class=\\"story-title entry-title\\"><a class=\\"story-title-link story-link\\" - rel=\\"bookmark\\" itemprop=\\"url\\" href=\\"http://www.thedailybeast.com/\\" - target=\\"_blank\\">Title of article</h2> <div class=\\"story-meta\\"> - <div class=\\"story-score \\"> - <div class=\\"story-score-diggscore diggscore-1sRANah\\">1190</div> - <div class=\\"story-score-details\\"> <div class=\\"arrow\\"></div> - <ul class=\\"story-score-details-list\\"> <li - class=\\"story-score-detail story-score-diggs\\"><span - class=\\"label\\">Diggs:</span> <span class=\\"count diggs-1sRANah\\">24</span> - </li> <li class=\\"story-score-detail story-score-twitter\\"><span - class=\\"label\\">Tweets:</span> <span class=\\"count tweets-1sRANah\\">69</span> - </li> <li class=\\"story-score-detail story-score-facebook\\"><span - class=\\"label\\">Facebook Shares:</span> <span - class=\\"count fb_shares-1sRANah\\">1097</span></li> </ul> </div> </div> - <span class=\\"story-meta-item story-source\\"> <a - itemprop=\\"publisher copyrightHolder sourceOrganization provider\\" - class=\\"story-meta-item-link story-source-link\\" - href=\\"/source/thedailybeast.com\\">The Daily Beast </a> </span> - <span class=\\"story-meta-item story-tag first-tag\\"> <a - itemprop=\\"keywords\\" rel=\\"tag\\" - class=\\"story-meta-item-link story-tag-link\\" href=\\"/tag/news\\">News</a> - </span> <abbr class=\\"published story-meta-item story-timestamp\\" - title=\\"2014-10-18 14:53:45\\"> <time datetime=\\"2014-10-18 14:53:45\\">18 Oct 2014</time> - </abbr> </div> </header> </div> <ul class=\\"story-actions\\"> <li - class=\\"story-action story-action-digg btn-story-action-container\\"> - <a class=\\"target digg-1sRANah\\" href=\\"#\\">Digg</a></li> <li - class=\\"story-action story-action-save btn-story-action-container\\"> - <a class=\\"target save-1sRANah\\" href=\\"#\\">Save</a></li> <li - class=\\"story-action story-action-share\\"><a - class=\\"target share-facebook\\" href=\\"https://www.facebook.com/\\">Facebook</a></li> - <li class=\\"story-action story-action-share\\"><a class=\\"target share-twitter\\" - href=\\"https://twitter.com/\\">Twitter</a></li> </ul> </article>" - } - """ - json = json.replace('\r\n', '').replace('\n', '').replace('\r', '') - response = mock.Mock(text=json) - results = digg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Title of article') - self.assertEqual(results[0]['url'], 'http://url.of.link') - self.assertEqual(results[0]['thumbnail'], 'http://url.of.image.jpeg') - self.assertEqual(results[0]['content'], '') - - json = """ - { - "status": "error", - "num": 10, - "next_position": 20 - } - """ - response = mock.Mock(text=json) - results = digg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py new file mode 100644 index 000000000..2057c1cb1 --- /dev/null +++ b/tests/unit/engines/test_seedpeer.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import seedpeer +from searx.testing import SearxTestCase + + +class TestBtdiggEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = seedpeer.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('seedpeer', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, seedpeer.response, None) + self.assertRaises(AttributeError, seedpeer.response, []) + self.assertRaises(AttributeError, seedpeer.response, '') + self.assertRaises(AttributeError, seedpeer.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(seedpeer.response(response), []) + + html = u""" + <html> + <head> + <script></script> + <script type="text/javascript" src="not_here.js"></script> + <script type="text/javascript"> + window.initialData= + {"data": {"list": [{"name": "Title", "seeds": "10", "peers": "20", "size": "1024", "hash": "abc123"}]}} + </script> + </head> + <body> + <table></table> + <table> + <thead><tr></tr></thead> + <tbody> + <tr> + <td><a href="link">Title</a></td> + <td>1 year</td> + <td>1 KB</td> + <td>10</td> + <td>20</td> + <td></td> + </tr> + </tbody> + </table> + </body> + </html> + """ + response = mock.Mock(text=html) + results = seedpeer.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://seedpeer.me/link') + self.assertEqual(results[0]['seed'], 10) + self.assertEqual(results[0]['leech'], 20) + self.assertEqual(results[0]['filesize'], 1024) + self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123') + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123') diff --git a/tests/unit/engines/test_startpage.py b/tests/unit/engines/test_startpage.py index a4704ce22..ac4454738 100644 --- a/tests/unit/engines/test_startpage.py +++ b/tests/unit/engines/test_startpage.py @@ -18,12 +18,9 @@ class TestStartpageEngine(SearxTestCase): self.assertIn('data', params) self.assertIn('query', params['data']) self.assertIn(query, params['data']['query']) - self.assertIn('with_language', params['data']) - self.assertIn('lang_fr', params['data']['with_language']) dicto['language'] = 'all' params = startpage.request(query, dicto) - self.assertNotIn('with_language', params['data']) def test_response(self): self.assertRaises(AttributeError, startpage.response, None) @@ -35,33 +32,32 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(startpage.response(response), []) html = """ - <li class="search-result search-item"> - <h3> - <a href='http://this.should.be.the.link/' id='title_2' name='title_2' > - This should be the title +<div class="w-gl__result"> + <a + class="w-gl__result-title" + href="http://this.should.be.the.link/" + data-onw="1" + rel="noopener noreferrer" + target="_blank"> + + <h3>This should be the title</h3> </a> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p class="search-item__body"> - This should be the content. - </p> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - - - <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata=" - class='proxy'> - Navigation avec Ixquick Proxy - </A> - - - <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid= - &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0= - &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'> - Mis en surbrillance - </A> - </p> - </li> - """ + <div class="w-gl__result-second-line-container"> + <div class="w-gl__result-url-container"> + <a + class="w-gl__result-url" + href="http://this.should.be.the.link/" + rel="noopener noreferrer" + target="_blank">https://www.cnbc.com/2019/10/12/dj-zedd-banned-in-china-for-liking-a-south-park-tweet.html</a> + </div> + <a + class="w-gl__anonymous-view-url" + href="https://eu-browse.startpage.com/do/proxy?ep=556b554d576b6f5054554546423167764b5445616455554d5342675441774659495246304848774f5267385453304941486b5949546c63704e33774f526b705544565647516d4a61554246304847674f4a556f6957415a4f436b455042426b6b4f7a64535a52784a56514a4f45307743446c567250445a4f4c52514e5677554e46776b4b545563704c7931554c5167465467644f42464d4f4255426f4d693152624634525741305845526c595746636b626d67494e42705743466c515252634f4267456e597a7346596b7856435134465345634f564249794b5752785643315863546769515773764a5163494c5877505246315865456f5141426b4f41774167596d6c5a4e30395758773442465251495677596c624770665a6b786344466b4151455663425249794d6a78525a55554157516f4342556766526b51314b57514e&ek=4q58686o5047786n6343527259445247576p6o38&ekdata=84abd523dc13cba5c65164d04d7d7263" + target="_blank">Anonymous View</a> + </div> + <p class="w-gl__description">This should be the content.</p> + </div> + """ # noqa response = mock.Mock(text=html.encode('utf-8')) results = startpage.response(response) self.assertEqual(type(results), list) @@ -69,72 +65,3 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') - - html = """ - <li class="search-result search-item"> - <h3> - <a href='http://www.google.com/aclk?sa=l&ai=C' id='title_2' name='title_2' > - This should be the title - </a> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p class="search-item__body"> - This should be the content. - </p> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - - - <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata=" - class='proxy'> - Navigation avec Ixquick Proxy - </A> - - - <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid= - &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0= - &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'> - Mis en surbrillance - </A> - </p> - </li> - <li class="search-result search-item"> - <h3> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p class="search-item__body"> - This should be the content. - </p> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - </p> - </li> - <li class="search-result search-item"> - <h3> - <a href='http://this.should.be.the.link/' id='title_2' name='title_2' > - This should be the title - </a> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - - - <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata=" - class='proxy'> - Navigation avec Ixquick Proxy - </A> - - - <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid= - &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0= - &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'> - Mis en surbrillance - </A> - </p> - </li> - """ - response = mock.Mock(text=html.encode('utf-8')) - results = startpage.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['content'], '') diff --git a/tests/unit/engines/test_www1x.py b/tests/unit/engines/test_www1x.py index 9df8de6bf..40f5200fd 100644 --- a/tests/unit/engines/test_www1x.py +++ b/tests/unit/engines/test_www1x.py @@ -12,46 +12,3 @@ class TestWww1xEngine(SearxTestCase): self.assertTrue('url' in params) self.assertTrue(query in params['url']) self.assertTrue('1x.com' in params['url']) - - def test_response(self): - self.assertRaises(AttributeError, www1x.response, None) - self.assertRaises(AttributeError, www1x.response, []) - self.assertRaises(AttributeError, www1x.response, '') - self.assertRaises(AttributeError, www1x.response, '[]') - - response = mock.Mock(text='<html></html>') - self.assertEqual(www1x.response(response), []) - html = """ - <?xml version="1.0" encoding="UTF-8"?><!DOCTYPE characters - [ - <!ELEMENT characters (character*) > - <!ELEMENT character (#PCDATA ) > - - <!ENTITY iexcl "¡" > - <!ENTITY cent "¢" > - <!ENTITY pound "£" > - ] - ><root><searchresult><![CDATA[<table border="0" cellpadding="0" cellspacing="0" width="100%"> - <tr> - <td style="min-width: 220px;" valign="top"> - <div style="font-size: 30px; margin: 0px 0px 20px 0px;">Photos</div> - <div> - <a href="/photo/123456" class="dynamiclink"> -<img border="0" class="searchresult" src="/images/user/testimage-123456.jpg" style="width: 125px; height: 120px;"> - </a> - <a title="sjoerd lammers street photography" href="/member/sjoerdlammers" class="dynamiclink"> -<img border="0" class="searchresult" src="/images/profile/60c48b394c677d2fa4d9e7d263aabf44-square.jpg"> - </a> - </div> - </td> - </table> - ]]></searchresult></root> - """ - response = mock.Mock(text=html) - results = www1x.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['url'], 'https://1x.com/photo/123456') - self.assertEqual(results[0]['thumbnail_src'], 'https://1x.com/images/user/testimage-123456.jpg') - self.assertEqual(results[0]['content'], '') - self.assertEqual(results[0]['template'], 'images.html') |