diff options
33 files changed, 868 insertions, 140 deletions
@@ -16,7 +16,10 @@ update_dev_packages() { pep8_check() { echo '[!] Running pep8 check' - pep8 --max-line-length=120 "$SEARX_DIR" "$BASE_DIR/tests" + # ignored rules: + # E402 module level import not at top of file + # W503 line break before binary operator + pep8 --max-line-length=120 --ignore "E402,W503" "$SEARX_DIR" "$BASE_DIR/tests" } unit_tests() { diff --git a/requirements-dev.txt b/requirements-dev.txt index d9ec779b3..38be888e0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ babel==2.2.0 -flake8==2.5.1 mock==1.0.1 nose2[coverage-plugin] +pep8==1.7.0 plone.testing==4.0.15 robotframework-selenium2library==1.7.4 robotsuite==1.7.0 diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 264d0cc1f..d92dc4246 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -114,8 +114,7 @@ def dbpedia(query): # dbpedia autocompleter, no HTTPS autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' - response = get(autocomplete_url - + urlencode(dict(QueryString=query))) + response = get(autocomplete_url + urlencode(dict(QueryString=query))) results = [] @@ -141,8 +140,7 @@ def google(query): # google autocompleter autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&' - response = get(autocomplete_url - + urlencode(dict(q=query))) + response = get(autocomplete_url + urlencode(dict(q=query))) results = [] diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index 06850dfe1..2664b795f 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -17,7 +17,7 @@ from urllib import urlencode from lxml import html -from yaml import load +from json import loads import re # engine dependent config @@ -36,6 +36,9 @@ safesearch_types = {2: 'STRICT', 0: 'OFF'} +_quote_keys_regex = re.compile('({|,)([a-z][a-z0-9]*):(")', re.I | re.U) + + # do search-request def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 @@ -65,22 +68,19 @@ def response(resp): dom = html.fromstring(resp.text) - # init regex for yaml-parsing - p = re.compile('({|,)([a-z]+):(")') - # parse results for result in dom.xpath('//div[@class="dg_u"]'): link = result.xpath('./a')[0] - # parse yaml-data (it is required to add a space, to make it parsable) - yaml_data = load(p.sub(r'\1\2: \3', link.attrib.get('m'))) + # parse json-data (it is required to add a space, to make it parsable) + json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('m'))) title = link.attrib.get('t1') ihk = link.attrib.get('ihk') # url = 'http://' + link.attrib.get('t3') - url = yaml_data.get('surl') - img_src = yaml_data.get('imgurl') + url = json_data.get('surl') + img_src = json_data.get('imgurl') # append result results.append({'template': 'images.html', diff --git a/searx/engines/blekko_images.py b/searx/engines/blekko_images.py index 93ac6616b..c0664f390 100644 --- a/searx/engines/blekko_images.py +++ b/searx/engines/blekko_images.py @@ -37,7 +37,7 @@ def request(query, params): c=c) if params['pageno'] != 1: - params['url'] += '&page={pageno}'.format(pageno=(params['pageno']-1)) + params['url'] += '&page={pageno}'.format(pageno=(params['pageno'] - 1)) # let Blekko know we wan't have profiling params['cookies']['tag_lesslogging'] = '1' diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index 192ed6ee9..c2b22f003 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -29,7 +29,7 @@ search_url = url + '/search?q={search_term}&p={pageno}' # do search-request def request(query, params): params['url'] = search_url.format(search_term=quote(query), - pageno=params['pageno']-1) + pageno=params['pageno'] - 1) return params diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py index 60c8d7ea7..135aeb324 100644 --- a/searx/engines/deviantart.py +++ b/searx/engines/deviantart.py @@ -24,7 +24,7 @@ paging = True # search-url base_url = 'https://www.deviantart.com/' -search_url = base_url+'browse/all/?offset={offset}&{query}' +search_url = base_url + 'browse/all/?offset={offset}&{query}' # do search-request diff --git a/searx/engines/digg.py b/searx/engines/digg.py index 000f66ba2..a10b38bb6 100644 --- a/searx/engines/digg.py +++ b/searx/engines/digg.py @@ -22,7 +22,7 @@ paging = True # search-url base_url = 'https://digg.com/' -search_url = base_url+'api/search/{query}.json?position={position}&format=html' +search_url = base_url + 'api/search/{query}.json?position={position}&format=html' # specific xpath variables results_xpath = '//article' diff --git a/searx/engines/faroo.py b/searx/engines/faroo.py index 43df14eef..9fa244e77 100644 --- a/searx/engines/faroo.py +++ b/searx/engines/faroo.py @@ -88,7 +88,7 @@ def response(resp): for result in search_res['results']: if result['news']: # timestamp (milliseconds since 1970) - publishedDate = datetime.datetime.fromtimestamp(result['date']/1000.0) # noqa + publishedDate = datetime.datetime.fromtimestamp(result['date'] / 1000.0) # noqa # append news result results.append({'url': result['url'], diff --git a/searx/engines/frinkiac.py b/searx/engines/frinkiac.py new file mode 100644 index 000000000..a9383f862 --- /dev/null +++ b/searx/engines/frinkiac.py @@ -0,0 +1,44 @@ +""" +Frinkiac (Images) + +@website https://www.frinkiac.com +@provide-api no +@using-api no +@results JSON +@stable no +@parse url, title, img_src +""" + +from json import loads +from urllib import urlencode + +categories = ['images'] + +BASE = 'https://frinkiac.com/' +SEARCH_URL = '{base}api/search?{query}' +RESULT_URL = '{base}?{query}' +THUMB_URL = '{base}img/{episode}/{timestamp}/medium.jpg' +IMAGE_URL = '{base}img/{episode}/{timestamp}.jpg' + + +def request(query, params): + params['url'] = SEARCH_URL.format(base=BASE, query=urlencode({'q': query})) + return params + + +def response(resp): + results = [] + response_data = loads(resp.text) + for result in response_data: + episode = result['Episode'] + timestamp = result['Timestamp'] + + results.append({'template': 'images.html', + 'url': RESULT_URL.format(base=BASE, + query=urlencode({'p': 'caption', 'e': episode, 't': timestamp})), + 'title': episode, + 'content': '', + 'thumbnail_src': THUMB_URL.format(base=BASE, episode=episode, timestamp=timestamp), + 'img_src': IMAGE_URL.format(base=BASE, episode=episode, timestamp=timestamp)}) + + return results diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index bdc9601ac..1cc243104 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -10,11 +10,11 @@ @parse url, title, content """ -from urllib import urlencode from cgi import escape -from lxml import etree +from json import loads from random import randint from time import time +from urllib import urlencode # engine dependent config categories = ['general'] @@ -27,11 +27,11 @@ safesearch = True base_url = 'https://gigablast.com/' search_string = 'search?{query}'\ '&n={number_of_results}'\ + '&c=main'\ '&s={offset}'\ - '&format=xml'\ + '&format=json'\ '&qh=0'\ - '&rxiyd={rxiyd}'\ - '&rand={rand}'\ + '&rxiwd={rxiwd}'\ '&qlang={lang}'\ '&ff={safesearch}' @@ -59,8 +59,8 @@ def request(query, params): search_path = search_string.format(query=urlencode({'q': query}), offset=offset, number_of_results=number_of_results, - rxiyd=randint(10000, 10000000), - rand=int(time()), + rxiwd=1, + # rand=int(time()), lang=language, safesearch=safesearch) @@ -73,18 +73,14 @@ def request(query, params): def response(resp): results = [] - dom = etree.fromstring(resp.content) - # parse results - for result in dom.xpath(results_xpath): - url = result.xpath(url_xpath)[0].text - title = result.xpath(title_xpath)[0].text - content = escape(result.xpath(content_xpath)[0].text) + response_json = loads(resp.text) + for result in response_json['results']: # append result - results.append({'url': url, - 'title': title, - 'content': content}) + results.append({'url': result['url'], + 'title': escape(result['title']), + 'content': escape(result['sum'])}) # return results return results diff --git a/searx/engines/google.py b/searx/engines/google.py index e6dacc3a8..dbca205a1 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -209,29 +209,29 @@ def response(resp): parsed_url = urlparse(url, google_hostname) # map result - if ((parsed_url.netloc == google_hostname and parsed_url.path.startswith(maps_path)) - or (parsed_url.netloc.startswith(map_hostname_start))): - x = result.xpath(map_near) - if len(x) > 0: - # map : near the location - results = results + parse_map_near(parsed_url, x, google_hostname) - else: - # map : detail about a location - results = results + parse_map_detail(parsed_url, result, google_hostname) - - # google news - elif (parsed_url.netloc == google_hostname - and parsed_url.path == search_path): - # skipping news results - pass - - # images result - elif (parsed_url.netloc == google_hostname - and parsed_url.path == images_path): - # only thumbnail image provided, - # so skipping image results - # results = results + parse_images(result, google_hostname) - pass + if parsed_url.netloc == google_hostname: + # TODO fix inside links + continue + # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start): + # print "yooooo"*30 + # x = result.xpath(map_near) + # if len(x) > 0: + # # map : near the location + # results = results + parse_map_near(parsed_url, x, google_hostname) + # else: + # # map : detail about a location + # results = results + parse_map_detail(parsed_url, result, google_hostname) + # # google news + # elif parsed_url.path == search_path: + # # skipping news results + # pass + + # # images result + # elif parsed_url.path == images_path: + # # only thumbnail image provided, + # # so skipping image results + # # results = results + parse_images(result, google_hostname) + # pass else: # normal result diff --git a/searx/engines/searchcode_code.py b/searx/engines/searchcode_code.py index bd5eb71d2..de8cd43be 100644 --- a/searx/engines/searchcode_code.py +++ b/searx/engines/searchcode_code.py @@ -20,7 +20,7 @@ paging = True # search-url url = 'https://searchcode.com/' -search_url = url+'api/codesearch_I/?{query}&p={pageno}' +search_url = url + 'api/codesearch_I/?{query}&p={pageno}' # special code-endings which are not recognised by the file ending code_endings = {'cs': 'c#', @@ -32,7 +32,7 @@ code_endings = {'cs': 'c#', # do search-request def request(query, params): params['url'] = search_url.format(query=urlencode({'q': query}), - pageno=params['pageno']-1) + pageno=params['pageno'] - 1) # Disable SSL verification # error: (60) SSL certificate problem: unable to get local issuer diff --git a/searx/engines/searchcode_doc.py b/searx/engines/searchcode_doc.py index 9453f31a4..f24fe6f90 100644 --- a/searx/engines/searchcode_doc.py +++ b/searx/engines/searchcode_doc.py @@ -19,13 +19,13 @@ paging = True # search-url url = 'https://searchcode.com/' -search_url = url+'api/search_IV/?{query}&p={pageno}' +search_url = url + 'api/search_IV/?{query}&p={pageno}' # do search-request def request(query, params): params['url'] = search_url.format(query=urlencode({'q': query}), - pageno=params['pageno']-1) + pageno=params['pageno'] - 1) # Disable SSL verification # error: (60) SSL certificate problem: unable to get local issuer diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py index 34ecabae7..fdd3711a9 100644 --- a/searx/engines/stackoverflow.py +++ b/searx/engines/stackoverflow.py @@ -22,7 +22,7 @@ paging = True # search-url url = 'https://stackoverflow.com/' -search_url = url+'search?{query}&page={pageno}' +search_url = url + 'search?{query}&page={pageno}' # specific xpath variables results_xpath = '//div[contains(@class,"question-summary")]' diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index a91cafa00..52dd0b92f 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -90,8 +90,8 @@ def response(resp): # check if search result starts with something like: "2 Sep 2014 ... " if re.match("^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]{2} [0-9]{4} \.\.\. ", content): - date_pos = content.find('...')+4 - date_string = content[0:date_pos-5] + date_pos = content.find('...') + 4 + date_string = content[0:date_pos - 5] published_date = parser.parse(date_string, dayfirst=True) # fix content string @@ -99,8 +99,8 @@ def response(resp): # check if search result starts with something like: "5 days ago ... " elif re.match("^[0-9]+ days? ago \.\.\. ", content): - date_pos = content.find('...')+4 - date_string = content[0:date_pos-5] + date_pos = content.find('...') + 4 + date_string = content[0:date_pos - 5] # calculate datetime published_date = datetime.now() - timedelta(days=int(re.match(r'\d+', date_string).group())) diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py index 2d31264ca..864436a52 100644 --- a/searx/engines/swisscows.py +++ b/searx/engines/swisscows.py @@ -10,6 +10,7 @@ @parse url, title, content """ +from cgi import escape from json import loads from urllib import urlencode, unquote import re @@ -77,7 +78,7 @@ def response(resp): # append result results.append({'url': result['SourceUrl'], - 'title': result['Title'], + 'title': escape(result['Title']), 'content': '', 'img_src': img_url, 'template': 'images.html'}) @@ -89,8 +90,8 @@ def response(resp): # append result results.append({'url': result_url, - 'title': result_title, - 'content': result_content}) + 'title': escape(result_title), + 'content': escape(result_content)}) # parse images for result in json.get('Images', []): @@ -99,7 +100,7 @@ def response(resp): # append result results.append({'url': result['SourceUrl'], - 'title': result['Title'], + 'title': escape(result['Title']), 'content': '', 'img_src': img_url, 'template': 'images.html'}) diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py index fc840d47c..9f3496b72 100644 --- a/searx/engines/wikidata.py +++ b/searx/engines/wikidata.py @@ -295,7 +295,7 @@ def get_geolink(claims, propertyName, defaultValue=''): if precision < 0.0003: zoom = 19 else: - zoom = int(15 - precision*8.8322 + precision*precision*0.625447) + zoom = int(15 - precision * 8.8322 + precision * precision * 0.625447) url = url_map\ .replace('{latitude}', str(value.get('latitude', 0)))\ @@ -318,6 +318,6 @@ def get_wikilink(result, wikiid): def get_wiki_firstlanguage(result, wikipatternid): for k in result.get('sitelinks', {}).keys(): - if k.endswith(wikipatternid) and len(k) == (2+len(wikipatternid)): + if k.endswith(wikipatternid) and len(k) == (2 + len(wikipatternid)): return k[0:2] return None diff --git a/searx/engines/wolframalpha_api.py b/searx/engines/wolframalpha_api.py index d61d25747..303c6c165 100644 --- a/searx/engines/wolframalpha_api.py +++ b/searx/engines/wolframalpha_api.py @@ -10,11 +10,18 @@ from urllib import urlencode from lxml import etree +from re import search # search-url base_url = 'http://api.wolframalpha.com/v2/query' search_url = base_url + '?appid={api_key}&{query}&format=plaintext' -api_key = '' +site_url = 'http://www.wolframalpha.com/input/?{query}' +api_key = '' # defined in settings.yml + +# xpath variables +failure_xpath = '/queryresult[attribute::success="false"]' +answer_xpath = '//pod[attribute::primary="true"]/subpod/plaintext' +input_xpath = '//pod[starts-with(attribute::title, "Input")]/subpod/plaintext' # do search-request @@ -45,16 +52,26 @@ def response(resp): search_results = etree.XML(resp.content) # return empty array if there are no results - if search_results.xpath('/queryresult[attribute::success="false"]'): + if search_results.xpath(failure_xpath): return [] - # parse result - result = search_results.xpath('//pod[attribute::primary="true"]/subpod/plaintext')[0].text - result = replace_pua_chars(result) + # parse answers + answers = search_results.xpath(answer_xpath) + if answers: + for answer in answers: + answer = replace_pua_chars(answer.text) + + results.append({'answer': answer}) + + # if there's no input section in search_results, check if answer has the input embedded (before their "=" sign) + try: + query_input = search_results.xpath(input_xpath)[0].text + except IndexError: + query_input = search(u'([^\uf7d9]+)', answers[0].text).group(1) - # append result - # TODO: shouldn't it bind the source too? - results.append({'answer': result}) + # append link to site + result_url = site_url.format(query=urlencode({'i': query_input.encode('utf-8')})) + results.append({'url': result_url, + 'title': query_input + " - Wolfram|Alpha"}) - # return results return results diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py new file mode 100644 index 000000000..291fee04d --- /dev/null +++ b/searx/engines/wolframalpha_noapi.py @@ -0,0 +1,86 @@ +# WolframAlpha (Maths) +# +# @website http://www.wolframalpha.com/ +# @provide-api yes (http://api.wolframalpha.com/v2/) +# +# @using-api no +# @results HTML +# @stable no +# @parse answer + +from re import search, sub +from json import loads +from urllib import urlencode +from lxml import html +import HTMLParser + +# search-url +url = 'http://www.wolframalpha.com/' +search_url = url + 'input/?{query}' + +# xpath variables +scripts_xpath = '//script' +title_xpath = '//title' +failure_xpath = '//p[attribute::class="pfail"]' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(query=urlencode({'i': query})) + + return params + + +# get response from search-request +def response(resp): + results = [] + line = None + + dom = html.fromstring(resp.text) + scripts = dom.xpath(scripts_xpath) + + # the answer is inside a js function + # answer can be located in different 'pods', although by default it should be in pod_0200 + possible_locations = ['pod_0200\.push\((.*)', + 'pod_0100\.push\((.*)'] + + # failed result + if dom.xpath(failure_xpath): + return results + + # get line that matches the pattern + for pattern in possible_locations: + for script in scripts: + try: + line = search(pattern, script.text_content()).group(1) + break + except AttributeError: + continue + if line: + break + + if line: + # extract answer from json + answer = line[line.find('{'):line.rfind('}') + 1] + try: + answer = loads(answer) + except Exception: + answer = loads(answer.encode('unicode-escape')) + answer = answer['stringified'] + + # clean plaintext answer + h = HTMLParser.HTMLParser() + answer = h.unescape(answer.decode('unicode-escape')) + answer = sub(r'\\', '', answer) + + results.append({'answer': answer}) + + # user input is in first part of title + title = dom.xpath(title_xpath)[0].text.encode('utf-8') + result_url = request(title[:-16], {})['url'] + + # append result + results.append({'url': result_url, + 'title': title.decode('utf-8')}) + + return results diff --git a/searx/engines/www1x.py b/searx/engines/www1x.py index ddb79bfea..1269a5422 100644 --- a/searx/engines/www1x.py +++ b/searx/engines/www1x.py @@ -22,7 +22,7 @@ paging = False # search-url base_url = 'https://1x.com' -search_url = base_url+'/backend/search.php?{query}' +search_url = base_url + '/backend/search.php?{query}' # do search-request diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index 1a599dc0a..f51634be0 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -43,7 +43,7 @@ def extract_url(xpath_results, search_url): if url.startswith('//'): # add http or https to this kind of url //example.com/ parsed_search_url = urlparse(search_url) - url = parsed_search_url.scheme+url + url = parsed_search_url.scheme + url elif url.startswith('/'): # fix relative url to the search engine url = urljoin(search_url, url) @@ -69,7 +69,7 @@ def normalize_url(url): p = parsed_url.path mark = p.find('/**') if mark != -1: - return unquote(p[mark+3:]).decode('utf-8') + return unquote(p[mark + 3:]).decode('utf-8') return url diff --git a/searx/engines/yandex.py b/searx/engines/yandex.py index edc6ad5f2..be3ec36ce 100644 --- a/searx/engines/yandex.py +++ b/searx/engines/yandex.py @@ -9,6 +9,7 @@ @parse url, title, content """ +from cgi import escape from urllib import urlencode from lxml import html from searx.search import logger @@ -38,7 +39,7 @@ content_xpath = './/div[@class="serp-item__text"]//text()' def request(query, params): lang = params['language'].split('_')[0] host = base_url.format(tld=language_map.get(lang) or default_tld) - params['url'] = host + search_url.format(page=params['pageno']-1, + params['url'] = host + search_url.format(page=params['pageno'] - 1, query=urlencode({'text': query})) return params @@ -51,8 +52,8 @@ def response(resp): for result in dom.xpath(results_xpath): try: res = {'url': result.xpath(url_xpath)[0], - 'title': ''.join(result.xpath(title_xpath)), - 'content': ''.join(result.xpath(content_xpath))} + 'title': escape(''.join(result.xpath(title_xpath))), + 'content': escape(''.join(result.xpath(content_xpath)))} except: logger.exception('yandex parse crash') continue diff --git a/searx/languages.py b/searx/languages.py index df5fabf74..b67da9d22 100644 --- a/searx/languages.py +++ b/searx/languages.py @@ -58,6 +58,7 @@ language_codes = ( ("ko_KR", "Korean", "Korea"), ("lt_LT", "Lithuanian", "Lithuania"), ("lv_LV", "Latvian", "Latvia"), + ("oc_OC", "Occitan", "Occitan"), ("nb_NO", "Norwegian", "Norway"), ("nl_BE", "Dutch", "Belgium"), ("nl_NL", "Dutch", "Netherlands"), diff --git a/searx/plugins/https_rewrite.py b/searx/plugins/https_rewrite.py index a24f15a28..0a58cc85d 100644 --- a/searx/plugins/https_rewrite.py +++ b/searx/plugins/https_rewrite.py @@ -103,10 +103,10 @@ def load_single_https_ruleset(rules_path): # into a valid python regex group rule_from = ruleset.attrib['from'].replace('$', '\\') if rule_from.endswith('\\'): - rule_from = rule_from[:-1]+'$' + rule_from = rule_from[:-1] + '$' rule_to = ruleset.attrib['to'].replace('$', '\\') if rule_to.endswith('\\'): - rule_to = rule_to[:-1]+'$' + rule_to = rule_to[:-1] + '$' # TODO, not working yet because of the hack above, # currently doing that in webapp.py diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 4761f6ae8..13c6a906e 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -92,7 +92,7 @@ def head(url, **kwargs): return request('head', url, **kwargs) -def post(url, data=None, **kwargs): +def post(url, data=None, **kwargs): return request('post', url, data=data, **kwargs) diff --git a/searx/settings.yml b/searx/settings.yml index e23e4c390..7e8182b0f 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -110,6 +110,11 @@ engines: # Or you can use the html non-stable engine, activated by default engine : flickr_noapi + - name : frinkiac + engine : frinkiac + shortcut : frk + disabled : True + - name : gigablast engine : gigablast shortcut : gb @@ -300,13 +305,15 @@ engines: engine : vimeo shortcut : vm -# You can use the engine using the official stable API, but you need an API key -# See : http://products.wolframalpha.com/api/ -# - name : wolframalpha -# shortcut : wa -# engine : wolframalpha_api -# api_key: 'apikey' # required! -# timeout: 6.0 + - name : wolframalpha + shortcut : wa + # You can use the engine using the official stable API, but you need an API key + # See : http://products.wolframalpha.com/api/ + # engine : wolframalpha_api + # api_key: 'apikey' # required! + engine : wolframalpha_noapi + timeout: 6.0 + categories : science #The blekko technology and team have joined IBM Watson! -> https://blekko.com/ # - name : blekko images diff --git a/searx/webapp.py b/searx/webapp.py index 794b7ea8c..71d65aa82 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -119,7 +119,8 @@ _category_names = (gettext('files'), gettext('videos'), gettext('it'), gettext('news'), - gettext('map')) + gettext('map'), + gettext('science')) outgoing_proxies = settings['outgoing'].get('proxies', None) diff --git a/tests/unit/engines/test_bing_images.py b/tests/unit/engines/test_bing_images.py index f42dff7e8..88538d8fa 100644 --- a/tests/unit/engines/test_bing_images.py +++ b/tests/unit/engines/test_bing_images.py @@ -39,28 +39,25 @@ class TestBingImagesEngine(SearxTestCase): html = """ <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"> - <a href="#" ihk="HN.608003696942779811" - m="{ns:"images",k:"5045", -mid:"659EB92C317974F34517A1CCAEBEF76A578E08DEE", -surl:"http://www.page.url/",imgurl:"http://test.url/Test%20Query.jpg", -oh:"238",tft:"0",oi:"http://www.image.url/Images/Test%20Query.jpg"}" - mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;" - t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1"> - <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&o=4&pid=1.7" - style="height:144px;" width="178" height="144"/> + <a href="/images/search?q=south&view=detailv2&&id=7E92863981CCFB89FBDD55205C742DFDA3290CF6&selectedIndex=9&ccid=vzvIfv5u&simid=608055786735667000&thid=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0" ihk="OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0" t1="South Carolina" t2="747 x 589 · 29 kB · gif" t3="www.digital-topo-maps.com/county-map/south-carolina.shtml" hh="236" hw="300" m='{ns:"images",k:"5117",mid:"7E92863981CCFB89FBDD55205C742DFDA3290CF6",md5:"bf3bc87efe6e0e476be8cc34bf6cd80e",surl:"http://www.digital-topo-maps.com/county-map/south-carolina.shtml",imgurl:"http://www.digital-topo-maps.com/county-map/south-carolina-county-map.gif",tid:"OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0",ow:"480",docid:"608055786735667000",oh:"378",tft:"45"}' mid="7E92863981CCFB89FBDD55205C742DFDA3290CF6" h="ID=images,5117.1"> + <img class="img_hid" src2="https://tse4.mm.bing.net/th?id=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0&w=210&h=154&c=7&rs=1&qlt=90&o=4&pid=1.1" style="width:210px;height:154px;" width="210" height="154"> </a> + </div> - """ + """ # noqa html = html.replace('\r\n', '').replace('\n', '').replace('\r', '') response = mock.Mock(text=html) results = bing_images.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Test Query') - self.assertEqual(results[0]['url'], 'http://www.page.url/') + self.assertEqual(results[0]['title'], 'South Carolina') + self.assertEqual(results[0]['url'], + 'http://www.digital-topo-maps.com/county-map/south-carolina.shtml') self.assertEqual(results[0]['content'], '') - self.assertEqual(results[0]['thumbnail_src'], 'https://www.bing.com/th?id=HN.608003696942779811') - self.assertEqual(results[0]['img_src'], 'http://test.url/Test%20Query.jpg') + self.assertEqual(results[0]['thumbnail_src'], + 'https://www.bing.com/th?id=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0') + self.assertEqual(results[0]['img_src'], + 'http://www.digital-topo-maps.com/county-map/south-carolina-county-map.gif') html = """ <a href="#" ihk="HN.608003696942779811" diff --git a/tests/unit/engines/test_frinkiac.py b/tests/unit/engines/test_frinkiac.py new file mode 100644 index 000000000..f3eb021d2 --- /dev/null +++ b/tests/unit/engines/test_frinkiac.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from json import dumps +from searx.engines import frinkiac +from searx.testing import SearxTestCase + + +class TestFrinkiacEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + request_dict = defaultdict(dict) + params = frinkiac.request(query, request_dict) + self.assertTrue('url' in params) + + def test_response(self): + self.assertRaises(AttributeError, frinkiac.response, None) + self.assertRaises(AttributeError, frinkiac.response, []) + self.assertRaises(AttributeError, frinkiac.response, '') + self.assertRaises(AttributeError, frinkiac.response, '[]') + + text = """ +[{"Id":770931, + "Episode":"S06E18", + "Timestamp":534616, + "Filename":""}, + {"Id":1657080, + "Episode":"S12E14", + "Timestamp":910868, + "Filename":""}, + {"Id":1943753, + "Episode":"S14E21", + "Timestamp":773439, + "Filename":""}, + {"Id":107835, + "Episode":"S02E03", + "Timestamp":531709, + "Filename":""}] + """ + + response = mock.Mock(text=text) + results = frinkiac.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 4) + self.assertEqual(results[0]['title'], u'S06E18') + self.assertEqual(results[0]['url'], 'https://frinkiac.com/?p=caption&e=S06E18&t=534616') + self.assertEqual(results[0]['thumbnail_src'], 'https://frinkiac.com/img/S06E18/534616/medium.jpg') + self.assertEqual(results[0]['img_src'], 'https://frinkiac.com/img/S06E18/534616.jpg') diff --git a/tests/unit/engines/test_gigablast.py b/tests/unit/engines/test_gigablast.py index 4ae0c51a9..cb96f3cd1 100644 --- a/tests/unit/engines/test_gigablast.py +++ b/tests/unit/engines/test_gigablast.py @@ -22,37 +22,64 @@ class TestGigablastEngine(SearxTestCase): self.assertRaises(AttributeError, gigablast.response, '') self.assertRaises(AttributeError, gigablast.response, '[]') - response = mock.Mock(content='<response></response>') + response = mock.Mock(text='{"results": []}') self.assertEqual(gigablast.response(response), []) - response = mock.Mock(content='<response></response>') - self.assertEqual(gigablast.response(response), []) - - xml = """<?xml version="1.0" encoding="UTF-8" ?> - <response> - <hits>5941888</hits> - <moreResultsFollow>1</moreResultsFollow> - <result> - <title><![CDATA[This should be the title]]></title> - <sum><![CDATA[This should be the content.]]></sum> - <url><![CDATA[http://this.should.be.the.link/]]></url> - <size>90.5</size> - <docId>145414002633</docId> - <siteId>2660021087</siteId> - <domainId>2660021087</domainId> - <spidered>1320519373</spidered> - <indexed>1320519373</indexed> - <pubdate>4294967295</pubdate> - <isModDate>0</isModDate> - <language><![CDATA[English]]></language> - <charset><![CDATA[UTF-8]]></charset> - </result> - </response> + json = """{"results": [ + { + "title":"South by Southwest 2016", + "dmozEntry":{ + "dmozCatId":1041152, + "directCatId":1, + "dmozCatStr":"Top: Regional: North America: United States", + "dmozTitle":"South by Southwest (SXSW)", + "dmozSum":"Annual music, film, and interactive conference.", + "dmozAnchor":"" + }, + "dmozEntry":{ + "dmozCatId":763945, + "directCatId":1, + "dmozCatStr":"Top: Regional: North America: United States", + "dmozTitle":"South by Southwest (SXSW)", + "dmozSum":"", + "dmozAnchor":"www.sxsw.com" + }, + "dmozEntry":{ + "dmozCatId":761446, + "directCatId":1, + "dmozCatStr":"Top: Regional: North America: United States", + "dmozTitle":"South by Southwest (SXSW)", + "dmozSum":"Music, film, and interactive conference and festival.", + "dmozAnchor":"" + }, + "indirectDmozCatId":1041152, + "indirectDmozCatId":763945, + "indirectDmozCatId":761446, + "contentType":"html", + "sum":"This should be the content.", + "url":"www.sxsw.com", + "hopCount":0, + "size":" 102k", + "sizeInBytes":104306, + "bytesUsedToComputeSummary":70000, + "docId":269411794364, + "docScore":586571136.000000, + "summaryGenTimeMS":12, + "summaryTagdbLookupTimeMS":0, + "summaryTitleRecLoadTimeMS":1, + "site":"www.sxsw.com", + "spidered":1452203608, + "firstIndexedDateUTC":1444167123, + "contentHash32":2170650347, + "language":"English", + "langAbbr":"en" + } +]} """ - response = mock.Mock(content=xml) + response = mock.Mock(text=json) results = gigablast.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'This should be the title') - self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') + self.assertEqual(results[0]['title'], 'South by Southwest 2016') + self.assertEqual(results[0]['url'], 'www.sxsw.com') self.assertEqual(results[0]['content'], 'This should be the content.') diff --git a/tests/unit/engines/test_wolframalpha_api.py b/tests/unit/engines/test_wolframalpha_api.py new file mode 100644 index 000000000..c80775795 --- /dev/null +++ b/tests/unit/engines/test_wolframalpha_api.py @@ -0,0 +1,307 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import wolframalpha_api +from searx.testing import SearxTestCase + + +class TestWolframAlphaAPIEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + api_key = 'XXXXXX-XXXXXXXXXX' + dicto = defaultdict(dict) + dicto['api_key'] = api_key + params = wolframalpha_api.request(query, dicto) + + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('wolframalpha.com', params['url']) + + self.assertIn('api_key', params) + self.assertIn(api_key, params['api_key']) + + def test_response(self): + self.assertRaises(AttributeError, wolframalpha_api.response, None) + self.assertRaises(AttributeError, wolframalpha_api.response, []) + self.assertRaises(AttributeError, wolframalpha_api.response, '') + self.assertRaises(AttributeError, wolframalpha_api.response, '[]') + + xml = '''<?xml version='1.0' encoding='UTF-8'?> + <queryresult success='false' error='false' /> + ''' + # test failure + response = mock.Mock(content=xml) + self.assertEqual(wolframalpha_api.response(response), []) + + xml = """<?xml version='1.0' encoding='UTF-8'?> + <queryresult success='true' + error='false' + numpods='6' + datatypes='' + timedout='' + timedoutpods='' + timing='0.684' + parsetiming='0.138' + parsetimedout='false' + recalculate='' + id='MSPa416020a7966dachc463600000f9c66cc21444cfg' + host='http://www3.wolframalpha.com' + server='6' + related='http://www3.wolframalpha.com/api/v2/relatedQueries.jsp?...' + version='2.6'> + <pod title='Input' + scanner='Identity' + id='Input' + position='100' + error='false' + numsubpods='1'> + <subpod title=''> + <plaintext>sqrt(-1)</plaintext> + </subpod> + </pod> + <pod title='Result' + scanner='Simplification' + id='Result' + position='200' + error='false' + numsubpods='1' + primary='true'> + <subpod title=''> + <plaintext></plaintext> + </subpod> + <states count='1'> + <state name='Step-by-step solution' + input='Result__Step-by-step solution' /> + </states> + </pod> + <pod title='Polar coordinates' + scanner='Numeric' + id='PolarCoordinates' + position='300' + error='false' + numsubpods='1'> + <subpod title=''> + <plaintext>r1 (radius), θ90° (angle)</plaintext> + </subpod> + </pod> + <pod title='Position in the complex plane' + scanner='Numeric' + id='PositionInTheComplexPlane' + position='400' + error='false' + numsubpods='1'> + <subpod title=''> + <plaintext></plaintext> + </subpod> + </pod> + <pod title='All 2nd roots of -1' + scanner='RootsOfUnity' + id='' + position='500' + error='false' + numsubpods='2'> + <subpod title=''> + <plaintext> (principal root)</plaintext> + </subpod> + <subpod title=''> + <plaintext>-</plaintext> + </subpod> + </pod> + <pod title='Plot of all roots in the complex plane' + scanner='RootsOfUnity' + id='PlotOfAllRootsInTheComplexPlane' + position='600' + error='false' + numsubpods='1'> + <subpod title=''> + <plaintext></plaintext> + </subpod> + </pod> + </queryresult> + """ + # test private user area char in response + response = mock.Mock(content=xml) + results = wolframalpha_api.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertIn('i', results[0]['answer']) + self.assertIn('sqrt(-1) - Wolfram|Alpha', results[1]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=sqrt%28-1%29', results[1]['url']) + + xml = """<?xml version='1.0' encoding='UTF-8'?> + <queryresult success='true' + error='false' + numpods='2' + datatypes='' + timedout='' + timedoutpods='' + timing='1.286' + parsetiming='0.255' + parsetimedout='false' + recalculate='' + id='MSPa195222ad740ede5214h30000480ca61h003d3gd6' + host='http://www3.wolframalpha.com' + server='20' + related='http://www3.wolframalpha.com/api/v2/relatedQueries.jsp?id=...' + version='2.6'> + <pod title='Indefinite integral' + scanner='Integral' + id='IndefiniteIntegral' + position='100' + error='false' + numsubpods='1' + primary='true'> + <subpod title=''> + <plaintext>∫1/xxlog(x)+constant</plaintext> + </subpod> + <states count='1'> + <state name='Step-by-step solution' + input='IndefiniteIntegral__Step-by-step solution' /> + </states> + <infos count='1'> + <info text='log(x) is the natural logarithm'> + <link url='http://reference.wolfram.com/mathematica/ref/Log.html' + text='Documentation' + title='Mathematica' /> + <link url='http://functions.wolfram.com/ElementaryFunctions/Log' + text='Properties' + title='Wolfram Functions Site' /> + <link url='http://mathworld.wolfram.com/NaturalLogarithm.html' + text='Definition' + title='MathWorld' /> + </info> + </infos> + </pod> + <pod title='Plots of the integral' + scanner='Integral' + id='Plot' + position='200' + error='false' + numsubpods='2'> + <subpod title=''> + <plaintext></plaintext> + <states count='1'> + <statelist count='2' + value='Complex-valued plot' + delimiters=''> + <state name='Complex-valued plot' + input='Plot__1_Complex-valued plot' /> + <state name='Real-valued plot' + input='Plot__1_Real-valued plot' /> + </statelist> + </states> + </subpod> + <subpod title=''> + <plaintext></plaintext> + <states count='1'> + <statelist count='2' + value='Complex-valued plot' + delimiters=''> + <state name='Complex-valued plot' + input='Plot__2_Complex-valued plot' /> + <state name='Real-valued plot' + input='Plot__2_Real-valued plot' /> + </statelist> + </states> + </subpod> + </pod> + <assumptions count='1'> + <assumption type='Clash' + word='integral' + template='Assuming "${word}" is ${desc1}. Use as ${desc2} instead' + count='2'> + <value name='IntegralsWord' + desc='an integral' + input='*C.integral-_*IntegralsWord-' /> + <value name='MathematicalFunctionIdentityPropertyClass' + desc='a function property' + input='*C.integral-_*MathematicalFunctionIdentityPropertyClass-' /> + </assumption> + </assumptions> + </queryresult> + """ + # test integral + response = mock.Mock(content=xml) + results = wolframalpha_api.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertIn('log(x)+c', results[0]['answer']) + self.assertIn('∫1/xx - Wolfram|Alpha'.decode('utf-8'), results[1]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=%E2%88%AB1%2Fx%EF%9D%8Cx', results[1]['url']) + + xml = """<?xml version='1.0' encoding='UTF-8'?> + <queryresult success='true' + error='false' + numpods='4' + datatypes='Solve' + timedout='' + timedoutpods='' + timing='0.79' + parsetiming='0.338' + parsetimedout='false' + recalculate='' + id='MSPa7481f7i06d25h3deh2900004810i3a78d9b4fdc' + host='http://www5b.wolframalpha.com' + server='23' + related='http://www5b.wolframalpha.com/api/v2/relatedQueries.jsp?id=...' + version='2.6'> + <pod title='Input interpretation' + scanner='Identity' + id='Input' + position='100' + error='false' + numsubpods='1'> + <subpod title=''> + <plaintext>solve x^2+x0</plaintext> + </subpod> + </pod> + <pod title='Results' + scanner='Solve' + id='Result' + position='200' + error='false' + numsubpods='2' + primary='true'> + <subpod title=''> + <plaintext>x-1</plaintext> + </subpod> + <subpod title=''> + <plaintext>x0</plaintext> + </subpod> + <states count='1'> + <state name='Step-by-step solution' + input='Result__Step-by-step solution' /> + </states> + </pod> + <pod title='Root plot' + scanner='Solve' + id='RootPlot' + position='300' + error='false' + numsubpods='1'> + <subpod title=''> + <plaintext></plaintext> + </subpod> + </pod> + <pod title='Number line' + scanner='Solve' + id='NumberLine' + position='400' + error='false' + numsubpods='1'> + <subpod title=''> + <plaintext></plaintext> + </subpod> + </pod> + </queryresult> + """ + # test ecuation with multiple answers + response = mock.Mock(content=xml) + results = wolframalpha_api.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 3) + self.assertIn('x=-1', results[0]['answer']) + self.assertIn('x=0', results[1]['answer']) + self.assertIn('solve x^2+x0 - Wolfram|Alpha'.decode('utf-8'), results[2]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=solve+x%5E2%2Bx%EF%9F%990', results[2]['url']) diff --git a/tests/unit/engines/test_wolframalpha_noapi.py b/tests/unit/engines/test_wolframalpha_noapi.py new file mode 100644 index 000000000..cad9593f2 --- /dev/null +++ b/tests/unit/engines/test_wolframalpha_noapi.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import wolframalpha_noapi +from searx.testing import SearxTestCase + + +class TestWolframAlphaNoAPIEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = wolframalpha_noapi.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('wolframalpha.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, wolframalpha_noapi.response, None) + self.assertRaises(AttributeError, wolframalpha_noapi.response, []) + self.assertRaises(AttributeError, wolframalpha_noapi.response, '') + self.assertRaises(AttributeError, wolframalpha_noapi.response, '[]') + + html = """ + <!DOCTYPE html> + <title> Parangaricutirimícuaro - Wolfram|Alpha</title> + <meta charset="utf-8" /> + <body> + <div id="closest"> + <p class="pfail">Wolfram|Alpha doesn't know how to interpret your input.</p> + <div id="dtips"> + <div class="tip"> + <span class="tip-title">Tip: </span> + Check your spelling, and use English + <span class="tip-extra"></span> + </div> + </div> + </div> + </body> + </html> + """ + # test failed query + response = mock.Mock(text=html) + self.assertEqual(wolframalpha_noapi.response(response), []) + + html = """ + <!DOCTYPE html> + <title> sqrt(-1) - Wolfram|Alpha</title> + <meta charset="utf-8" /> + <body> + <script type="text/javascript"> + try { + if (typeof context.jsonArray.popups.pod_0100 == "undefined" ) { + context.jsonArray.popups.pod_0100 = []; + } + context.jsonArray.popups.pod_0100.push( {"stringified": "sqrt(-1)","mInput": "","mOutput": ""}); + } catch(e) { } + + try { + if (typeof context.jsonArray.popups.pod_0200 == "undefined" ) { + context.jsonArray.popups.pod_0200 = []; + } + context.jsonArray.popups.pod_0200.push( {"stringified": "i","mInput": "","mOutput": ""}); + } catch(e) { } + </script> + </body> + </html> + """ + # test plaintext + response = mock.Mock(text=html) + results = wolframalpha_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertEquals('i', results[0]['answer']) + self.assertIn('sqrt(-1) - Wolfram|Alpha', results[1]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=+sqrt%28-1%29', results[1]['url']) + + html = """ + <!DOCTYPE html> + <title> integral 1/x - Wolfram|Alpha</title> + <meta charset="utf-8" /> + <body> + <script type="text/javascript"> + try { + if (typeof context.jsonArray.popups.pod_0100 == "undefined" ) { + context.jsonArray.popups.pod_0100 = []; + } + context.jsonArray.popups.pod_0100.push( {"stringified": "integral 1\/x dx = log(x)+constant"}); + } catch(e) { } + </script> + </body> + </html> + """ + # test integral + response = mock.Mock(text=html) + results = wolframalpha_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertIn('log(x)+c', results[0]['answer']) + self.assertIn('integral 1/x - Wolfram|Alpha', results[1]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=+integral+1%2Fx', results[1]['url']) + + html = """ + <!DOCTYPE html> + <title> ∫1/x x - Wolfram|Alpha</title> + <meta charset="utf-8" /> + <body> + <script type="text/javascript"> + try { + if (typeof context.jsonArray.popups.pod_0100 == "undefined" ) { + context.jsonArray.popups.pod_0100 = []; + } + context.jsonArray.popups.pod_0100.push( {"stringified": "integral 1\/x dx = log(x)+constant"}); + } catch(e) { } + </script> + </body> + </html> + """ + # test input in mathematical notation + response = mock.Mock(text=html) + results = wolframalpha_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertIn('log(x)+c', results[0]['answer']) + self.assertIn('∫1/x x - Wolfram|Alpha'.decode('utf-8'), results[1]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=+%E2%88%AB1%2Fx+%EF%9D%8Cx', results[1]['url']) + + html = """ + <!DOCTYPE html> + <title> 1 euro to yen - Wolfram|Alpha</title> + <meta charset="utf-8" /> + <body> + <script type="text/javascript"> + try { + if (typeof context.jsonArray.popups.pod_0100 == "undefined" ) { + context.jsonArray.popups.pod_0100 = []; + } + context.jsonArray.popups.pod_0100.push( {"stringified": "convert euro1 (euro) to Japanese yen"}); + } catch(e) { } + + try { + if (typeof context.jsonArray.popups.pod_0200 == "undefined" ) { + context.jsonArray.popups.pod_0200 = []; + } + context.jsonArray.popups.pod_0200.push( {"stringified": "¥130.5 (Japanese yen)"}); + } catch(e) { } + </script> + </body> + </html> + """ + # test output with htmlentity + response = mock.Mock(text=html) + results = wolframalpha_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertIn('¥'.decode('utf-8'), results[0]['answer']) + self.assertIn('1 euro to yen - Wolfram|Alpha', results[1]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=+1+euro+to+yen', results[1]['url']) + + html = """ + <!DOCTYPE html> + <title> distance from nairobi to kyoto in inches - Wolfram|Alpha</title> + <meta charset="utf-8" /> + <body> + <script type="text/javascript"> + try { + if (typeof context.jsonArray.popups.pod_0100 == "undefined" ) { + context.jsonArray.popups.pod_0100 = []; + } +[...].pod_0100.push( {"stringified": "convert distance | from | Nairobi, Kenya\nto | Kyoto, Japan to inches"}); + } catch(e) { } + + try { + if (typeof context.jsonArray.popups.pod_0200 == "undefined" ) { + context.jsonArray.popups.pod_0200 = []; + } +pod_0200.push({"stringified": "4.295×10^8 inches","mOutput": "Quantity[4.295×10^8,&quot;Inches&quot;]"}); + + } catch(e) { } + </script> + </body> + </html> + """ + # test output with utf-8 character + response = mock.Mock(text=html) + results = wolframalpha_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 2) + self.assertIn('4.295×10^8 inches'.decode('utf-8'), results[0]['answer']) + self.assertIn('distance from nairobi to kyoto in inches - Wolfram|Alpha', results[1]['title']) + self.assertEquals('http://www.wolframalpha.com/input/?i=+distance+from+nairobi+to+kyoto+in+inches', + results[1]['url']) |