From 78d3f3d6b116e0aca8af8b939d5a98ea74bb8de3 Mon Sep 17 00:00:00 2001 From: a01200356 Date: Sat, 27 Feb 2016 19:06:44 -0600 Subject: [enh] infobox for wolframalpha TODO: - infobox styles - unit tests ISSUES: - no_api version needs to re-call server for additional pods, such as plots. therefore, it's even slower than before. comment out the part that calls get_async_pod if requests reach timeout or increase timeout in settings.yml. --- searx/engines/wolframalpha_api.py | 84 ++++++++++++++++++++-------- searx/engines/wolframalpha_noapi.py | 106 +++++++++++++++++++++++++++++------- 2 files changed, 147 insertions(+), 43 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/wolframalpha_api.py b/searx/engines/wolframalpha_api.py index 303c6c165..ad80fb0f4 100644 --- a/searx/engines/wolframalpha_api.py +++ b/searx/engines/wolframalpha_api.py @@ -1,40 +1,56 @@ -# Wolfram Alpha (Maths) +# Wolfram Alpha (Science) # -# @website http://www.wolframalpha.com -# @provide-api yes (http://api.wolframalpha.com/v2/) +# @website https://www.wolframalpha.com +# @provide-api yes (https://api.wolframalpha.com/v2/) # # @using-api yes # @results XML # @stable yes -# @parse result +# @parse url, infobox from urllib import urlencode from lxml import etree -from re import search # search-url -base_url = 'http://api.wolframalpha.com/v2/query' -search_url = base_url + '?appid={api_key}&{query}&format=plaintext' -site_url = 'http://www.wolframalpha.com/input/?{query}' +search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}' +site_url = 'https://www.wolframalpha.com/input/?{query}' api_key = '' # defined in settings.yml # xpath variables failure_xpath = '/queryresult[attribute::success="false"]' answer_xpath = '//pod[attribute::primary="true"]/subpod/plaintext' input_xpath = '//pod[starts-with(attribute::title, "Input")]/subpod/plaintext' +pods_xpath = '//pod' +subpods_xpath = './subpod' +pod_title_xpath = './@title' +plaintext_xpath = './plaintext' +image_xpath = './img' +img_src_xpath = './@src' +img_alt_xpath = './@alt' + +# pods to display as image in infobox +# this pods do return a plaintext, but they look better and are more useful as images +image_pods = {'Visual representation', + 'Manipulatives illustration'} # do search-request def request(query, params): params['url'] = search_url.format(query=urlencode({'input': query}), api_key=api_key) + params['headers']['Referer'] = site_url.format(query=urlencode({'i': query})) return params # replace private user area characters to make text legible def replace_pua_chars(text): - pua_chars = {u'\uf74c': 'd', + pua_chars = {u'\uf522': u'\u2192', + u'\uf7b1': u'\u2115', + u'\uf7b4': u'\u211a', + u'\uf7b5': u'\u211d', + u'\uf7bd': u'\u2124', + u'\uf74c': 'd', u'\uf74d': u'\u212f', u'\uf74e': 'i', u'\uf7d9': '='} @@ -55,23 +71,45 @@ def response(resp): if search_results.xpath(failure_xpath): return [] - # parse answers - answers = search_results.xpath(answer_xpath) - if answers: - for answer in answers: - answer = replace_pua_chars(answer.text) + infobox_title = search_results.xpath(input_xpath) + if infobox_title: + infobox_title = replace_pua_chars(infobox_title[0].text) + + pods = search_results.xpath(pods_xpath) + result_chunks = [] + for pod in pods: + pod_title = replace_pua_chars(pod.xpath(pod_title_xpath)[0]) + + subpods = pod.xpath(subpods_xpath) + if not subpods: + continue + + for subpod in subpods: + content = subpod.xpath(plaintext_xpath)[0].text + image = subpod.xpath(image_xpath) + if content and pod_title not in image_pods: + content = replace_pua_chars(content) + result_chunks.append({'label': pod_title, 'value': content}) - results.append({'answer': answer}) + # if there's no input pod, infobox_title is content of first pod + if not infobox_title: + infobox_title = content + + elif image: + result_chunks.append({'label': pod_title, + 'image': {'src': image[0].xpath(img_src_xpath)[0], + 'alt': image[0].xpath(img_alt_xpath)[0]}}) + + if not result_chunks: + return [] - # if there's no input section in search_results, check if answer has the input embedded (before their "=" sign) - try: - query_input = search_results.xpath(input_xpath)[0].text - except IndexError: - query_input = search(u'([^\uf7d9]+)', answers[0].text).group(1) + results.append({'infobox': infobox_title, + 'attributes': result_chunks, + 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) # append link to site - result_url = site_url.format(query=urlencode({'i': query_input.encode('utf-8')})) - results.append({'url': result_url, - 'title': query_input + " - Wolfram|Alpha"}) + results.append({'url': resp.request.headers['Referer'], + 'title': 'Wolfram|Alpha', + 'content': infobox_title}) return results diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py index 639dccf24..a5fbdeae7 100644 --- a/searx/engines/wolframalpha_noapi.py +++ b/searx/engines/wolframalpha_noapi.py @@ -1,23 +1,23 @@ -# WolframAlpha (Maths) +# Wolfram|Alpha (Science) # -# @website http://www.wolframalpha.com/ -# @provide-api yes (http://api.wolframalpha.com/v2/) +# @website https://www.wolframalpha.com/ +# @provide-api yes (https://api.wolframalpha.com/v2/) # # @using-api no -# @results HTML +# @results JSON # @stable no -# @parse answer +# @parse url, infobox from cgi import escape from json import loads from time import time from urllib import urlencode +from lxml.etree import XML from searx.poolrequests import get as http_get # search-url url = 'https://www.wolframalpha.com/' -search_url = url + 'input/?{query}' search_url = url + 'input/json.jsp'\ '?async=true'\ @@ -33,13 +33,25 @@ search_url = url + 'input/json.jsp'\ '&sponsorcategories=true'\ '&statemethod=deploybutton' -# xpath variables -scripts_xpath = '//script' -title_xpath = '//title' -failure_xpath = '//p[attribute::class="pfail"]' +referer_url = url + 'input/?{query}' + token = {'value': '', 'last_updated': None} +# xpath variables +success_xpath = '/pod[attribute::error="false"]' +plaintext_xpath = './plaintext' +title_xpath = './@title' +image_xpath = './img' +img_src_xpath = './img/@src' +img_alt_xpath = './img/@alt' + +# pods to display as image in infobox +# this pods do return a plaintext, but they look better and are more useful as images +image_pods = {'Visual representation', + 'Manipulatives illustration', + 'Symbol'} + # seems, wolframalpha resets its token in every hour def obtain_token(): @@ -62,13 +74,42 @@ def request(query, params): if time() - token['last_updated'] > 3600: obtain_token() params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value']) - params['headers']['Referer'] = 'https://www.wolframalpha.com/input/?i=' + query + params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query})) return params +# get additional pod +# NOTE: this makes an additional requests to server, so the response will take longer and might reach timeout +def get_async_pod(url): + pod = {'subpods': []} + + try: + resp = http_get(url, timeout=2.0) + + resp_pod = XML(resp.content) + if resp_pod.xpath(success_xpath): + + for subpod in resp_pod: + plaintext = subpod.xpath(plaintext_xpath)[0].text + if plaintext: + pod['subpods'].append({'title': subpod.xpath(title_xpath)[0], + 'plaintext': plaintext}) + elif subpod.xpath(image_xpath): + pod['subpods'].append({'title': subpod.xpath(title_xpath)[0], + 'plaintext': '', + 'img': {'src': subpod.xpath(img_src_xpath)[0], + 'alt': subpod.xpath(img_alt_xpath)[0]}}) + except: + pass + + return pod + + # get response from search-request def response(resp): + results = [] + resp_json = loads(resp.text) if not resp_json['queryresult']['success']: @@ -76,20 +117,45 @@ def response(resp): # TODO handle resp_json['queryresult']['assumptions'] result_chunks = [] + infobox_title = None for pod in resp_json['queryresult']['pods']: pod_title = pod.get('title', '') + if 'subpods' not in pod: - continue + # comment this section if your requests always reach timeout + if pod['async']: + result = get_async_pod(pod['async']) + if result: + pod = result + else: + continue + + # infobox title is input or text content on first pod + if pod_title.startswith('Input') or not infobox_title: + try: + infobox_title = pod['subpods'][0]['plaintext'] + except: + infobox_title = '' + pass + for subpod in pod['subpods']: - if 'img' in subpod: - result_chunks.append(u'

{0}
{2}

' - .format(escape(pod_title or subpod['img']['alt']), - escape(subpod['img']['src']), - escape(subpod['img']['alt']))) + if subpod['plaintext'] != '' and pod_title not in image_pods: + # append unless it's not an actual answer + if subpod['plaintext'] != '(requires interactivity)': + result_chunks.append({'label': pod_title, 'value': subpod['plaintext']}) + + elif 'img' in subpod: + result_chunks.append({'label': pod_title, 'image': subpod['img']}) if not result_chunks: return [] - return [{'url': resp.request.headers['Referer'].decode('utf-8'), - 'title': 'Wolframalpha', - 'content': ''.join(result_chunks)}] + results.append({'infobox': infobox_title, + 'attributes': result_chunks, + 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) + + results.append({'url': resp.request.headers['Referer'], + 'title': 'Wolfram|Alpha', + 'content': infobox_title}) + + return results -- cgit v1.2.3 From 4d8996eb4d54a0938ad9dc6ad105de2b850fa033 Mon Sep 17 00:00:00 2001 From: a01200356 Date: Sun, 28 Feb 2016 00:47:36 -0600 Subject: [enh] unit tests for wolframalpha --- searx/engines/wolframalpha_api.py | 47 ++++++++++++++++------------ searx/engines/wolframalpha_noapi.py | 61 +++++++++++++++++++++---------------- 2 files changed, 61 insertions(+), 47 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/wolframalpha_api.py b/searx/engines/wolframalpha_api.py index ad80fb0f4..9a13d7422 100644 --- a/searx/engines/wolframalpha_api.py +++ b/searx/engines/wolframalpha_api.py @@ -19,9 +19,10 @@ api_key = '' # defined in settings.yml # xpath variables failure_xpath = '/queryresult[attribute::success="false"]' answer_xpath = '//pod[attribute::primary="true"]/subpod/plaintext' -input_xpath = '//pod[starts-with(attribute::title, "Input")]/subpod/plaintext' +input_xpath = '//pod[starts-with(attribute::id, "Input")]/subpod/plaintext' pods_xpath = '//pod' subpods_xpath = './subpod' +pod_id_xpath = './@id' pod_title_xpath = './@title' plaintext_xpath = './plaintext' image_xpath = './img' @@ -30,8 +31,8 @@ img_alt_xpath = './@alt' # pods to display as image in infobox # this pods do return a plaintext, but they look better and are more useful as images -image_pods = {'Visual representation', - 'Manipulatives illustration'} +image_pods = {'VisualRepresentation', + 'Illustration'} # do search-request @@ -45,15 +46,15 @@ def request(query, params): # replace private user area characters to make text legible def replace_pua_chars(text): - pua_chars = {u'\uf522': u'\u2192', - u'\uf7b1': u'\u2115', - u'\uf7b4': u'\u211a', - u'\uf7b5': u'\u211d', - u'\uf7bd': u'\u2124', - u'\uf74c': 'd', - u'\uf74d': u'\u212f', - u'\uf74e': 'i', - u'\uf7d9': '='} + pua_chars = {u'\uf522': u'\u2192', # rigth arrow + u'\uf7b1': u'\u2115', # set of natural numbers + u'\uf7b4': u'\u211a', # set of rational numbers + u'\uf7b5': u'\u211d', # set of real numbers + u'\uf7bd': u'\u2124', # set of integer numbers + u'\uf74c': 'd', # differential + u'\uf74d': u'\u212f', # euler's number + u'\uf74e': 'i', # imaginary number + u'\uf7d9': '='} # equals sign for k, v in pua_chars.iteritems(): text = text.replace(k, v) @@ -71,30 +72,35 @@ def response(resp): if search_results.xpath(failure_xpath): return [] - infobox_title = search_results.xpath(input_xpath) - if infobox_title: - infobox_title = replace_pua_chars(infobox_title[0].text) + try: + infobox_title = search_results.xpath(input_xpath)[0].text + except: + infobox_title = None pods = search_results.xpath(pods_xpath) result_chunks = [] for pod in pods: - pod_title = replace_pua_chars(pod.xpath(pod_title_xpath)[0]) + pod_id = pod.xpath(pod_id_xpath)[0] + pod_title = pod.xpath(pod_title_xpath)[0] subpods = pod.xpath(subpods_xpath) if not subpods: continue + # Appends either a text or an image, depending on which one is more suitable for subpod in subpods: content = subpod.xpath(plaintext_xpath)[0].text image = subpod.xpath(image_xpath) - if content and pod_title not in image_pods: - content = replace_pua_chars(content) - result_chunks.append({'label': pod_title, 'value': content}) - # if there's no input pod, infobox_title is content of first pod + if content and pod_id not in image_pods: + + # if no input pod was found, title is first plaintext pod if not infobox_title: infobox_title = content + content = replace_pua_chars(content) + result_chunks.append({'label': pod_title, 'value': content}) + elif image: result_chunks.append({'label': pod_title, 'image': {'src': image[0].xpath(img_src_xpath)[0], @@ -103,6 +109,7 @@ def response(resp): if not result_chunks: return [] + # append infobox results.append({'infobox': infobox_title, 'attributes': result_chunks, 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py index a5fbdeae7..d10716a2a 100644 --- a/searx/engines/wolframalpha_noapi.py +++ b/searx/engines/wolframalpha_noapi.py @@ -48,8 +48,8 @@ img_alt_xpath = './img/@alt' # pods to display as image in infobox # this pods do return a plaintext, but they look better and are more useful as images -image_pods = {'Visual representation', - 'Manipulatives illustration', +image_pods = {'VisualRepresentation', + 'Illustration', 'Symbol'} @@ -82,26 +82,35 @@ def request(query, params): # get additional pod # NOTE: this makes an additional requests to server, so the response will take longer and might reach timeout def get_async_pod(url): - pod = {'subpods': []} - try: resp = http_get(url, timeout=2.0) - - resp_pod = XML(resp.content) - if resp_pod.xpath(success_xpath): - - for subpod in resp_pod: - plaintext = subpod.xpath(plaintext_xpath)[0].text - if plaintext: - pod['subpods'].append({'title': subpod.xpath(title_xpath)[0], - 'plaintext': plaintext}) - elif subpod.xpath(image_xpath): - pod['subpods'].append({'title': subpod.xpath(title_xpath)[0], - 'plaintext': '', - 'img': {'src': subpod.xpath(img_src_xpath)[0], - 'alt': subpod.xpath(img_alt_xpath)[0]}}) except: - pass + return None + + if resp: + return parse_async_pod(resp) + + +def parse_async_pod(resp): + pod = {'subpods': []} + + resp_pod = XML(resp.content) + + if resp_pod.xpath(success_xpath): + for subpod in resp_pod: + new_subpod = {'title': subpod.xpath(title_xpath)[0]} + + plaintext = subpod.xpath(plaintext_xpath)[0].text + if plaintext: + new_subpod['plaintext'] = plaintext + else: + new_subpod['plaintext'] = '' + + if subpod.xpath(image_xpath): + new_subpod['img'] = {'src': subpod.xpath(img_src_xpath)[0], + 'alt': subpod.xpath(img_alt_xpath)[0]} + + pod['subpods'].append(new_subpod) return pod @@ -119,6 +128,7 @@ def response(resp): result_chunks = [] infobox_title = None for pod in resp_json['queryresult']['pods']: + pod_id = pod.get('id', '') pod_title = pod.get('title', '') if 'subpods' not in pod: @@ -127,19 +137,16 @@ def response(resp): result = get_async_pod(pod['async']) if result: pod = result + else: + continue else: continue - # infobox title is input or text content on first pod - if pod_title.startswith('Input') or not infobox_title: - try: - infobox_title = pod['subpods'][0]['plaintext'] - except: - infobox_title = '' - pass + if pod_id == 'Input' or not infobox_title: + infobox_title = pod['subpods'][0]['plaintext'] for subpod in pod['subpods']: - if subpod['plaintext'] != '' and pod_title not in image_pods: + if subpod['plaintext'] != '' and pod_id not in image_pods: # append unless it's not an actual answer if subpod['plaintext'] != '(requires interactivity)': result_chunks.append({'label': pod_title, 'value': subpod['plaintext']}) -- cgit v1.2.3 From 4cea71e3bb34020dff0f3e28f5c1398c0b0d8278 Mon Sep 17 00:00:00 2001 From: a01200356 Date: Sun, 28 Feb 2016 00:52:47 -0600 Subject: [fix] merge with 79705450dfdf321c19839bce23c56d9d4a86ba68 --- searx/engines/wolframalpha_api.py | 4 ++-- searx/engines/wolframalpha_noapi.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/wolframalpha_api.py b/searx/engines/wolframalpha_api.py index 9a13d7422..4526c825f 100644 --- a/searx/engines/wolframalpha_api.py +++ b/searx/engines/wolframalpha_api.py @@ -112,10 +112,10 @@ def response(resp): # append infobox results.append({'infobox': infobox_title, 'attributes': result_chunks, - 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) + 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]}) # append link to site - results.append({'url': resp.request.headers['Referer'], + results.append({'url': resp.request.headers['Referer'].decode('utf8'), 'title': 'Wolfram|Alpha', 'content': infobox_title}) diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py index d10716a2a..7962d923c 100644 --- a/searx/engines/wolframalpha_noapi.py +++ b/searx/engines/wolframalpha_noapi.py @@ -159,9 +159,9 @@ def response(resp): results.append({'infobox': infobox_title, 'attributes': result_chunks, - 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]}) + 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]}) - results.append({'url': resp.request.headers['Referer'], + results.append({'url': resp.request.headers['Referer'].decode('utf8'), 'title': 'Wolfram|Alpha', 'content': infobox_title}) -- cgit v1.2.3 From 8f3b33de23090e6d11a5a29a239719bb6915d2ec Mon Sep 17 00:00:00 2001 From: a01200356 Date: Sun, 28 Feb 2016 02:05:52 -0600 Subject: [fix] remove unnecesary async calls in wolframalpha_noapi setting async to false in the request did the job, lol. --- searx/engines/wolframalpha_noapi.py | 56 ++----------------------------------- 1 file changed, 2 insertions(+), 54 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py index 7962d923c..59629b833 100644 --- a/searx/engines/wolframalpha_noapi.py +++ b/searx/engines/wolframalpha_noapi.py @@ -20,7 +20,7 @@ from searx.poolrequests import get as http_get url = 'https://www.wolframalpha.com/' search_url = url + 'input/json.jsp'\ - '?async=true'\ + '?async=false'\ '&banners=raw'\ '&debuggingdata=false'\ '&format=image,plaintext,imagemap,minput,moutput'\ @@ -38,14 +38,6 @@ referer_url = url + 'input/?{query}' token = {'value': '', 'last_updated': None} -# xpath variables -success_xpath = '/pod[attribute::error="false"]' -plaintext_xpath = './plaintext' -title_xpath = './@title' -image_xpath = './img' -img_src_xpath = './img/@src' -img_alt_xpath = './img/@alt' - # pods to display as image in infobox # this pods do return a plaintext, but they look better and are more useful as images image_pods = {'VisualRepresentation', @@ -79,42 +71,6 @@ def request(query, params): return params -# get additional pod -# NOTE: this makes an additional requests to server, so the response will take longer and might reach timeout -def get_async_pod(url): - try: - resp = http_get(url, timeout=2.0) - except: - return None - - if resp: - return parse_async_pod(resp) - - -def parse_async_pod(resp): - pod = {'subpods': []} - - resp_pod = XML(resp.content) - - if resp_pod.xpath(success_xpath): - for subpod in resp_pod: - new_subpod = {'title': subpod.xpath(title_xpath)[0]} - - plaintext = subpod.xpath(plaintext_xpath)[0].text - if plaintext: - new_subpod['plaintext'] = plaintext - else: - new_subpod['plaintext'] = '' - - if subpod.xpath(image_xpath): - new_subpod['img'] = {'src': subpod.xpath(img_src_xpath)[0], - 'alt': subpod.xpath(img_alt_xpath)[0]} - - pod['subpods'].append(new_subpod) - - return pod - - # get response from search-request def response(resp): results = [] @@ -132,15 +88,7 @@ def response(resp): pod_title = pod.get('title', '') if 'subpods' not in pod: - # comment this section if your requests always reach timeout - if pod['async']: - result = get_async_pod(pod['async']) - if result: - pod = result - else: - continue - else: - continue + continue if pod_id == 'Input' or not infobox_title: infobox_title = pod['subpods'][0]['plaintext'] -- cgit v1.2.3