diff options
Diffstat (limited to 'searx/engines/yahoo.py')
| -rw-r--r-- | searx/engines/yahoo.py | 37 |
1 files changed, 22 insertions, 15 deletions
diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 5387aaf54..36c1a11f8 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -14,6 +14,7 @@ from lxml import html from searx.engines.xpath import extract_text, extract_url from searx.url_utils import unquote, urlencode +from searx.utils import match_language, eval_xpath # engine dependent config categories = ['general'] @@ -39,6 +40,8 @@ time_range_dict = {'day': ['1d', 'd'], 'week': ['1w', 'w'], 'month': ['1m', 'm']} +language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'} + # remove yahoo-specific tracking-url def parse_url(url_string): @@ -73,13 +76,13 @@ def _get_url(query, offset, language, time_range): def _get_language(params): if params['language'] == 'all': return 'en' - elif params['language'][:2] == 'zh': - if params['language'] == 'zh' or params['language'] == 'zh-CH': - return 'szh' - else: - return 'tzh' - else: - return params['language'].split('-')[0] + + language = match_language(params['language'], supported_languages, language_aliases) + if language not in language_aliases.values(): + language = language.split('-')[0] + language = language.replace('-', '_').lower() + + return language # do search-request @@ -106,21 +109,21 @@ def response(resp): dom = html.fromstring(resp.text) try: - results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0] + results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0] .split()[0].replace(',', '')) results.append({'number_of_results': results_num}) except: pass # parse results - for result in dom.xpath(results_xpath): + for result in eval_xpath(dom, results_xpath): try: - url = parse_url(extract_url(result.xpath(url_xpath), search_url)) - title = extract_text(result.xpath(title_xpath)[0]) + url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url)) + title = extract_text(eval_xpath(result, title_xpath)[0]) except: continue - content = extract_text(result.xpath(content_xpath)[0]) + content = extract_text(eval_xpath(result, content_xpath)[0]) # append result results.append({'url': url, @@ -128,7 +131,7 @@ def response(resp): 'content': content}) # if no suggestion found, return results - suggestions = dom.xpath(suggestion_xpath) + suggestions = eval_xpath(dom, suggestion_xpath) if not suggestions: return results @@ -145,9 +148,13 @@ def response(resp): def _fetch_supported_languages(resp): supported_languages = [] dom = html.fromstring(resp.text) - options = dom.xpath('//div[@id="yschlang"]/span/label/input') + options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input') for option in options: - code = option.xpath('./@value')[0][5:].replace('_', '-') + code_parts = eval_xpath(option, './@value')[0][5:].split('_') + if len(code_parts) == 2: + code = code_parts[0] + '-' + code_parts[1].upper() + else: + code = code_parts[0] supported_languages.append(code) return supported_languages |