From 149802c56926bf48520c98932c4c36b8152b3d2d Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 5 Aug 2016 23:34:56 -0500 Subject: [enh] add supported_languages on engines and auto-generate languages.py --- searx/engines/bing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx/engines/bing.py') diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 58db61251..052b66448 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -32,7 +32,7 @@ def request(query, params): offset = (params['pageno'] - 1) * 10 + 1 if params['language'] != 'all': - query = u'language:{} {}'.format(params['language'].split('_')[0].upper(), + query = u'language:{} {}'.format(params['language'].split('-')[0].upper(), query.decode('utf-8')).encode('utf-8') search_path = search_string.format( -- cgit v1.2.3 From f62ce21f50b540315a708ebfbf36878ddec9d1c4 Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 5 Nov 2016 20:51:38 -0600 Subject: [mod] fetch supported languages for several engines utils/fetch_languages.py gets languages supported by each engine and generates engines_languages.json with each engine's supported language. --- searx/engines/bing.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'searx/engines/bing.py') diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 052b66448..354003399 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -15,12 +15,14 @@ from urllib import urlencode from lxml import html +from requests import get from searx.engines.xpath import extract_text # engine dependent config categories = ['general'] paging = True language_support = True +supported_languages_url = 'https://www.bing.com/account/general' # search-url base_url = 'https://www.bing.com/' @@ -81,3 +83,16 @@ def response(resp): # return results return results + + +# get supported languages from their site +def fetch_supported_languages(): + supported_languages = [] + response = get(supported_languages_url) + dom = html.fromstring(response.text) + options = dom.xpath('//div[@id="limit-languages"]//input') + for option in options: + code = option.xpath('./@id')[0].replace('_', '-') + supported_languages.append(code) + + return supported_languages -- cgit v1.2.3 From af35eee10b98940c51c6e5e18629de514b4bd48d Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 15 Dec 2016 00:34:43 -0600 Subject: tests for _fetch_supported_languages in engines and refactor method to make it testable without making requests --- searx/engines/bing.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'searx/engines/bing.py') diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 354003399..b2ad7b6cf 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -15,7 +15,6 @@ from urllib import urlencode from lxml import html -from requests import get from searx.engines.xpath import extract_text # engine dependent config @@ -86,10 +85,9 @@ def response(resp): # get supported languages from their site -def fetch_supported_languages(): +def _fetch_supported_languages(resp): supported_languages = [] - response = get(supported_languages_url) - dom = html.fromstring(response.text) + dom = html.fromstring(resp.text) options = dom.xpath('//div[@id="limit-languages"]//input') for option in options: code = option.xpath('./@id')[0].replace('_', '-') -- cgit v1.2.3