From 149802c56926bf48520c98932c4c36b8152b3d2d Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 5 Aug 2016 23:34:56 -0500 Subject: [enh] add supported_languages on engines and auto-generate languages.py --- searx/engines/swisscows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'searx/engines/swisscows.py') diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py index 72184e428..68632a15a 100644 --- a/searx/engines/swisscows.py +++ b/searx/engines/swisscows.py @@ -36,8 +36,8 @@ def request(query, params): ui_language = 'browser' region = 'browser' else: - region = params['language'].replace('_', '-') - ui_language = params['language'].split('_')[0] + region = params['language'] + ui_language = params['language'].split('-')[0] search_path = search_string.format( query=urlencode({'query': query, -- cgit v1.2.3 From a11948c71bfe7b2aac6e50e7634874d5073c7d84 Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 29 Oct 2016 21:04:01 -0500 Subject: Add language support for more engines. --- searx/engines/swisscows.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'searx/engines/swisscows.py') diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py index 68632a15a..5c6b051a9 100644 --- a/searx/engines/swisscows.py +++ b/searx/engines/swisscows.py @@ -18,6 +18,12 @@ import re categories = ['general', 'images'] paging = True language_support = True +supported_languages = ["ar-SA", "es-AR", "en-AU", "de-AT", "fr-BE", "nl-BE", "pt-BR", "bg-BG", "en-CA", "fr-CA", + "es-CL", "zh-CN", "hr-HR", "cs-CZ", "da-DK", "et-EE", "fi-FI", "fr-FR", "de-DE", "el-GR", + "zh-HK", "hu-HU", "en-IN", "en-IE", "he-IL", "it-IT", "ja-JP", "ko-KR", "lv-LV", "lt-LT", + "en-MY", "es-MX", "nl-NL", "en-NZ", "nb-NO", "en-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU", + "en-SG", "sk-SK", "sl-SI", "en-ZA", "es-ES", "sv-SE", "de-CH", "fr-CH", "zh-TW", "th-TH", + "tr-TR", "uk-UA", "en-GB", "en-US", "es-US"] # search-url base_url = 'https://swisscows.ch/' @@ -35,6 +41,8 @@ def request(query, params): if params['language'] == 'all': ui_language = 'browser' region = 'browser' + elif params['language'].split('-')[0] == 'no': + region = 'nb-NO' else: region = params['language'] ui_language = params['language'].split('-')[0] -- cgit v1.2.3 From f62ce21f50b540315a708ebfbf36878ddec9d1c4 Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 5 Nov 2016 20:51:38 -0600 Subject: [mod] fetch supported languages for several engines utils/fetch_languages.py gets languages supported by each engine and generates engines_languages.json with each engine's supported language. --- searx/engines/swisscows.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'searx/engines/swisscows.py') diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py index 5c6b051a9..7f85019a6 100644 --- a/searx/engines/swisscows.py +++ b/searx/engines/swisscows.py @@ -13,17 +13,13 @@ from json import loads from urllib import urlencode, unquote import re +from requests import get +from lxml.html import fromstring # engine dependent config categories = ['general', 'images'] paging = True language_support = True -supported_languages = ["ar-SA", "es-AR", "en-AU", "de-AT", "fr-BE", "nl-BE", "pt-BR", "bg-BG", "en-CA", "fr-CA", - "es-CL", "zh-CN", "hr-HR", "cs-CZ", "da-DK", "et-EE", "fi-FI", "fr-FR", "de-DE", "el-GR", - "zh-HK", "hu-HU", "en-IN", "en-IE", "he-IL", "it-IT", "ja-JP", "ko-KR", "lv-LV", "lt-LT", - "en-MY", "es-MX", "nl-NL", "en-NZ", "nb-NO", "en-PH", "pl-PL", "pt-PT", "ro-RO", "ru-RU", - "en-SG", "sk-SK", "sl-SI", "en-ZA", "es-ES", "sv-SE", "de-CH", "fr-CH", "zh-TW", "th-TH", - "tr-TR", "uk-UA", "en-GB", "en-US", "es-US"] # search-url base_url = 'https://swisscows.ch/' @@ -114,3 +110,16 @@ def response(resp): # return results return results + + +# get supported languages from their site +def fetch_supported_languages(): + supported_languages = [] + response = get(base_url) + dom = fromstring(response.text) + options = dom.xpath('//div[@id="regions-popup"]//ul/li/a') + for option in options: + code = option.xpath('./@data-val')[0] + supported_languages.append(code) + + return supported_languages -- cgit v1.2.3 From af35eee10b98940c51c6e5e18629de514b4bd48d Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 15 Dec 2016 00:34:43 -0600 Subject: tests for _fetch_supported_languages in engines and refactor method to make it testable without making requests --- searx/engines/swisscows.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'searx/engines/swisscows.py') diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py index 7f85019a6..d8a454039 100644 --- a/searx/engines/swisscows.py +++ b/searx/engines/swisscows.py @@ -13,7 +13,6 @@ from json import loads from urllib import urlencode, unquote import re -from requests import get from lxml.html import fromstring # engine dependent config @@ -25,6 +24,8 @@ language_support = True base_url = 'https://swisscows.ch/' search_string = '?{query}&page={page}' +supported_languages_url = base_url + # regex regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment') regex_json_remove_start = re.compile(r'^initialData:\s*') @@ -113,10 +114,9 @@ def response(resp): # get supported languages from their site -def fetch_supported_languages(): +def _fetch_supported_languages(resp): supported_languages = [] - response = get(base_url) - dom = fromstring(response.text) + dom = fromstring(resp.text) options = dom.xpath('//div[@id="regions-popup"]//ul/li/a') for option in options: code = option.xpath('./@data-val')[0] -- cgit v1.2.3