From 149802c56926bf48520c98932c4c36b8152b3d2d Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 5 Aug 2016 23:34:56 -0500 Subject: [enh] add supported_languages on engines and auto-generate languages.py --- searx/engines/gigablast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx/engines/gigablast.py') diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 5430eb3ba..e139842fa 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -48,7 +48,7 @@ def request(query, params): if params['language'] == 'all': language = 'xx' else: - language = params['language'][0:2] + language = params['language'].split('-')[0] if params['safesearch'] >= 1: safesearch = 1 -- cgit v1.2.3 From c677aee58a4eca1015262eb24530620a333ddcef Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 6 Aug 2016 22:19:21 -0500 Subject: filter langauges --- searx/engines/gigablast.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'searx/engines/gigablast.py') diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index e139842fa..125ffa0a6 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -40,6 +40,12 @@ url_xpath = './/url' title_xpath = './/title' content_xpath = './/sum' +supported_languages = ["en", "fr", "es", "ru", "tr", "ja", "zh-CN", "zh-TW", "ko", "de", + "nl", "it", "fi", "sv", "no", "pt", "vi", "ar", "he", "id", "el", + "th", "hi", "bn", "pl", "tl", "la", "eo", "ca", "bg", "tx", "sr", + "hu", "da", "lt", "cs", "gl", "ka", "gd", "go", "ro", "ga", "lv", + "hy", "is", "ag", "gv", "io", "fa", "te", "vv", "mg", "ku", "lb", "et"] + # do search-request def request(query, params): @@ -48,7 +54,9 @@ def request(query, params): if params['language'] == 'all': language = 'xx' else: - language = params['language'].split('-')[0] + language = params['language'].replace('-', '_').lower() + if language.split('-')[0] != 'zh': + language = language.split('-')[0] if params['safesearch'] >= 1: safesearch = 1 -- cgit v1.2.3 From a11948c71bfe7b2aac6e50e7634874d5073c7d84 Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 29 Oct 2016 21:04:01 -0500 Subject: Add language support for more engines. --- searx/engines/gigablast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx/engines/gigablast.py') diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index 125ffa0a6..f012e1df2 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -44,7 +44,7 @@ supported_languages = ["en", "fr", "es", "ru", "tr", "ja", "zh-CN", "zh-TW", "ko "nl", "it", "fi", "sv", "no", "pt", "vi", "ar", "he", "id", "el", "th", "hi", "bn", "pl", "tl", "la", "eo", "ca", "bg", "tx", "sr", "hu", "da", "lt", "cs", "gl", "ka", "gd", "go", "ro", "ga", "lv", - "hy", "is", "ag", "gv", "io", "fa", "te", "vv", "mg", "ku", "lb", "et"] + "hy", "is", "ag", "gv", "io", "fa", "te", "vv", "mg", "ku", "lb", "et"] # do search-request -- cgit v1.2.3 From f62ce21f50b540315a708ebfbf36878ddec9d1c4 Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 5 Nov 2016 20:51:38 -0600 Subject: [mod] fetch supported languages for several engines utils/fetch_languages.py gets languages supported by each engine and generates engines_languages.json with each engine's supported language. --- searx/engines/gigablast.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'searx/engines/gigablast.py') diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index f012e1df2..e598e55c4 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -14,6 +14,8 @@ from json import loads from random import randint from time import time from urllib import urlencode +from requests import get +from lxml.html import fromstring # engine dependent config categories = ['general'] @@ -40,11 +42,7 @@ url_xpath = './/url' title_xpath = './/title' content_xpath = './/sum' -supported_languages = ["en", "fr", "es", "ru", "tr", "ja", "zh-CN", "zh-TW", "ko", "de", - "nl", "it", "fi", "sv", "no", "pt", "vi", "ar", "he", "id", "el", - "th", "hi", "bn", "pl", "tl", "la", "eo", "ca", "bg", "tx", "sr", - "hu", "da", "lt", "cs", "gl", "ka", "gd", "go", "ro", "ga", "lv", - "hy", "is", "ag", "gv", "io", "fa", "te", "vv", "mg", "ku", "lb", "et"] +supported_languages_url = 'https://gigablast.com/search?&rxikd=1' # do search-request @@ -90,3 +88,17 @@ def response(resp): # return results return results + + +# get supported languages from their site +def fetch_supported_languages(): + supported_languages = [] + response = get(supported_languages_url) + dom = fromstring(response.text) + links = dom.xpath('//span[@id="menu2"]/a') + for link in links: + code = link.xpath('./@href')[0][-2:] + if code != 'xx' and code not in supported_languages: + supported_languages.append(code) + + return supported_languages -- cgit v1.2.3 From af35eee10b98940c51c6e5e18629de514b4bd48d Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 15 Dec 2016 00:34:43 -0600 Subject: tests for _fetch_supported_languages in engines and refactor method to make it testable without making requests --- searx/engines/gigablast.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'searx/engines/gigablast.py') diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py index e598e55c4..827b9cd03 100644 --- a/searx/engines/gigablast.py +++ b/searx/engines/gigablast.py @@ -14,7 +14,6 @@ from json import loads from random import randint from time import time from urllib import urlencode -from requests import get from lxml.html import fromstring # engine dependent config @@ -91,10 +90,9 @@ def response(resp): # get supported languages from their site -def fetch_supported_languages(): +def _fetch_supported_languages(resp): supported_languages = [] - response = get(supported_languages_url) - dom = fromstring(response.text) + dom = fromstring(resp.text) links = dom.xpath('//span[@id="menu2"]/a') for link in links: code = link.xpath('./@href')[0][-2:] -- cgit v1.2.3