Merge pull request #1446 from MarcAbonce/language_aliases_fix

[fix] Fix queries in Hebrew and Norwegian so they give results in the right language

9 files changed, 12 insertions, 13 deletions

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index c2f9f3da4..f32b57202 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -113,8 +113,7 @@ def load_engine(engine_data):
                iso_lang not in getattr(engine, 'supported_languages'):
                 language_aliases[iso_lang] = engine_lang
 
-        if language_aliases:
-            setattr(engine, 'language_aliases', language_aliases)
+        setattr(engine, 'language_aliases', language_aliases)
 
     # assign language fetching method if auxiliary method exists
     if hasattr(engine, '_fetch_supported_languages'):
diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py
index 876011f1d..e2495200c 100644
--- a/searx/engines/bing_images.py
+++ b/searx/engines/bing_images.py
@@ -55,7 +55,7 @@ def request(query, params):
         query=urlencode({'q': query}),
         offset=offset)
 
-    language = match_language(params['language'], supported_languages).lower()
+    language = match_language(params['language'], supported_languages, language_aliases).lower()
 
     params['cookies']['SRCHHPGUSR'] = \
         'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py
index 7002ac861..bf17f9168 100644
--- a/searx/engines/bing_videos.py
+++ b/searx/engines/bing_videos.py
@@ -48,7 +48,7 @@ def request(query, params):
         'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
 
     # language cookie
-    language = match_language(params['language'], supported_languages).lower()
+    language = match_language(params['language'], supported_languages, language_aliases).lower()
     params['cookies']['_EDGE_S'] = 'mkt=' + language + '&F=1'
 
     # query and paging
diff --git a/searx/engines/google.py b/searx/engines/google.py
index 62e7d1170..7a4390fc6 100644
--- a/searx/engines/google.py
+++ b/searx/engines/google.py
@@ -166,7 +166,7 @@ def extract_text_from_dom(result, xpath):
 def request(query, params):
     offset = (params['pageno'] - 1) * 10
 
-    language = match_language(params['language'], supported_languages)
+    language = match_language(params['language'], supported_languages, language_aliases)
     language_array = language.split('-')
     if params['language'].find('-') > 0:
         country = params['language'].split('-')[1]
@@ -381,10 +381,10 @@ def attributes_to_html(attributes):
 def _fetch_supported_languages(resp):
     supported_languages = {}
     dom = html.fromstring(resp.text)
-    options = dom.xpath('//table//td/font/label/span')
+    options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')
     for option in options:
-        code = option.xpath('./@id')[0][1:]
-        name = option.text.title()
+        code = option.xpath('./@value')[0].split('_')[-1]
+        name = option.xpath('./@data-name')[0].title()
         supported_languages[code] = {"name": name}
 
     return supported_languages
diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py
index aadcb76df..bf5995c88 100644
--- a/searx/engines/google_news.py
+++ b/searx/engines/google_news.py
@@ -51,7 +51,7 @@ def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       search_options=urlencode(search_options))
 
-    language = match_language(params['language'], supported_languages).split('-')[0]
+    language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
     if language:
         params['url'] += '&lr=lang_' + language
 
diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index 4b0f1c87c..1bef07cc7 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -46,7 +46,7 @@ def request(query, params):
                                    offset=offset)
 
     # add language tag
-    language = match_language(params['language'], supported_languages)
+    language = match_language(params['language'], supported_languages, language_aliases)
     params['url'] += '&locale=' + language.replace('-', '_').lower()
 
     return params
diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py
index ff4df24b7..0001d56bf 100644
--- a/searx/engines/swisscows.py
+++ b/searx/engines/swisscows.py
@@ -36,7 +36,7 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
 
 # do search-request
 def request(query, params):
-    region = match_language(params['language'], supported_languages)
+    region = match_language(params['language'], supported_languages, language_aliases)
     ui_language = region.split('-')[0]
 
     search_path = search_string.format(
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index c315b30da..2485b6528 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -68,7 +68,7 @@ def response(resp):
     html = fromstring(resp.text)
     search_results = html.xpath(wikidata_ids_xpath)
 
-    language = match_language(resp.search_params['language'], supported_languages).split('-')[0]
+    language = match_language(resp.search_params['language'], supported_languages, language_aliases).split('-')[0]
 
     # TODO: make requests asynchronous to avoid timeout when result_count > 1
     for search_result in search_results[:result_count]:
diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index 6cd17e378..23f23db4d 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -31,7 +31,7 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
 
 # set language in base_url
 def url_lang(lang):
-    return match_language(lang, supported_languages).split('-')[0]
+    return match_language(lang, supported_languages, language_aliases).split('-')[0]
 
 
 # do search-request