[fix] qwant - API error::locale must be one ..

The request function should not request a language (aka locale) that is not
supported by qwant. Select a locale like zh-TW ends in qwant's API error:

  ERROR searx.engines.qwant news: exception : \
  API error::locale must be one of the following values: \
    en_gb, en_ie, en_us, en_ca, en_my, en_au, en_nz, de_de, de_ch, de_at, fr_fr, \
    fr_be, fr_ch, fr_ca, fr_ad, fc_ca, co_fr, es_es, es_ar, es_cl, es_co, es_mx, \
    es_pe, es_ad, ca_es, ca_ad, ca_fr, eu_es, eu_fr, it_it, it_ch, pt_pt, pt_ad, \
    nl_be, nl_nl

The existing searx.utils.match_language function is unsuitable for this purpose,
it is replaced by function searx.locales.get_engine_locale that is based on the
methods from the babel package.

The quant's _fetch_supported_languages function has been revised to filter out
languages 8aka locales) not supported by qwant.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

2 files changed, 205 insertions, 189 deletions

diff --git a/searx/data/engines_languages.json b/searx/data/engines_languages.json
index 1d4d9b2e3..3ee6fad79 100644
--- a/searx/data/engines_languages.json
+++ b/searx/data/engines_languages.json
@@ -1396,170 +1396,155 @@
     "sv",
     "zh"
   ],
-  "qwant": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
-  "qwant images": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
-  "qwant news": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
-  "qwant videos": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
+  "qwant": {
+    "bg-BG": "bg_BG",
+    "ca-ES": "ca_ES",
+    "cs-CZ": "cs_CZ",
+    "da-DK": "da_DK",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "el-GR": "el_GR",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "et-EE": "et_EE",
+    "fi-FI": "fi_FI",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "hu-HU": "hu_HU",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "ko-KR": "ko_KR",
+    "nb-NO": "nb_NO",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pl-PL": "pl_PL",
+    "pt-PT": "pt_PT",
+    "ro-RO": "ro_RO",
+    "sv-SE": "sv_SE",
+    "th-TH": "th_TH",
+    "zh-CN": "zh_CN",
+    "zh-HK": "zh_HK"
+  },
+  "qwant images": {
+    "bg-BG": "bg_BG",
+    "ca-ES": "ca_ES",
+    "cs-CZ": "cs_CZ",
+    "da-DK": "da_DK",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "el-GR": "el_GR",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "et-EE": "et_EE",
+    "fi-FI": "fi_FI",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "hu-HU": "hu_HU",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "ko-KR": "ko_KR",
+    "nb-NO": "nb_NO",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pl-PL": "pl_PL",
+    "pt-PT": "pt_PT",
+    "ro-RO": "ro_RO",
+    "sv-SE": "sv_SE",
+    "th-TH": "th_TH",
+    "zh-CN": "zh_CN",
+    "zh-HK": "zh_HK"
+  },
+  "qwant news": {
+    "ca-ES": "ca_ES",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pt-PT": "pt_PT"
+  },
+  "qwant videos": {
+    "bg-BG": "bg_BG",
+    "ca-ES": "ca_ES",
+    "cs-CZ": "cs_CZ",
+    "da-DK": "da_DK",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "el-GR": "el_GR",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "et-EE": "et_EE",
+    "fi-FI": "fi_FI",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "hu-HU": "hu_HU",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "ko-KR": "ko_KR",
+    "nb-NO": "nb_NO",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pl-PL": "pl_PL",
+    "pt-PT": "pt_PT",
+    "ro-RO": "ro_RO",
+    "sv-SE": "sv_SE",
+    "th-TH": "th_TH",
+    "zh-CN": "zh_CN",
+    "zh-HK": "zh_HK"
+  },
   "startpage": {
     "af": {
       "alias": "afrikaans"
diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index e8b1d7739..47956fc97 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -30,11 +30,11 @@ from datetime import (
 from json import loads
 from urllib.parse import urlencode
 from flask_babel import gettext
+import babel
 
-from searx.utils import match_language
 from searx.exceptions import SearxEngineAPIException
 from searx.network import raise_for_httperror
-
+from searx.locales import get_engine_locale
 
 # about
 about = {
@@ -52,12 +52,26 @@ paging = True
 supported_languages_url = about['website']
 qwant_categ = None  # web|news|inages|videos
 
+# fmt: off
+qwant_news_locales = [
+    'ca_ad', 'ca_es', 'ca_fr', 'co_fr', 'de_at', 'de_ch', 'de_de', 'en_au',
+    'en_ca', 'en_gb', 'en_ie', 'en_my', 'en_nz', 'en_us', 'es_ad', 'es_ar',
+    'es_cl', 'es_co', 'es_es', 'es_mx', 'es_pe', 'eu_es', 'eu_fr', 'fc_ca',
+    'fr_ad', 'fr_be', 'fr_ca', 'fr_ch', 'fr_fr', 'it_ch', 'it_it', 'nl_be',
+    'nl_nl', 'pt_ad', 'pt_pt',
+]
+# fmt: on
+
 # search-url
 url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}'
 
 
 def request(query, params):
     """Qwant search request"""
+
+    if not query:
+        return None
+
     count = 10  # web: count must be equal to 10
 
     if qwant_categ == 'images':
@@ -77,16 +91,9 @@ def request(query, params):
         count=count,
     )
 
-    # add language tag
-    if params['language'] == 'all':
-        params['url'] += '&locale=en_US'
-    else:
-        language = match_language(
-            params['language'],
-            supported_languages,
-            language_aliases,
-        )
-        params['url'] += '&locale=' + language.replace('-', '_')
+    # add quant's locale
+    q_locale = get_engine_locale(params['language'], supported_languages, default='en_US')
+    params['url'] += '&locale=' + q_locale
 
     params['raise_for_httperror'] = False
     return params
@@ -230,19 +237,43 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
 def _fetch_supported_languages(resp):
-    # list of regions is embedded in page as a js object
-    response_text = resp.text
-    response_text = response_text[response_text.find('INITIAL_PROPS') :]
-    response_text = response_text[response_text.find('{') : response_text.find('</script>')]
-
-    regions_json = loads(response_text)
-
-    supported_languages = []
-    for country, langs in regions_json['locales'].items():
-        for lang in langs['langs']:
-            lang_code = "{lang}-{country}".format(lang=lang, country=country)
-            supported_languages.append(lang_code)
+
+    text = resp.text
+    text = text[text.find('INITIAL_PROPS') :]
+    text = text[text.find('{') : text.find('</script>')]
+
+    q_initial_props = loads(text)
+    q_locales = q_initial_props.get('locales')
+    q_valid_locales = []
+
+    for country, v in q_locales.items():
+        for lang in v['langs']:
+            _locale = "{lang}_{country}".format(lang=lang, country=country)
+
+            if qwant_categ == 'news' and _locale.lower() not in qwant_news_locales:
+                # qwant-news does not support all locales from qwant-web:
+                continue
+
+            q_valid_locales.append(_locale)
+
+    supported_languages = {}
+
+    for q_locale in q_valid_locales:
+        try:
+            locale = babel.Locale.parse(q_locale, sep='_')
+        except babel.core.UnknownLocaleError:
+            print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
+            continue
+
+        # note: supported_languages (dict)
+        #
+        #   dict's key is a string build up from a babel.Locale object / the
+        #   notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
+        #   language) notation and dict's values are the locale strings used by
+        #   the engine.
+
+        searxng_locale = locale.language + '-' + locale.territory  # --> params['language']
+        supported_languages[searxng_locale] = q_locale
 
     return supported_languages