Merge pull request #1652 from return42/mod-qwant

Improve qwant engine

1 files changed, 71 insertions, 42 deletions

diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index a1799491a..6de2176d0 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -9,16 +9,16 @@ https://www.qwant.com/ queries.
 This implementation is used by different qwant engines in the settings.yml::
 
   - name: qwant
-    categories: general
+    qwant_categ: web
     ...
   - name: qwant news
-    categories: news
+    qwant_categ: news
     ...
   - name: qwant images
-    categories: images
+    qwant_categ: images
     ...
   - name: qwant videos
-    categories: videos
+    qwant_categ: videos
     ...
 
 """
@@ -30,11 +30,11 @@ from datetime import (
 from json import loads
 from urllib.parse import urlencode
 from flask_babel import gettext
+import babel
 
-from searx.utils import match_language
 from searx.exceptions import SearxEngineAPIException
 from searx.network import raise_for_httperror
-
+from searx.locales import get_engine_locale
 
 # about
 about = {
@@ -50,13 +50,20 @@ about = {
 categories = []
 paging = True
 supported_languages_url = about['website']
+qwant_categ = None  # web|news|inages|videos
 
-category_to_keyword = {
-    'general': 'web',
-    'news': 'news',
-    'images': 'images',
-    'videos': 'videos',
-}
+safesearch = True
+safe_search_map = {0: '&safesearch=0', 1: '&safesearch=1', 2: '&safesearch=2'}
+
+# fmt: off
+qwant_news_locales = [
+    'ca_ad', 'ca_es', 'ca_fr', 'co_fr', 'de_at', 'de_ch', 'de_de', 'en_au',
+    'en_ca', 'en_gb', 'en_ie', 'en_my', 'en_nz', 'en_us', 'es_ad', 'es_ar',
+    'es_cl', 'es_co', 'es_es', 'es_mx', 'es_pe', 'eu_es', 'eu_fr', 'fc_ca',
+    'fr_ad', 'fr_be', 'fr_ca', 'fr_ch', 'fr_fr', 'it_ch', 'it_it', 'nl_be',
+    'nl_nl', 'pt_ad', 'pt_pt',
+]
+# fmt: on
 
 # search-url
 url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}'
@@ -64,10 +71,13 @@ url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={o
 
 def request(query, params):
     """Qwant search request"""
-    keyword = category_to_keyword[categories[0]]
+
+    if not query:
+        return None
+
     count = 10  # web: count must be equal to 10
 
-    if keyword == 'images':
+    if qwant_categ == 'images':
         count = 50
         offset = (params['pageno'] - 1) * count
         # count + offset must be lower than 250
@@ -78,22 +88,18 @@ def request(query, params):
         offset = min(offset, 40)
 
     params['url'] = url.format(
-        keyword=keyword,
+        keyword=qwant_categ,
         query=urlencode({'q': query}),
         offset=offset,
         count=count,
     )
 
-    # add language tag
-    if params['language'] == 'all':
-        params['url'] += '&locale=en_US'
-    else:
-        language = match_language(
-            params['language'],
-            supported_languages,
-            language_aliases,
-        )
-        params['url'] += '&locale=' + language.replace('-', '_')
+    # add quant's locale
+    q_locale = get_engine_locale(params['language'], supported_languages, default='en_US')
+    params['url'] += '&locale=' + q_locale
+
+    # add safesearch option
+    params['url'] += safe_search_map.get(params['safesearch'], '')
 
     params['raise_for_httperror'] = False
     return params
@@ -103,7 +109,6 @@ def response(resp):
     """Get response from Qwant's search request"""
     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 
-    keyword = category_to_keyword[categories[0]]
     results = []
 
     # load JSON result
@@ -125,7 +130,7 @@ def response(resp):
     # raise for other errors
     raise_for_httperror(resp)
 
-    if keyword == 'web':
+    if qwant_categ == 'web':
         # The WEB query contains a list named 'mainline'.  This list can contain
         # different result types (e.g. mainline[0]['type'] returns type of the
         # result items in mainline[0]['items']
@@ -136,7 +141,7 @@ def response(resp):
         # result['items'].
         mainline = data.get('result', {}).get('items', [])
         mainline = [
-            {'type': keyword, 'items': mainline},
+            {'type': qwant_categ, 'items': mainline},
         ]
 
     # return empty array if there are no results
@@ -146,7 +151,7 @@ def response(resp):
     for row in mainline:
 
         mainline_type = row.get('type', 'web')
-        if mainline_type != keyword:
+        if mainline_type != qwant_categ:
             continue
 
         if mainline_type == 'ads':
@@ -238,19 +243,43 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
 def _fetch_supported_languages(resp):
-    # list of regions is embedded in page as a js object
-    response_text = resp.text
-    response_text = response_text[response_text.find('INITIAL_PROPS') :]
-    response_text = response_text[response_text.find('{') : response_text.find('</script>')]
-
-    regions_json = loads(response_text)
-
-    supported_languages = []
-    for country, langs in regions_json['locales'].items():
-        for lang in langs['langs']:
-            lang_code = "{lang}-{country}".format(lang=lang, country=country)
-            supported_languages.append(lang_code)
+
+    text = resp.text
+    text = text[text.find('INITIAL_PROPS') :]
+    text = text[text.find('{') : text.find('</script>')]
+
+    q_initial_props = loads(text)
+    q_locales = q_initial_props.get('locales')
+    q_valid_locales = []
+
+    for country, v in q_locales.items():
+        for lang in v['langs']:
+            _locale = "{lang}_{country}".format(lang=lang, country=country)
+
+            if qwant_categ == 'news' and _locale.lower() not in qwant_news_locales:
+                # qwant-news does not support all locales from qwant-web:
+                continue
+
+            q_valid_locales.append(_locale)
+
+    supported_languages = {}
+
+    for q_locale in q_valid_locales:
+        try:
+            locale = babel.Locale.parse(q_locale, sep='_')
+        except babel.core.UnknownLocaleError:
+            print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
+            continue
+
+        # note: supported_languages (dict)
+        #
+        #   dict's key is a string build up from a babel.Locale object / the
+        #   notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
+        #   language) notation and dict's values are the locale strings used by
+        #   the engine.
+
+        searxng_locale = locale.language + '-' + locale.territory  # --> params['language']
+        supported_languages[searxng_locale] = q_locale
 
     return supported_languages