From 216716db70753cf91e449e4b82d6331e71aa0d5a Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Wed, 14 Mar 2018 23:18:42 -0600
Subject: [fix] change duckduckgo url to avoid error response

---
 searx/engines/duckduckgo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
index 6f8797fed..e51332618 100644
--- a/searx/engines/duckduckgo.py
+++ b/searx/engines/duckduckgo.py
@@ -27,7 +27,7 @@ supported_languages_url = 'https://duckduckgo.com/d2030.js'
 time_range_support = True
 
 # search-url
-url = 'https://duckduckgo.com/html?{query}&s={offset}&api=/d.js&o=json&dc={dc_param}'
+url = 'https://duckduckgo.com/html?{query}&s={offset}&dc={dc_param}'
 time_range_url = '&df={range}'
 
 time_range_dict = {'day': 'd',
-- 
cgit v1.2.3


From d1eae9359f8c5920632a730744ea2208070f06da Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Wed, 14 Feb 2018 16:17:46 -0600
Subject: fix fetch_langauges to be more accurate

Add languages supported by either all default general engines or 10 engines.
---
 searx/engines/duckduckgo.py | 2 +-
 searx/engines/google.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
index e51332618..02ccff8e5 100644
--- a/searx/engines/duckduckgo.py
+++ b/searx/engines/duckduckgo.py
@@ -23,7 +23,7 @@ from searx.url_utils import urlencode
 categories = ['general']
 paging = True
 language_support = True
-supported_languages_url = 'https://duckduckgo.com/d2030.js'
+supported_languages_url = 'https://duckduckgo.com/util/u172.js'
 time_range_support = True
 
 # search-url
diff --git a/searx/engines/google.py b/searx/engines/google.py
index 0a8678362..99c0d2b45 100644
--- a/searx/engines/google.py
+++ b/searx/engines/google.py
@@ -72,7 +72,7 @@ country_to_hostname = {
     'RO': 'www.google.ro',  # Romania
     'RU': 'www.google.ru',  # Russia
     'SK': 'www.google.sk',  # Slovakia
-    'SL': 'www.google.si',  # Slovenia (SL -> si)
+    'SI': 'www.google.si',  # Slovenia
     'SE': 'www.google.se',  # Sweden
     'TH': 'www.google.co.th',  # Thailand
     'TR': 'www.google.com.tr',  # Turkey
-- 
cgit v1.2.3


From d04e471ce53c5efd224a4ed0e7b5d88fb0d3a093 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Wed, 21 Mar 2018 22:30:29 +0100
Subject: add findx engine for general, images and videos

---
 searx/engines/findx.py | 119 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 119 insertions(+)
 create mode 100644 searx/engines/findx.py

(limited to 'searx/engines')

diff --git a/searx/engines/findx.py b/searx/engines/findx.py
new file mode 100644
index 000000000..30a26402a
--- /dev/null
+++ b/searx/engines/findx.py
@@ -0,0 +1,119 @@
+"""
+FindX (General, Images, Videos)
+
+@website     https://www.findx.com
+@provide-api no
+@using-api   no
+@results     HTML
+@stable      no
+@parse       url, title, content, embedded, img_src, thumbnail_src
+"""
+
+from dateutil import parser
+from json import loads
+import re
+
+from lxml import html
+
+from searx import logger
+from searx.engines.xpath import extract_text
+from searx.engines.youtube_noapi import base_youtube_url, embedded_url
+from searx.url_utils import urlencode
+
+
+paging = True
+results_xpath = '//script[@id="initial-state"]'
+search_url = 'https://www.findx.com/{category}?{q}'
+type_map = {
+    'none': 'web',
+    'general': 'web',
+    'images': 'images',
+    'videos': 'videos',
+}
+
+
+def request(query, params):
+    category = 'general'
+    if 'category' in params and len(params['category']) == 1:
+        category = params['category'][0]
+
+    params['url'] = search_url.format(
+        category=type_map[category],
+        q=urlencode({
+            'q': query,
+            'page': params['pageno']
+        })
+    )
+    return params
+
+
+def response(resp):
+    dom = html.fromstring(resp.text)
+    results_raw_json = dom.xpath(results_xpath)
+    results_json = loads(extract_text(results_raw_json))
+
+    if len(results_json['web']['results']) > 0:
+        return _general_results(results_json['web']['results'])
+
+    if len(results_json['images']['results']) > 0:
+        return _images_results(results_json['images']['results'])
+
+    if len(results_json['video']['results']) > 0:
+        return _videos_results(results_json['video']['results'])
+
+    return []
+
+
+def _general_results(general_results):
+    results = []
+    for result in general_results:
+        results.append({
+            'url': result['url'],
+            'title': result['title'],
+            'content': result['sum'],
+        })
+    return results
+
+
+def _images_results(image_results):
+    results = []
+    for result in image_results:
+        results.append({
+            'url': result['sourceURL'],
+            'title': result['title'],
+            'content': result['source'],
+            'thumbnail_src': _extract_url(result['assets']['thumb']['url']),
+            'img_src': _extract_url(result['assets']['file']['url']),
+            'template': 'images.html',
+        })
+    return results
+
+
+def _videos_results(video_results):
+    results = []
+    for result in video_results:
+        if not result['kind'].startswith('youtube'):
+            logger.warn('Unknown video kind in findx: {}'.format(result['kind']))
+            continue
+
+        description = result['snippet']['description']
+        if len(description) > 300:
+            description = description[:300] + '...'
+
+        results.append({
+            'url': base_youtube_url + result['id'],
+            'title': result['snippet']['title'],
+            'content': description,
+            'thumbnail': _extract_url(result['snippet']['thumbnails']['default']['url']),
+            'publishedDate': parser.parse(result['snippet']['publishedAt']),
+            'embedded': embedded_url.format(videoid=result['id']),
+            'template': 'videos.html',
+        })
+    return results
+
+
+def _extract_url(url):
+    matching = re.search('(/https?://[^)]+)', url)
+    if matching:
+        return matching.group(0)[1:]
+    return ''
-- 
cgit v1.2.3


From 8cc529e9a3976e48676676600379ce43f690dd8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Thu, 22 Mar 2018 11:02:24 +0100
Subject: forward category to engine without highlighting on the ui

---
 searx/engines/findx.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/findx.py b/searx/engines/findx.py
index 30a26402a..db4a1aa5f 100644
--- a/searx/engines/findx.py
+++ b/searx/engines/findx.py
@@ -33,12 +33,8 @@ type_map = {
 
 
 def request(query, params):
-    category = 'general'
-    if 'category' in params and len(params['category']) == 1:
-        category = params['category'][0]
-
     params['url'] = search_url.format(
-        category=type_map[category],
+        category=type_map[params['category']],
         q=urlencode({
             'q': query,
             'page': params['pageno']
-- 
cgit v1.2.3


From 772c048d01c7585fd60afca1ce30a1914e6e5b4a Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Wed, 28 Feb 2018 22:30:48 -0600
Subject: refactor engine's search language handling

Add match_language function in utils to match any user given
language code with a list of engine's supported languages.

Also add language_aliases dict on each engine to translate
standard language codes into the custom codes used by the engine.
---
 searx/engines/__init__.py               | 21 ++++++++++++++-
 searx/engines/archlinux.py              |  4 +--
 searx/engines/bing.py                   |  6 +++--
 searx/engines/bing_images.py            | 23 ++--------------
 searx/engines/bing_news.py              |  6 ++---
 searx/engines/bing_videos.py            |  7 ++---
 searx/engines/dailymotion.py            |  3 ++-
 searx/engines/duckduckgo.py             | 47 +++++++++++++--------------------
 searx/engines/duckduckgo_definitions.py |  7 ++---
 searx/engines/duckduckgo_images.py      |  5 +++-
 searx/engines/google.py                 | 29 ++++++++++----------
 searx/engines/google_news.py            |  6 +++--
 searx/engines/qwant.py                  | 13 +++------
 searx/engines/swisscows.py              |  8 +++---
 searx/engines/wikidata.py               |  5 ++--
 searx/engines/wikipedia.py              |  9 ++-----
 searx/engines/yahoo.py                  | 24 ++++++++---------
 searx/engines/yahoo_news.py             |  7 +++--
 18 files changed, 109 insertions(+), 121 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index af3cf8110..c2f9f3da4 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -20,13 +20,14 @@ import sys
 import threading
 from os.path import realpath, dirname
 from io import open
+from babel.localedata import locale_identifiers
 from flask_babel import gettext
 from operator import itemgetter
 from json import loads
 from requests import get
 from searx import settings
 from searx import logger
-from searx.utils import load_module
+from searx.utils import load_module, match_language
 
 
 logger = logger.getChild('engines')
@@ -38,6 +39,8 @@ engines = {}
 categories = {'general': []}
 
 languages = loads(open(engine_dir + '/../data/engines_languages.json', 'r', encoding='utf-8').read())
+babel_langs = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]
+               for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())]
 
 engine_shortcuts = {}
 engine_default_args = {'paging': False,
@@ -97,6 +100,22 @@ def load_engine(engine_data):
     if engine_data['name'] in languages:
         setattr(engine, 'supported_languages', languages[engine_data['name']])
 
+    # find custom aliases for non standard language codes
+    if hasattr(engine, 'supported_languages'):
+        if hasattr(engine, 'language_aliases'):
+            language_aliases = getattr(engine, 'language_aliases')
+        else:
+            language_aliases = {}
+
+        for engine_lang in getattr(engine, 'supported_languages'):
+            iso_lang = match_language(engine_lang, babel_langs, fallback=None)
+            if iso_lang and iso_lang != engine_lang and not engine_lang.startswith(iso_lang) and \
+               iso_lang not in getattr(engine, 'supported_languages'):
+                language_aliases[iso_lang] = engine_lang
+
+        if language_aliases:
+            setattr(engine, 'language_aliases', language_aliases)
+
     # assign language fetching method if auxiliary method exists
     if hasattr(engine, '_fetch_supported_languages'):
         setattr(engine, 'fetch_supported_languages',
diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py
index 245bc50b2..fc08112af 100644
--- a/searx/engines/archlinux.py
+++ b/searx/engines/archlinux.py
@@ -99,13 +99,13 @@ supported_languages = dict(lang_urls, **main_langs)
 
 # do search-request
 def request(query, params):
-    # translate the locale (e.g. 'en_US') to language code ('en')
+    # translate the locale (e.g. 'en-US') to language code ('en')
     language = locale_to_lang_code(params['language'])
 
     # if our language is hosted on the main site, we need to add its name
     # to the query in order to narrow the results to that language
     if language in main_langs:
-        query += '(' + main_langs[language] + ')'
+        query += b' (' + main_langs[language] + b')'
 
     # prepare the request parameters
     query = urlencode({'search': query})
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 2e58d0293..c6d41782b 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -16,12 +16,14 @@
 from lxml import html
 from searx.engines.xpath import extract_text
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = ['general']
 paging = True
 language_support = True
 supported_languages_url = 'https://www.bing.com/account/general'
+language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'}
 
 # search-url
 base_url = 'https://www.bing.com/'
@@ -32,9 +34,9 @@ search_string = 'search?{query}&first={offset}'
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    lang = params['language'].split('-')[0].upper()
+    lang = match_language(params['language'], supported_languages, language_aliases)
 
-    query = u'language:{} {}'.format(lang, query.decode('utf-8')).encode('utf-8')
+    query = u'language:{} {}'.format(lang.split('-')[0].upper(), query.decode('utf-8')).encode('utf-8')
 
     search_path = search_string.format(
         query=urlencode({'q': query}),
diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py
index 15679056c..66e14c01f 100644
--- a/searx/engines/bing_images.py
+++ b/searx/engines/bing_images.py
@@ -19,6 +19,7 @@ from lxml import html
 from json import loads
 import re
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = ['images']
@@ -46,26 +47,6 @@ safesearch_types = {2: 'STRICT',
 _quote_keys_regex = re.compile('({|,)([a-z][a-z0-9]*):(")', re.I | re.U)
 
 
-# get supported region code
-def get_region_code(lang, lang_list=None):
-    region = None
-    if lang in (lang_list or supported_languages):
-        region = lang
-    elif lang.startswith('no'):
-        region = 'nb-NO'
-    else:
-        # try to get a supported country code with language
-        lang = lang.split('-')[0]
-        for lc in (lang_list or supported_languages):
-            if lang == lc.split('-')[0]:
-                region = lc
-                break
-    if region:
-        return region.lower()
-    else:
-        return 'en-us'
-
-
 # do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
@@ -74,7 +55,7 @@ def request(query, params):
         query=urlencode({'q': query}),
         offset=offset)
 
-    language = get_region_code(params['language'])
+    language = match_language(params['language'], supported_languages).lower()
 
     params['cookies']['SRCHHPGUSR'] = \
         'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py
index c609a1949..39048a1fc 100644
--- a/searx/engines/bing_news.py
+++ b/searx/engines/bing_news.py
@@ -14,8 +14,8 @@
 from datetime import datetime
 from dateutil import parser
 from lxml import etree
-from searx.utils import list_get
-from searx.engines.bing import _fetch_supported_languages, supported_languages_url
+from searx.utils import list_get, match_language
+from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases
 from searx.url_utils import urlencode, urlparse, parse_qsl
 
 # engine dependent config
@@ -71,7 +71,7 @@ def request(query, params):
 
     offset = (params['pageno'] - 1) * 10 + 1
 
-    language = params['language']
+    language = match_language(params['language'], supported_languages, language_aliases)
 
     params['url'] = _get_url(query, language, offset, params['time_range'])
 
diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py
index 312a82ba1..7002ac861 100644
--- a/searx/engines/bing_videos.py
+++ b/searx/engines/bing_videos.py
@@ -12,9 +12,10 @@
 
 from json import loads
 from lxml import html
-from searx.engines.bing_images import _fetch_supported_languages, supported_languages_url, get_region_code
+from searx.engines.bing_images import _fetch_supported_languages, supported_languages_url
 from searx.engines.xpath import extract_text
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 
 categories = ['videos']
@@ -47,8 +48,8 @@ def request(query, params):
         'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')
 
     # language cookie
-    region = get_region_code(params['language'], lang_list=supported_languages)
-    params['cookies']['_EDGE_S'] = 'mkt=' + region + '&F=1'
+    language = match_language(params['language'], supported_languages).lower()
+    params['cookies']['_EDGE_S'] = 'mkt=' + language + '&F=1'
 
     # query and paging
     params['url'] = search_url.format(query=urlencode({'q': query}),
diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py
index cfa76796d..8268b6257 100644
--- a/searx/engines/dailymotion.py
+++ b/searx/engines/dailymotion.py
@@ -15,6 +15,7 @@
 from json import loads
 from datetime import datetime
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = ['videos']
@@ -32,7 +33,7 @@ supported_languages_url = 'https://api.dailymotion.com/languages'
 
 # do search-request
 def request(query, params):
-    locale = params['language']
+    locale = match_language(params['language'], supported_languages)
 
     params['url'] = search_url.format(
         query=urlencode({'search': query, 'localization': locale}),
diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
index 02ccff8e5..2c5dc50d8 100644
--- a/searx/engines/duckduckgo.py
+++ b/searx/engines/duckduckgo.py
@@ -18,6 +18,7 @@ from json import loads
 from searx.engines.xpath import extract_text
 from searx.poolrequests import get
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = ['general']
@@ -26,6 +27,16 @@ language_support = True
 supported_languages_url = 'https://duckduckgo.com/util/u172.js'
 time_range_support = True
 
+language_aliases = {
+    'ar-SA': 'ar-XA',
+    'es-419': 'es-XL',
+    'ja': 'jp-JP',
+    'ko': 'kr-KR',
+    'sl-SI': 'sl-SL',
+    'zh-TW': 'tzh-TW',
+    'zh-HK': 'tzh-HK'
+}
+
 # search-url
 url = 'https://duckduckgo.com/html?{query}&s={offset}&dc={dc_param}'
 time_range_url = '&df={range}'
@@ -42,34 +53,12 @@ content_xpath = './/a[@class="result__snippet"]'
 
 
 # match query's language to a region code that duckduckgo will accept
-def get_region_code(lang, lang_list=None):
-    # custom fixes for languages
-    if lang[:2] == 'ja':
-        region_code = 'jp-jp'
-    elif lang[:2] == 'sl':
-        region_code = 'sl-sl'
-    elif lang == 'zh-TW':
-        region_code = 'tw-tzh'
-    elif lang == 'zh-HK':
-        region_code = 'hk-tzh'
-    elif lang[-2:] == 'SA':
-        region_code = 'xa-' + lang.split('-')[0]
-    elif lang[-2:] == 'GB':
-        region_code = 'uk-' + lang.split('-')[0]
-    else:
-        region_code = lang.split('-')
-        if len(region_code) == 2:
-            # country code goes first
-            region_code = region_code[1].lower() + '-' + region_code[0].lower()
-        else:
-            # tries to get a country code from language
-            region_code = region_code[0].lower()
-            for lc in (lang_list or supported_languages):
-                lc = lc.split('-')
-                if region_code == lc[0]:
-                    region_code = lc[1].lower() + '-' + lc[0].lower()
-                    break
-    return region_code
+def get_region_code(lang, lang_list=[]):
+    lang_code = match_language(lang, lang_list, language_aliases, 'wt-WT')
+    lang_parts = lang_code.split('-')
+
+    # country code goes first
+    return lang_parts[1].lower() + '-' + lang_parts[0].lower()
 
 
 # do search-request
@@ -79,7 +68,7 @@ def request(query, params):
 
     offset = (params['pageno'] - 1) * 30
 
-    region_code = get_region_code(params['language'])
+    region_code = get_region_code(params['language'], supported_languages)
     params['url'] = url.format(
         query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
 
diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py
index 21c6a6578..957a13ea6 100644
--- a/searx/engines/duckduckgo_definitions.py
+++ b/searx/engines/duckduckgo_definitions.py
@@ -2,9 +2,9 @@ import json
 from lxml import html
 from re import compile
 from searx.engines.xpath import extract_text
-from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url
+from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, language_aliases
 from searx.url_utils import urlencode
-from searx.utils import html_to_text
+from searx.utils import html_to_text, match_language
 
 url = 'https://api.duckduckgo.com/'\
     + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
@@ -24,7 +24,8 @@ def result_to_text(url, text, htmlResult):
 
 def request(query, params):
     params['url'] = url.format(query=urlencode({'q': query}))
-    params['headers']['Accept-Language'] = params['language'].split('-')[0]
+    language = match_language(params['language'], supported_languages, language_aliases)
+    params['headers']['Accept-Language'] = language.split('-')[0]
     return params
 
 
diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py
index 7b0e72694..7905d0bcd 100644
--- a/searx/engines/duckduckgo_images.py
+++ b/searx/engines/duckduckgo_images.py
@@ -15,7 +15,10 @@
 
 from json import loads
 from searx.engines.xpath import extract_text
-from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, get_region_code
+from searx.engines.duckduckgo import (
+    _fetch_supported_languages, supported_languages_url,
+    get_region_code, language_aliases
+)
 from searx.poolrequests import get
 from searx.url_utils import urlencode
 
diff --git a/searx/engines/google.py b/searx/engines/google.py
index 99c0d2b45..93075e2dc 100644
--- a/searx/engines/google.py
+++ b/searx/engines/google.py
@@ -14,6 +14,7 @@ from lxml import html, etree
 from searx.engines.xpath import extract_text, extract_url
 from searx import logger
 from searx.url_utils import urlencode, urlparse, parse_qsl
+from searx.utils import match_language
 
 logger = logger.getChild('google engine')
 
@@ -165,22 +166,20 @@ def extract_text_from_dom(result, xpath):
 def request(query, params):
     offset = (params['pageno'] - 1) * 10
 
+    language = match_language(params['language'], supported_languages)
+    language_array = language.split('-')
+    if params['language'].find('-') > 0:
+        country = params['language'].split('-')[1]
+    elif len(language_array) == 2:
+        country = language_array[1]
+    else:
+        country = 'US'
+
     # temporary fix until a way of supporting en-US is found
-    if params['language'] == 'en-US':
-        params['language'] = 'en-GB'
+    if language == 'en-US':
+        country = 'GB'
 
-    if params['language'][:2] == 'jv':
-        language = 'jw'
-        country = 'ID'
-        url_lang = 'lang_jw'
-    else:
-        language_array = params['language'].lower().split('-')
-        if len(language_array) == 2:
-            country = language_array[1]
-        else:
-            country = 'US'
-        language = language_array[0] + ',' + language_array[0] + '-' + country
-        url_lang = 'lang_' + language_array[0]
+    url_lang = 'lang_' + language
 
     if use_locale_domain:
         google_hostname = country_to_hostname.get(country.upper(), default_hostname)
@@ -196,7 +195,7 @@ def request(query, params):
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_search.format(range=time_range_dict[params['time_range']])
 
-    params['headers']['Accept-Language'] = language
+    params['headers']['Accept-Language'] = language + ',' + language + '-' + country
     params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
 
     params['google_hostname'] = google_hostname
diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py
index 8b8e7175d..41abf0a01 100644
--- a/searx/engines/google_news.py
+++ b/searx/engines/google_news.py
@@ -13,6 +13,7 @@
 from lxml import html
 from searx.engines.google import _fetch_supported_languages, supported_languages_url
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 # search-url
 categories = ['news']
@@ -50,8 +51,9 @@ def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       search_options=urlencode(search_options))
 
-    language_array = params['language'].lower().split('-')
-    params['url'] += '&lr=lang_' + language_array[0]
+    language = match_language(params['language'], supported_languages).split('-')[0]
+    if language:
+        params['url'] += '&lr=lang_' + language
 
     return params
 
diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index 408c2b3de..239193b96 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -14,6 +14,7 @@ from datetime import datetime
 from json import loads
 from searx.utils import html_to_text
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = None
@@ -45,16 +46,8 @@ def request(query, params):
                                    offset=offset)
 
     # add language tag
-    if params['language'] == 'no' or params['language'].startswith('no-'):
-        params['language'] = params['language'].replace('no', 'nb', 1)
-    if params['language'].find('-') < 0:
-        # tries to get a country code from language
-        for lang in supported_languages:
-            lc = lang.split('-')
-            if params['language'] == lc[0]:
-                params['language'] = lang
-                break
-    params['url'] += '&locale=' + params['language'].replace('-', '_').lower()
+    language = match_language(params['language'], supported_languages)
+    params['url'] += '&locale=' + language.replace('-', '_').lower()
 
     return params
 
diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py
index 45e9d87a9..ff4df24b7 100644
--- a/searx/engines/swisscows.py
+++ b/searx/engines/swisscows.py
@@ -14,6 +14,7 @@ from json import loads
 import re
 from lxml.html import fromstring
 from searx.url_utils import unquote, urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = ['general', 'images']
@@ -35,11 +36,8 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
 
 # do search-request
 def request(query, params):
-    if params['language'].split('-')[0] == 'no':
-        region = 'nb-NO'
-    else:
-        region = params['language']
-        ui_language = params['language'].split('-')[0]
+    region = match_language(params['language'], supported_languages)
+    ui_language = region.split('-')[0]
 
     search_path = search_string.format(
         query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index 1f31a1f88..1fdbc9869 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -16,6 +16,7 @@ from searx.poolrequests import get
 from searx.engines.xpath import extract_text
 from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 from json import loads
 from lxml.html import fromstring
@@ -56,7 +57,7 @@ calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
 
 
 def request(query, params):
-    language = params['language'].split('-')[0]
+    language = match_language(params['language'], supported_languages).split('-')[0]
 
     params['url'] = url_search.format(
         query=urlencode({'label': query, 'language': language}))
@@ -68,7 +69,7 @@ def response(resp):
     html = fromstring(resp.text)
     wikidata_ids = html.xpath(wikidata_ids_xpath)
 
-    language = resp.search_params['language'].split('-')[0]
+    language = match_language(resp.search_params['language'], supported_languages).split('-')[0]
 
     # TODO: make requests asynchronous to avoid timeout when result_count > 1
     for wikidata_id in wikidata_ids[:result_count]:
diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index fe82f5115..6cd17e378 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -13,6 +13,7 @@
 from json import loads
 from lxml.html import fromstring
 from searx.url_utils import quote, urlencode
+from searx.utils import match_language
 
 # search-url
 base_url = u'https://{language}.wikipedia.org/'
@@ -30,13 +31,7 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
 
 # set language in base_url
 def url_lang(lang):
-    lang = lang.split('-')[0]
-    if lang not in supported_languages:
-        language = 'en'
-    else:
-        language = lang
-
-    return language
+    return match_language(lang, supported_languages).split('-')[0]
 
 
 # do search-request
diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py
index 626a398b5..ba4cb6af8 100644
--- a/searx/engines/yahoo.py
+++ b/searx/engines/yahoo.py
@@ -14,6 +14,7 @@
 from lxml import html
 from searx.engines.xpath import extract_text, extract_url
 from searx.url_utils import unquote, urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = ['general']
@@ -39,6 +40,8 @@ time_range_dict = {'day': ['1d', 'd'],
                    'week': ['1w', 'w'],
                    'month': ['1m', 'm']}
 
+language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'}
+
 
 # remove yahoo-specific tracking-url
 def parse_url(url_string):
@@ -70,23 +73,16 @@ def _get_url(query, offset, language, time_range):
                                         lang=language)
 
 
-def _get_language(params):
-    if params['language'][:2] == 'zh':
-        if params['language'] == 'zh' or params['language'] == 'zh-CH':
-            return 'szh'
-        else:
-            return 'tzh'
-    else:
-        return params['language'].split('-')[0]
-
-
 # do search-request
 def request(query, params):
     if params['time_range'] and params['time_range'] not in time_range_dict:
         return params
 
     offset = (params['pageno'] - 1) * 10 + 1
-    language = _get_language(params)
+    language = match_language(params['language'], supported_languages, language_aliases)
+    if language not in language_aliases.values():
+        language = language.split('-')[0]
+    language = language.replace('-', '_').lower()
 
     params['url'] = _get_url(query, offset, language, params['time_range'])
 
@@ -145,7 +141,11 @@ def _fetch_supported_languages(resp):
     dom = html.fromstring(resp.text)
     options = dom.xpath('//div[@id="yschlang"]/span/label/input')
     for option in options:
-        code = option.xpath('./@value')[0][5:].replace('_', '-')
+        code_parts = option.xpath('./@value')[0][5:].split('_')
+        if len(code_parts) == 2:
+            code = code_parts[0] + '-' + code_parts[1].upper()
+        else:
+            code = code_parts[0]
         supported_languages.append(code)
 
     return supported_languages
diff --git a/searx/engines/yahoo_news.py b/searx/engines/yahoo_news.py
index 69e9aef4f..b61384d06 100644
--- a/searx/engines/yahoo_news.py
+++ b/searx/engines/yahoo_news.py
@@ -13,9 +13,12 @@ import re
 from datetime import datetime, timedelta
 from lxml import html
 from searx.engines.xpath import extract_text, extract_url
-from searx.engines.yahoo import parse_url, _fetch_supported_languages, supported_languages_url
+from searx.engines.yahoo import (
+    parse_url, _fetch_supported_languages, supported_languages_url, language_aliases
+)
 from dateutil import parser
 from searx.url_utils import urlencode
+from searx.utils import match_language
 
 # engine dependent config
 categories = ['news']
@@ -38,7 +41,7 @@ suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a'
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    language = params['language'].split('-')[0]
+    language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
 
     params['url'] = search_url.format(offset=offset,
                                       query=urlencode({'p': query}),
-- 
cgit v1.2.3


From 343e555ee96ab8b40b5ed4dce287331685eff3fa Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sun, 8 Apr 2018 20:35:34 -0500
Subject: [fix] append http if no scheme is provided in xpath's extact_url

This solves a bug with Yahoo where some results don't specify
a protocol.
---
 searx/engines/xpath.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py
index c8c56da44..50f98d935 100644
--- a/searx/engines/xpath.py
+++ b/searx/engines/xpath.py
@@ -53,7 +53,7 @@ def extract_url(xpath_results, search_url):
     if url.startswith('//'):
         # add http or https to this kind of url //example.com/
         parsed_search_url = urlparse(search_url)
-        url = u'{0}:{1}'.format(parsed_search_url.scheme, url)
+        url = u'{0}:{1}'.format(parsed_search_url.scheme or 'http', url)
     elif url.startswith('/'):
         # fix relative url to the search engine
         url = urljoin(search_url, url)
-- 
cgit v1.2.3


From 835d1edd5834c3c8117dc4614cb0b0b4316d3153 Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sun, 8 Apr 2018 20:56:05 -0500
Subject: [fix] google news xpath

---
 searx/engines/google_news.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py
index 41abf0a01..aadcb76df 100644
--- a/searx/engines/google_news.py
+++ b/searx/engines/google_news.py
@@ -68,8 +68,8 @@ def response(resp):
     for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'):
         try:
             r = {
-                'url': result.xpath('.//a[@class="l _PMs"]')[0].attrib.get("href"),
-                'title': ''.join(result.xpath('.//a[@class="l _PMs"]//text()')),
+                'url': result.xpath('.//a[@class="l lLrAF"]')[0].attrib.get("href"),
+                'title': ''.join(result.xpath('.//a[@class="l lLrAF"]//text()')),
                 'content': ''.join(result.xpath('.//div[@class="st"]//text()')),
             }
         except:
-- 
cgit v1.2.3


From b12857a70dd947a804e667d864ba56055b528ee0 Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sun, 8 Apr 2018 21:17:00 -0500
Subject: [fix] make search requests on wikidata more accurate

---
 searx/engines/wikidata.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index 1fdbc9869..fe53609c1 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -27,7 +27,7 @@ result_count = 1
 # urls
 wikidata_host = 'https://www.wikidata.org'
 url_search = wikidata_host \
-    + '/wiki/Special:ItemDisambiguation?{query}'
+    + '/w/index.php?{query}'
 
 wikidata_api = wikidata_host + '/w/api.php'
 url_detail = wikidata_api\
@@ -40,7 +40,7 @@ url_map = 'https://www.openstreetmap.org/'\
 url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
 
 # xpaths
-wikidata_ids_xpath = '//div/ul[@class="wikibase-disambiguation"]/li/a/@title'
+wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'
 title_xpath = '//*[contains(@class,"wikibase-title-label")]'
 description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'
 property_xpath = '//div[@id="{propertyid}"]'
@@ -57,22 +57,21 @@ calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
 
 
 def request(query, params):
-    language = match_language(params['language'], supported_languages).split('-')[0]
-
     params['url'] = url_search.format(
-        query=urlencode({'label': query, 'language': language}))
+        query=urlencode({'search': query}))
     return params
 
 
 def response(resp):
     results = []
     html = fromstring(resp.text)
-    wikidata_ids = html.xpath(wikidata_ids_xpath)
+    search_results = html.xpath(wikidata_ids_xpath)
 
     language = match_language(resp.search_params['language'], supported_languages).split('-')[0]
 
     # TODO: make requests asynchronous to avoid timeout when result_count > 1
-    for wikidata_id in wikidata_ids[:result_count]:
+    for search_result in search_results[:result_count]:
+        wikidata_id = search_result.split('/')[-1]
         url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
         htmlresponse = get(url)
         jsonresponse = loads(htmlresponse.text)
-- 
cgit v1.2.3


From f7f9c50393785c8d6ad6c40a4c507b292ea438b1 Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Wed, 18 Apr 2018 22:55:37 -0500
Subject: [fix] force English results in Google when using en-US

---
 searx/engines/google.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/google.py b/searx/engines/google.py
index 93075e2dc..62e7d1170 100644
--- a/searx/engines/google.py
+++ b/searx/engines/google.py
@@ -91,7 +91,7 @@ url_map = 'https://www.openstreetmap.org/'\
 search_path = '/search'
 search_url = ('https://{hostname}' +
               search_path +
-              '?{query}&start={offset}&gws_rd=cr&gbv=1&lr={lang}&ei=x')
+              '?{query}&start={offset}&gws_rd=cr&gbv=1&lr={lang}&hl={lang_short}&ei=x')
 
 time_range_search = "&tbs=qdr:{range}"
 time_range_dict = {'day': 'd',
@@ -175,10 +175,6 @@ def request(query, params):
     else:
         country = 'US'
 
-    # temporary fix until a way of supporting en-US is found
-    if language == 'en-US':
-        country = 'GB'
-
     url_lang = 'lang_' + language
 
     if use_locale_domain:
@@ -191,7 +187,8 @@ def request(query, params):
     params['url'] = search_url.format(offset=offset,
                                       query=urlencode({'q': query}),
                                       hostname=google_hostname,
-                                      lang=url_lang)
+                                      lang=url_lang,
+                                      lang_short=language)
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_search.format(range=time_range_dict[params['time_range']])
 
-- 
cgit v1.2.3


From 25b48143ea6385a3dbb3003d0a39cbaa301e703b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Sat, 21 Apr 2018 20:02:09 +0200
Subject: fix findx engine

---
 searx/engines/findx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/findx.py b/searx/engines/findx.py
index db4a1aa5f..87c9d503c 100644
--- a/searx/engines/findx.py
+++ b/searx/engines/findx.py
@@ -49,7 +49,7 @@ def response(resp):
     results_json = loads(extract_text(results_raw_json))
 
     if len(results_json['web']['results']) > 0:
-        return _general_results(results_json['web']['results'])
+        return _general_results(results_json['web']['results']['webSearch']['results'])
 
     if len(results_json['images']['results']) > 0:
         return _images_results(results_json['images']['results'])
-- 
cgit v1.2.3


From a305dae9d88fe3c37ef18e4ce5cbe75f07c40a45 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Sun, 22 Apr 2018 13:46:13 +0300
Subject: switched currency convert to duckduckgo and added Bitcoin to currency
 list

---
 searx/engines/currency_convert.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py
index 9c1c2f7b3..26e716b32 100644
--- a/searx/engines/currency_convert.py
+++ b/searx/engines/currency_convert.py
@@ -11,7 +11,7 @@ if sys.version_info[0] == 3:
     unicode = str
 
 categories = []
-url = 'https://finance.google.com/finance/converter?a=1&from={0}&to={1}'
+url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'
 weight = 100
 
 parser_re = re.compile(b'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
@@ -63,16 +63,13 @@ def request(query, params):
 
 
 def response(resp):
+    """remove first and last lines to get only json"""
+    json_resp = resp.text[resp.text.find('\n')+1:resp.text.rfind('\n')-2]
     results = []
-    pat = '<span class=bld>(.+) {0}</span>'.format(
-        resp.search_params['to'].upper())
-
     try:
-        conversion_rate = re.findall(pat, resp.text)[0]
-        conversion_rate = float(conversion_rate)
+        conversion_rate = float(json.loads(json_resp)['conversion']['converted-amount'])
     except:
         return results
-
     answer = '{0} {1} = {2} {3}, 1 {1} ({5}) = {4} {3} ({6})'.format(
         resp.search_params['amount'],
         resp.search_params['from'],
@@ -83,7 +80,7 @@ def response(resp):
         resp.search_params['to_name'],
     )
 
-    url = 'https://finance.google.com/finance?q={0}{1}'.format(
+    url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'.format(
         resp.search_params['from'].upper(), resp.search_params['to'])
 
     results.append({'answer': answer, 'url': url})
-- 
cgit v1.2.3


From cd1374d2f2859005743e31eb7cff154fe4dd2fc4 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Sun, 22 Apr 2018 14:12:32 +0300
Subject: added whitespaces

---
 searx/engines/currency_convert.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py
index 26e716b32..8eab8f673 100644
--- a/searx/engines/currency_convert.py
+++ b/searx/engines/currency_convert.py
@@ -44,7 +44,6 @@ def request(query, params):
     if not m:
         # wrong query
         return params
-
     amount, from_currency, to_currency = m.groups()
     amount = float(amount)
     from_currency = name_to_iso4217(from_currency.strip())
@@ -64,7 +63,7 @@ def request(query, params):
 
 def response(resp):
     """remove first and last lines to get only json"""
-    json_resp = resp.text[resp.text.find('\n')+1:resp.text.rfind('\n')-2]
+    json_resp = resp.text[resp.text.find('\n') + 1:resp.text.rfind('\n') - 2]
     results = []
     try:
         conversion_rate = float(json.loads(json_resp)['conversion']['converted-amount'])
-- 
cgit v1.2.3


From 3ec35a06f7840f6e387eb4f518f6673394efb21a Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Thu, 26 Apr 2018 22:42:31 +0300
Subject: Added Engine

---
 searx/engines/acgsou.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 searx/engines/acgsou.py

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
new file mode 100644
index 000000000..10655cb84
--- /dev/null
+++ b/searx/engines/acgsou.py
@@ -0,0 +1,85 @@
+"""
+ Acgsou (Japanese Animation/Music/Comics Bittorrent tracker)
+
+ @website      https://www.acgsou.com/
+ @provide-api  no
+ @using-api    no
+ @results      HTML
+ @stable       no (HTML can change)
+ @parse        url, title, content, seed, leech, torrentfile
+"""
+
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
+from searx.utils import get_torrent_size, int_or_zero
+
+# engine dependent config
+categories = ['files', 'images', 'videos', 'music']
+paging = True
+
+# search-url
+base_url = 'https://www.acgsou.com/'
+search_url = base_url + 'search.php?{query}&page={offset}'
+# xpath queries
+xpath_results = '//table[contains(@class, "list_style table_fixed")]//tr[not(th)]'
+xpath_category = './/td[2]/a[1]'
+xpath_title = './/td[3]/a[last()]'
+xpath_torrent_links = './/td[3]/a'
+xpath_filesize = './/td[4]/text()'
+
+# do search-request
+def request(query, params):
+    query = urlencode({'keyword': query})
+    params['url'] = search_url.format(query=query, offset=params['pageno'])
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = [] 
+    dom = html.fromstring(resp.text)
+    print(resp.text)
+    for result in dom.xpath(xpath_results): 
+        # defaults
+        filesize = 0
+        magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce"
+        torrent_link = ""
+
+        # category in which our torrent belongs
+        try:
+            category = extract_text(result.xpath(xpath_category)[0])
+        except:
+            pass
+
+        # torrent title
+        page_a = result.xpath(xpath_title)[0]
+        title = extract_text(page_a)
+
+        # link to the page
+        href = base_url + page_a.attrib.get('href')
+        
+        #magnet link
+        magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5])
+
+        # let's try to calculate the torrent size
+        try:
+            filesize_info = result.xpath(xpath_filesize)[0]
+            filesize = filesize_info[:-2]
+            filesize_multiplier = filesize_info[-2:]
+            filesize = get_torrent_size(filesize, filesize_multiplier)
+        except :
+            pass
+
+        # content string contains all information not included into template
+        content = 'Category: "{category}".'
+        content = content.format(category=category)
+
+        results.append({'url': href,
+                        'title': title,
+                        'content': content,
+                        'filesize': filesize,
+                        'magnetlink': magnet_link,
+                        'template': 'torrent.html'})
+
+    return results
-- 
cgit v1.2.3


From fb364ffae7f5a41ab4b6b4ecbc2a9194da532a5c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Fri, 27 Apr 2018 08:23:58 +0300
Subject: Hopefully fix code style errors

---
 searx/engines/acgsou.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index 10655cb84..ad4234571 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -28,14 +28,11 @@ xpath_title = './/td[3]/a[last()]'
 xpath_torrent_links = './/td[3]/a'
 xpath_filesize = './/td[4]/text()'
 
-# do search-request
 def request(query, params):
     query = urlencode({'keyword': query})
     params['url'] = search_url.format(query=query, offset=params['pageno'])
     return params
 
-
-# get response from search-request
 def response(resp):
     results = [] 
     dom = html.fromstring(resp.text)
@@ -46,23 +43,17 @@ def response(resp):
         magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce"
         torrent_link = ""
 
-        # category in which our torrent belongs
         try:
             category = extract_text(result.xpath(xpath_category)[0])
         except:
             pass
 
-        # torrent title
         page_a = result.xpath(xpath_title)[0]
         title = extract_text(page_a)
-
-        # link to the page
         href = base_url + page_a.attrib.get('href')
-        
-        #magnet link
+
         magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5])
 
-        # let's try to calculate the torrent size
         try:
             filesize_info = result.xpath(xpath_filesize)[0]
             filesize = filesize_info[:-2]
@@ -70,16 +61,14 @@ def response(resp):
             filesize = get_torrent_size(filesize, filesize_multiplier)
         except :
             pass
-
-        # content string contains all information not included into template
+        #I didn't add download/seed/leech count since as I figured out they are generated randowmly everytime
         content = 'Category: "{category}".'
         content = content.format(category=category)
-
+        
         results.append({'url': href,
                         'title': title,
                         'content': content,
                         'filesize': filesize,
                         'magnetlink': magnet_link,
                         'template': 'torrent.html'})
-
     return results
-- 
cgit v1.2.3


From b8b23d800640c3fb07e0110008ddb633277b045e Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Fri, 27 Apr 2018 08:29:17 +0300
Subject: Hopefully fix code style errors(again)

---
 searx/engines/acgsou.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index ad4234571..5a1571011 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -28,16 +28,18 @@ xpath_title = './/td[3]/a[last()]'
 xpath_torrent_links = './/td[3]/a'
 xpath_filesize = './/td[4]/text()'
 
+
 def request(query, params):
     query = urlencode({'keyword': query})
     params['url'] = search_url.format(query=query, offset=params['pageno'])
     return params
 
+
 def response(resp):
     results = [] 
     dom = html.fromstring(resp.text)
     print(resp.text)
-    for result in dom.xpath(xpath_results): 
+    for result in dom.xpath(xpath_results):
         # defaults
         filesize = 0
         magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce"
@@ -59,12 +61,12 @@ def response(resp):
             filesize = filesize_info[:-2]
             filesize_multiplier = filesize_info[-2:]
             filesize = get_torrent_size(filesize, filesize_multiplier)
-        except :
+        except:
             pass
         #I didn't add download/seed/leech count since as I figured out they are generated randowmly everytime
         content = 'Category: "{category}".'
         content = content.format(category=category)
-        
+
         results.append({'url': href,
                         'title': title,
                         'content': content,
-- 
cgit v1.2.3


From 86f47689fea8df37571e8413223fe1a3847fb0b8 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Fri, 27 Apr 2018 08:32:43 +0300
Subject: Hopefully fix code style errors(again)(again)

---
 searx/engines/acgsou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index 5a1571011..73f0361e5 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -63,7 +63,7 @@ def response(resp):
             filesize = get_torrent_size(filesize, filesize_multiplier)
         except:
             pass
-        #I didn't add download/seed/leech count since as I figured out they are generated randowmly everytime
+        # I didn't add download/seed/leech count since as I figured out they are generated randowmly everytime
         content = 'Category: "{category}".'
         content = content.format(category=category)
 
-- 
cgit v1.2.3


From e3bd2670d7d886ba7fb7540781fec130d1e80c9c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Fri, 27 Apr 2018 08:36:11 +0300
Subject: Traling whitespace

---
 searx/engines/acgsou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index 73f0361e5..ebe253649 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -36,7 +36,7 @@ def request(query, params):
 
 
 def response(resp):
-    results = [] 
+    results = []
     dom = html.fromstring(resp.text)
     print(resp.text)
     for result in dom.xpath(xpath_results):
-- 
cgit v1.2.3


From dcc9fdb47fd16aa65216846dc52470ada016753c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Fri, 27 Apr 2018 15:36:15 +0300
Subject: Added unit test

---
 searx/engines/acgsou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index ebe253649..c1d8cccfa 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -63,7 +63,7 @@ def response(resp):
             filesize = get_torrent_size(filesize, filesize_multiplier)
         except:
             pass
-        # I didn't add download/seed/leech count since as I figured out they are generated randowmly everytime
+        # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime
         content = 'Category: "{category}".'
         content = content.format(category=category)
 
-- 
cgit v1.2.3


From f17e54a396a77f8668d227a41d37737aff340a84 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Fri, 27 Apr 2018 15:40:42 +0300
Subject: Forgot to remove print command used for debugging

---
 searx/engines/acgsou.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index c1d8cccfa..9cedeeecf 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -38,7 +38,6 @@ def request(query, params):
 def response(resp):
     results = []
     dom = html.fromstring(resp.text)
-    print(resp.text)
     for result in dom.xpath(xpath_results):
         # defaults
         filesize = 0
-- 
cgit v1.2.3


From f6695cea5960ab2a8c87b36dd1894abe04e108dd Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Fri, 27 Apr 2018 17:55:42 +0300
Subject: Fix nyaa to use https

---
 searx/engines/nyaa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py
index 6a8e598c4..c57979a5f 100644
--- a/searx/engines/nyaa.py
+++ b/searx/engines/nyaa.py
@@ -1,7 +1,7 @@
 """
  Nyaa.si (Anime Bittorrent tracker)
 
- @website      http://www.nyaa.si/
+ @website      https://nyaa.si/
  @provide-api  no
  @using-api    no
  @results      HTML
@@ -19,7 +19,7 @@ categories = ['files', 'images', 'videos', 'music']
 paging = True
 
 # search-url
-base_url = 'http://www.nyaa.si/'
+base_url = 'https://nyaa.si/'
 search_url = base_url + '?page=search&{query}&offset={offset}'
 
 # xpath queries
-- 
cgit v1.2.3


From 3b89ce9bfa07250da9be005263d6588c13a5ccb0 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Thu, 3 May 2018 08:34:21 +0300
Subject: Switch acgsou to http

---
 searx/engines/acgsou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index 9cedeeecf..c82379c27 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -19,7 +19,7 @@ categories = ['files', 'images', 'videos', 'music']
 paging = True
 
 # search-url
-base_url = 'https://www.acgsou.com/'
+base_url = 'http://www.acgsou.com/'
 search_url = base_url + 'search.php?{query}&page={offset}'
 # xpath queries
 xpath_results = '//table[contains(@class, "list_style table_fixed")]//tr[not(th)]'
-- 
cgit v1.2.3


From 059c2ccb9576f8aae36b4e1c0bf6a024d75701ea Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@airmail.cc>
Date: Sat, 5 May 2018 23:41:13 +0300
Subject: Fix acgsou encoding error on python2 and add unicode characters to
 unit testt

---
 searx/engines/acgsou.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py
index c82379c27..cca28f0db 100644
--- a/searx/engines/acgsou.py
+++ b/searx/engines/acgsou.py
@@ -63,7 +63,7 @@ def response(resp):
         except:
             pass
         # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime
-        content = 'Category: "{category}".'
+        content = u'Category: "{category}".'
         content = content.format(category=category)
 
         results.append({'url': href,
-- 
cgit v1.2.3


From b843c1351aa7ba22d5f37b8fd5412b07ee622239 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Wed, 16 May 2018 22:26:32 +0200
Subject: add missing parameter to gigablast search url

Closes #1293
---
 searx/engines/gigablast.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'searx/engines')

diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py
index ed9380ec0..9b9b9bd77 100644
--- a/searx/engines/gigablast.py
+++ b/searx/engines/gigablast.py
@@ -34,6 +34,7 @@ search_string = 'search?{query}'\
     '&qlang={lang}'\
     '&ff={safesearch}'\
     '&rxiec={rxieu}'\
+    '&ulse={ulse}'\
     '&rand={rxikd}'  # current unix timestamp
 
 # specific xpath variables
@@ -64,6 +65,7 @@ def request(query, params):
                                        number_of_results=number_of_results,
                                        rxikd=int(time() * 1000),
                                        rxieu=random.randint(1000000000, 9999999999),
+                                       ulse=random.randint(100000000, 999999999),
                                        lang=language,
                                        safesearch=safesearch)
 
-- 
cgit v1.2.3


From 75b276f408487db8fecc6eab7abd6126323a7efe Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sun, 20 May 2018 18:10:22 -0500
Subject: fix bing "garbage" results (issue #1275)

---
 searx/engines/bing.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index c6d41782b..2da40619d 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -16,7 +16,7 @@
 from lxml import html
 from searx.engines.xpath import extract_text
 from searx.url_utils import urlencode
-from searx.utils import match_language
+from searx.utils import match_language, gen_useragent
 
 # engine dependent config
 categories = ['general']
@@ -43,6 +43,9 @@ def request(query, params):
         offset=offset)
 
     params['url'] = base_url + search_path
+
+    params['headers']['User-Agent'] = gen_useragent('Windows NT 6.3; WOW64')
+
     return params
 
 
-- 
cgit v1.2.3


From 16915116e0a1cc406a6ff39e0a01ab3497b188f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Mon, 21 May 2018 16:58:39 +0200
Subject: fix piratebay url

Closes #1229
---
 searx/engines/piratebay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py
index a5af8d824..2f3f22a97 100644
--- a/searx/engines/piratebay.py
+++ b/searx/engines/piratebay.py
@@ -18,7 +18,7 @@ categories = ['videos', 'music', 'files']
 paging = True
 
 # search-url
-url = 'https://thepiratebay.se/'
+url = 'https://thepiratebay.org/'
 search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'
 
 # piratebay specific type-definitions
-- 
cgit v1.2.3


From 57e7e9da980bdd582d051ec08fbf24e86deb4da2 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 14 Jun 2018 11:39:54 +0200
Subject: [fix] use html result page in google images (previous endpoint
 stopped working)

---
 searx/engines/google_images.py | 41 ++++++++++++++++-------------------------
 1 file changed, 16 insertions(+), 25 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py
index a380170ca..504831a10 100644
--- a/searx/engines/google_images.py
+++ b/searx/engines/google_images.py
@@ -13,7 +13,7 @@
 from datetime import date, timedelta
 from json import loads
 from lxml import html
-from searx.url_utils import urlencode
+from searx.url_utils import urlencode, urlparse, parse_qs
 
 
 # engine dependent config
@@ -25,10 +25,9 @@ number_of_results = 100
 
 search_url = 'https://www.google.com/search'\
     '?{query}'\
-    '&asearch=ichunk'\
-    '&async=_id:rg_s,_pms:s'\
     '&tbm=isch'\
-    '&yv=2'\
+    '&gbv=1'\
+    '&sa=G'\
     '&{search_options}'
 time_range_attr = "qdr:{range}"
 time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
@@ -66,30 +65,22 @@ def request(query, params):
 def response(resp):
     results = []
 
-    g_result = loads(resp.text)
-
-    dom = html.fromstring(g_result[1][1])
+    dom = html.fromstring(resp.text)
 
     # parse results
-    for result in dom.xpath('//div[@data-ved]'):
-
-        try:
-            metadata = loads(''.join(result.xpath('./div[contains(@class, "rg_meta")]/text()')))
-        except:
-            continue
-
-        thumbnail_src = metadata['tu']
-
-        # http to https
-        thumbnail_src = thumbnail_src.replace("http://", "https://")
-
+    for img in dom.xpath('//a'):
+        r = {
+            'title': u' '.join(img.xpath('.//div[class="rg_ilmbg"]//text()')),
+            'content': '',
+            'template': 'images.html',
+        }
+        url = urlparse(img.xpath('.//@href')[0])
+        query = parse_qs(url.query)
+        r['url'] = query['imgrefurl'][0]
+        r['img_src'] = query['imgurl'][0]
+        r['thumbnail_src'] = r['img_src']
         # append result
-        results.append({'url': metadata['ru'],
-                        'title': metadata['pt'],
-                        'content': metadata['s'],
-                        'thumbnail_src': thumbnail_src,
-                        'img_src': metadata['ou'],
-                        'template': 'images.html'})
+        results.append(r)
 
     # return results
     return results
-- 
cgit v1.2.3


From f82ead3e303d75ba63a370dc038311e172e1330d Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Thu, 14 Jun 2018 11:50:03 +0200
Subject: [fix] add py3 compatibility to base engine - closes #1319

---
 searx/engines/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/base.py b/searx/engines/base.py
index be0b7d247..f1b1cf671 100755
--- a/searx/engines/base.py
+++ b/searx/engines/base.py
@@ -55,7 +55,7 @@ shorcut_dict = {
 def request(query, params):
     # replace shortcuts with API advanced search keywords
     for key in shorcut_dict.keys():
-        query = re.sub(str(key), str(shorcut_dict[key]), query)
+        query = re.sub(key, shorcut_dict[key], str(query))
 
     # basic search
     offset = (params['pageno'] - 1) * number_of_results
-- 
cgit v1.2.3


From acaef6600e34159d2edb7bf0ef6b5f34471136e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Bourrel?= <leo.bourrel@epitech.eu>
Date: Thu, 5 Jul 2018 10:11:45 +0200
Subject: Update path to wikidata image

---
 searx/engines/wikidata.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index fe53609c1..ffc1c8d0f 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -54,7 +54,7 @@ value_xpath = './/div[contains(@class,"wikibase-statementview-mainsnak")]'\
     + '/*/div[contains(@class,"wikibase-snakview-value")]'
 language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator")]'
 calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
-
+media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
 
 def request(query, params):
     params['url'] = url_search.format(
@@ -313,7 +313,7 @@ def add_image(result):
     for property_id in property_ids:
         image = result.xpath(property_xpath.replace('{propertyid}', property_id))
         if image:
-            image_name = image[0].xpath(value_xpath)
+            image_name = image[0].xpath(media_xpath)
             image_src = url_image.replace('{filename}', extract_text(image_name[0]))
             return image_src
 
-- 
cgit v1.2.3


From 7a474db61bd9ba9a08111758b058f81cb5175db4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Bourrel?= <leo.bourrel@epitech.eu>
Date: Fri, 6 Jul 2018 10:31:01 +0200
Subject: Fix formatting

---
 searx/engines/wikidata.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'searx/engines')

diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index ffc1c8d0f..c315b30da 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -56,6 +56,7 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator
 calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
 media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
 
+
 def request(query, params):
     params['url'] = url_search.format(
         query=urlencode({'search': query}))
-- 
cgit v1.2.3


From 845ba79e7532dedad2bb96e5d68504b6729bee14 Mon Sep 17 00:00:00 2001
From: Adam Tauber <asciimoo@gmail.com>
Date: Mon, 13 Aug 2018 12:03:27 +0200
Subject: [fix] update qwant search url

thx to @dalf for the fix (#1365)
---
 searx/engines/qwant.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'searx/engines')

diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index 239193b96..4b0f1c87c 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -28,7 +28,7 @@ category_to_keyword = {'general': 'web',
                        'social media': 'social'}
 
 # search-url
-url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}'
+url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}&t={keyword}&uiv=4'
 
 
 # do search-request
-- 
cgit v1.2.3