From c89c05bcebb44699c94d6da2fcb8ad376b60fe67 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 25 Feb 2020 18:44:28 +0100 Subject: bugfix: google-news and bing-news has changed the language parameter closes: https://github.com/asciimoo/searx/issues/1838 Signed-off-by: Markus Heiser --- searx/engines/bing_news.py | 3 ++- searx/engines/google_news.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 669130c42..827555bd7 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -58,8 +58,9 @@ def _get_url(query, language, offset, time_range): offset=offset, interval=time_range_dict[time_range]) else: + # e.g. setmkt=de-de&setlang=de search_path = search_string.format( - query=urlencode({'q': query, 'setmkt': language}), + query=urlencode({'q': query, 'setlang': language}), offset=offset) return base_url + search_path diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index 9c837b45b..c9cc75435 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -54,7 +54,7 @@ def request(query, params): if params['language'] != 'all': language = match_language(params['language'], supported_languages, language_aliases).split('-')[0] if language: - params['url'] += '&lr=lang_' + language + params['url'] += '&hl=' + language return params -- cgit v1.2.3 From e0c99d9dcbe4c2eee0a7c6f4a7326a8376467640 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 1 Mar 2020 08:01:36 +0100 Subject: bugfix: fetch_supported_languages bing, -news, -videos, -images Signed-off-by: Markus Heiser --- searx/engines/bing.py | 23 ++++++++++++++--------- searx/engines/bing_images.py | 21 ++------------------- searx/engines/bing_news.py | 3 ++- searx/engines/bing_videos.py | 2 +- 4 files changed, 19 insertions(+), 30 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/bing.py b/searx/engines/bing.py index b193f7c60..c8fc4fa2e 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -110,13 +110,18 @@ def response(resp): # get supported languages from their site def _fetch_supported_languages(resp): - supported_languages = [] + lang_tags = set() + + setmkt = re.compile('setmkt=([^&]*)') dom = html.fromstring(resp.text) - options = eval_xpath(dom, '//div[@id="limit-languages"]//input') - for option in options: - code = eval_xpath(option, './@id')[0].replace('_', '-') - if code == 'nb': - code = 'no' - supported_languages.append(code) - - return supported_languages + lang_links = eval_xpath(dom, "//li/a[contains(@href, 'setmkt')]") + + for a in lang_links: + href = eval_xpath(a, './@href')[0] + match = setmkt.search(href) + l_tag = match.groups()[0] + _lang, _nation = l_tag.split('-',1) + l_tag = _lang.lower() + '-' + _nation.upper() + lang_tags.add(l_tag) + + return list(lang_tags) diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index 44e2c3bbc..138ed11c6 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -18,6 +18,8 @@ import re from searx.url_utils import urlencode from searx.utils import match_language +from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases + # engine dependent config categories = ['images'] paging = True @@ -103,22 +105,3 @@ def response(resp): continue return results - - -# get supported languages from their site -def _fetch_supported_languages(resp): - supported_languages = [] - dom = html.fromstring(resp.text) - - regions_xpath = '//div[@id="region-section-content"]' \ - + '//ul[@class="b_vList"]/li/a/@href' - - regions = dom.xpath(regions_xpath) - for region in regions: - code = re.search('setmkt=[^\&]+', region).group()[7:] - if code == 'nb-NO': - code = 'no-NO' - - supported_languages.append(code) - - return supported_languages diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 827555bd7..14fd7b99a 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -15,9 +15,10 @@ from datetime import datetime from dateutil import parser from lxml import etree from searx.utils import list_get, match_language -from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases from searx.url_utils import urlencode, urlparse, parse_qsl +from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases + # engine dependent config categories = ['news'] paging = True diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py index f1e636819..6e7b6d3aa 100644 --- a/searx/engines/bing_videos.py +++ b/searx/engines/bing_videos.py @@ -12,10 +12,10 @@ from json import loads from lxml import html -from searx.engines.bing_images import _fetch_supported_languages, supported_languages_url from searx.url_utils import urlencode from searx.utils import match_language +from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases categories = ['videos'] paging = True -- cgit v1.2.3 From 1c853f9573a85d9885d3c99b7e3210c834e5e5f3 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 1 Mar 2020 11:07:59 +0100 Subject: bing_news: parital rollback of c89c05bc The bing_news bug (discussed in #1838) was caused by wrong language tags, which was fixed e0c99d9d / no need to change the bing_news search string. closes: https://github.com/asciimoo/searx/issues/1838 Signed-off-by: Markus Heiser --- searx/engines/bing.py | 2 +- searx/engines/bing_news.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'searx/engines') diff --git a/searx/engines/bing.py b/searx/engines/bing.py index c8fc4fa2e..afb776acd 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -120,7 +120,7 @@ def _fetch_supported_languages(resp): href = eval_xpath(a, './@href')[0] match = setmkt.search(href) l_tag = match.groups()[0] - _lang, _nation = l_tag.split('-',1) + _lang, _nation = l_tag.split('-', 1) l_tag = _lang.lower() + '-' + _nation.upper() lang_tags.add(l_tag) diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 14fd7b99a..d13be777c 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -61,7 +61,7 @@ def _get_url(query, language, offset, time_range): else: # e.g. setmkt=de-de&setlang=de search_path = search_string.format( - query=urlencode({'q': query, 'setlang': language}), + query=urlencode({'q': query, 'setmkt': language}), offset=offset) return base_url + search_path -- cgit v1.2.3 From ad7a6e6e1022923850343b2c19e47bbd9fbad050 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 1 Mar 2020 17:28:59 +0100 Subject: bugfix(!biv) : bing-video do not like "older" User-Agents When selecting other languages than 'en', bing-video did not handle the language correct and gave very bad results. Since User-Agent is normaly rotated in searx, the behavior of a !biv search was unpredictable and paging was broken. Signed-off-by: Markus Heiser --- searx/engines/bing_videos.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'searx/engines') diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py index 6e7b6d3aa..f048f0d8e 100644 --- a/searx/engines/bing_videos.py +++ b/searx/engines/bing_videos.py @@ -67,6 +67,10 @@ def request(query, params): if params['time_range'] in time_range_dict: params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']]) + # bing videos did not like "older" versions < 70.0.1 when selectin other + # languages then 'en' .. very strange ?!?! + params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:73.0.1) Gecko/20100101 Firefox/73.0.1' + return params -- cgit v1.2.3