diff options
Diffstat (limited to 'searx/engines')
28 files changed, 459 insertions, 123 deletions
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index ae132f48d..07d5b226c 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -44,6 +44,7 @@ ENGINE_DEFAULT_ARGS = { "enable_http": False, "using_tor_proxy": False, "display_error_messages": True, + "send_accept_language_header": False, "tokens": [], "about": {}, } @@ -149,7 +150,11 @@ def set_loggers(engine, engine_name): engine.logger = logger.getChild(engine_name) # the engine may have load some other engines # may sure the logger is initialized - for module_name, module in sys.modules.items(): + # use sys.modules.copy() to avoid "RuntimeError: dictionary changed size during iteration" + # see https://github.com/python/cpython/issues/89516 + # and https://docs.python.org/3.10/library/sys.html#sys.modules + modules = sys.modules.copy() + for module_name, module in modules.items(): if ( module_name.startswith("searx.engines") and module_name != "searx.engines.__init__" diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 4c037de85..8d024fed0 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -8,7 +8,8 @@ import re from urllib.parse import urlencode, urlparse, parse_qs from lxml import html -from searx.utils import eval_xpath, extract_text, match_language +from searx.utils import eval_xpath, extract_text, eval_xpath_list, match_language +from searx.network import multi_requests, Request about = { "website": 'https://www.bing.com', @@ -24,6 +25,7 @@ categories = ['general', 'web'] paging = True time_range_support = False safesearch = False +send_accept_language_header = True supported_languages_url = 'https://www.bing.com/account/general' language_aliases = {} @@ -67,7 +69,6 @@ def request(query, params): logger.debug("headers.Referer --> %s", referer) params['url'] = base_url + search_path - params['headers']['Accept-Language'] = "en-US,en;q=0.5" params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' return params @@ -79,30 +80,48 @@ def response(resp): dom = html.fromstring(resp.text) - for result in eval_xpath(dom, '//div[@class="sa_cc"]'): - - # IMO //div[@class="sa_cc"] does no longer match - logger.debug('found //div[@class="sa_cc"] --> %s', result) - - link = eval_xpath(result, './/h3/a')[0] - url = link.attrib.get('href') - title = extract_text(link) - content = extract_text(eval_xpath(result, './/p')) - - # append result - results.append({'url': url, 'title': title, 'content': content}) - # parse results again if nothing is found yet - for result in eval_xpath(dom, '//li[@class="b_algo"]'): + + url_to_resolve = [] + url_to_resolve_index = [] + for i, result in enumerate(eval_xpath_list(dom, '//li[@class="b_algo"]')): link = eval_xpath(result, './/h2/a')[0] url = link.attrib.get('href') title = extract_text(link) content = extract_text(eval_xpath(result, './/p')) + # get the real URL either using the URL shown to user or following the Bing URL + if url.startswith('https://www.bing.com/ck/a?'): + url_cite = extract_text(eval_xpath(result, './/div[@class="b_attribution"]/cite')) + # Bing can shorten the URL either at the end or in the middle of the string + if ( + url_cite.startswith('https://') + and '…' not in url_cite + and '...' not in url_cite + and '›' not in url_cite + ): + # no need for an additional HTTP request + url = url_cite + else: + # resolve the URL with an additional HTTP request + url_to_resolve.append(url.replace('&ntb=1', '&ntb=F')) + url_to_resolve_index.append(i) + url = None # remove the result if the HTTP Bing redirect raise an exception + # append result results.append({'url': url, 'title': title, 'content': content}) + # resolve all Bing redirections in parallel + request_list = [ + Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve + ] + response_list = multi_requests(request_list) + for i, redirect_response in enumerate(response_list): + if not isinstance(redirect_response, Exception): + results[url_to_resolve_index[i]]['url'] = redirect_response.headers['location'] + + # get number_of_results try: result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()')) if "-" in result_len_container: diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index cb69dc172..107ce3cff 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -31,6 +31,7 @@ categories = ['images', 'web'] paging = True safesearch = True time_range_support = True +send_accept_language_header = True supported_languages_url = 'https://www.bing.com/account/general' number_of_results = 28 diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 22856541b..7eea17bb4 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -34,6 +34,7 @@ about = { categories = ['news'] paging = True time_range_support = True +send_accept_language_header = True # search-url base_url = 'https://www.bing.com/' diff --git a/searx/engines/bing_videos.py b/searx/engines/bing_videos.py index ae8e8d49a..9be8eeaef 100644 --- a/searx/engines/bing_videos.py +++ b/searx/engines/bing_videos.py @@ -30,6 +30,7 @@ categories = ['videos', 'web'] paging = True safesearch = True time_range_support = True +send_accept_language_header = True number_of_results = 28 base_url = 'https://www.bing.com/' @@ -70,10 +71,6 @@ def request(query, params): if params['time_range'] in time_range_dict: params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']]) - # bing videos did not like "older" versions < 70.0.1 when selectin other - # languages then 'en' .. very strange ?!?! - params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:73.0.1) Gecko/20100101 Firefox/73.0.1' - return params diff --git a/searx/engines/demo_online.py b/searx/engines/demo_online.py index e53b3c15e..08add5371 100644 --- a/searx/engines/demo_online.py +++ b/searx/engines/demo_online.py @@ -19,7 +19,8 @@ list in ``settings.yml``: from json import loads from urllib.parse import urlencode -engine_type = 'offline' +engine_type = 'online' +send_accept_language_header = True categories = ['general'] disabled = True timeout = 2.0 diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 71da72677..17f0fae1c 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -31,6 +31,7 @@ categories = ['general', 'web'] paging = True supported_languages_url = 'https://duckduckgo.com/util/u588.js' time_range_support = True +send_accept_language_header = True language_aliases = { 'ar-SA': 'ar-XA', diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py index ad3c92169..a73ee55ff 100644 --- a/searx/engines/duckduckgo_definitions.py +++ b/searx/engines/duckduckgo_definitions.py @@ -27,6 +27,8 @@ about = { "results": 'JSON', } +send_accept_language_header = True + URL = 'https://api.duckduckgo.com/' + '?{query}&format=json&pretty=0&no_redirect=1&d=1' WIKIDATA_PREFIX = ['http://www.wikidata.org/entity/', 'https://www.wikidata.org/entity/'] @@ -62,7 +64,6 @@ def request(query, params): params['url'] = URL.format(query=urlencode({'q': query})) language = match_language(params['language'], supported_languages, language_aliases) language = language.split('-')[0] - params['headers']['Accept-Language'] = language return params diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py index 7d844b543..19f649ef4 100644 --- a/searx/engines/duckduckgo_images.py +++ b/searx/engines/duckduckgo_images.py @@ -30,6 +30,7 @@ about = { categories = ['images', 'web'] paging = True safesearch = True +send_accept_language_header = True # search-url images_url = 'https://duckduckgo.com/i.js?{query}&s={offset}&p={safesearch}&o=json&vqd={vqd}' diff --git a/searx/engines/emojipedia.py b/searx/engines/emojipedia.py new file mode 100644 index 000000000..020bf689b --- /dev/null +++ b/searx/engines/emojipedia.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Emojipedia + +Emojipedia is an emoji reference website which documents the meaning and +common usage of emoji characters in the Unicode Standard. It is owned by Zedge +since 2021. Emojipedia is a voting member of The Unicode Consortium.[1] + +[1] https://en.wikipedia.org/wiki/Emojipedia +""" + +from urllib.parse import urlencode +from lxml import html + +from searx.utils import ( + eval_xpath_list, + eval_xpath_getindex, + extract_text, +) + +about = { + "website": 'https://emojipedia.org', + "wikidata_id": 'Q22908129', + "official_api_documentation": None, + "use_official_api": False, + "require_api_key": False, + "results": 'HTML', +} + +categories = [] +paging = False +time_range_support = False + +base_url = 'https://emojipedia.org' +search_url = base_url + '/search/?{query}' + + +def request(query, params): + params['url'] = search_url.format( + query=urlencode({'q': query}), + ) + return params + + +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + for result in eval_xpath_list(dom, "//ol[@class='search-results']/li"): + + extracted_desc = extract_text(eval_xpath_getindex(result, './/p', 0)) + + if 'No results found.' in extracted_desc: + break + + link = eval_xpath_getindex(result, './/h2/a', 0) + + url = base_url + link.attrib.get('href') + title = extract_text(link) + content = extracted_desc + + res = {'url': url, 'title': title, 'content': content} + + results.append(res) + + return results diff --git a/searx/engines/google.py b/searx/engines/google.py index acc87307b..5e80f6dcc 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -45,6 +45,7 @@ categories = ['general', 'web'] paging = True time_range_support = True safesearch = True +send_accept_language_header = True use_mobile_ui = False supported_languages_url = 'https://www.google.com/preferences?#languages' @@ -112,7 +113,7 @@ filter_mapping = {0: 'off', 1: 'medium', 2: 'high'} # ------------------------ # google results are grouped into <div class="jtfYYd ..." ../> -results_xpath = '//div[@class="jtfYYd"]' +results_xpath = '//div[contains(@class, "jtfYYd")]' # google *sections* are no usual *results*, we ignore them g_section_with_header = './g-section-with-header' @@ -241,16 +242,6 @@ def get_lang_info(params, lang_list, custom_aliases, supported_any_language): # language. ret_val['params']['lr'] = "lang_" + lang_list.get(lang_country, language) - # Accept-Language: fr-CH, fr;q=0.8, en;q=0.6, *;q=0.5 - ret_val['headers']['Accept-Language'] = ','.join( - [ - lang_country, - language + ';q=0.8,', - 'en;q=0.6', - '*;q=0.5', - ] - ) - return ret_val @@ -298,6 +289,7 @@ def request(query, params): query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]}) params['url'] = query_url + params['cookies']['CONSENT'] = "YES+" params['headers'].update(lang_info['headers']) if use_mobile_ui: params['headers']['Accept'] = '*/*' diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index fc192d62d..e1f676dd6 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -51,6 +51,7 @@ paging = False use_locale_domain = True time_range_support = True safesearch = True +send_accept_language_header = True filter_mapping = {0: 'images', 1: 'active', 2: 'active'} @@ -125,23 +126,13 @@ def request(query, params): """Google-Video search request""" lang_info = get_lang_info(params, supported_languages, language_aliases, False) - logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) query_url = ( 'https://' + lang_info['subdomain'] + '/search' + "?" - + urlencode( - { - 'q': query, - 'tbm': "isch", - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - 'num': 30, - } - ) + + urlencode({'q': query, 'tbm': "isch", **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'num': 30}) ) if params['time_range'] in time_range_dict: @@ -150,6 +141,7 @@ def request(query, params): query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]}) params['url'] = query_url + params['cookies']['CONSENT'] = "YES+" params['headers'].update(lang_info['headers']) params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' return params diff --git a/searx/engines/google_news.py b/searx/engines/google_news.py index 162e4348e..8f5a4b104 100644 --- a/searx/engines/google_news.py +++ b/searx/engines/google_news.py @@ -14,7 +14,6 @@ ignores some parameters from the common :ref:`google API`: # pylint: disable=invalid-name import binascii -from datetime import datetime import re from urllib.parse import urlencode from base64 import b64decode @@ -71,13 +70,13 @@ time_range_support = True # # safesearch : results are identitical for safesearch=0 and safesearch=2 safesearch = False +send_accept_language_header = True def request(query, params): """Google-News search request""" lang_info = get_lang_info(params, supported_languages, language_aliases, False) - logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) # google news has only one domain lang_info['subdomain'] = 'news.google.com' @@ -98,22 +97,14 @@ def request(query, params): + lang_info['subdomain'] + '/search' + "?" - + urlencode( - { - 'q': query, - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - 'gl': lang_info['country'], - } - ) + + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'gl': lang_info['country']}) + ('&ceid=%s' % ceid) ) # ceid includes a ':' character which must not be urlencoded params['url'] = query_url + params['cookies']['CONSENT'] = "YES+" params['headers'].update(lang_info['headers']) params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' - params['headers']['Cookie'] = "CONSENT=YES+cb.%s-14-p0.en+F+941;" % datetime.now().strftime("%Y%m%d") return params diff --git a/searx/engines/google_play_apps.py b/searx/engines/google_play_apps.py new file mode 100644 index 000000000..6506a446a --- /dev/null +++ b/searx/engines/google_play_apps.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +""" + Google Play Apps +""" + +from urllib.parse import urlencode +from lxml import html +from searx.utils import ( + eval_xpath, + extract_url, + extract_text, + eval_xpath_list, + eval_xpath_getindex, +) + +about = { + "website": "https://play.google.com/", + "wikidata_id": "Q79576", + "use_official_api": False, + "require_api_key": False, + "results": "HTML", +} + +categories = ["files", "apps"] +send_accept_language_header = True + +search_url = "https://play.google.com/store/search?{query}&c=apps" + + +def request(query, params): + params["url"] = search_url.format(query=urlencode({"q": query})) + params['cookies']['CONSENT'] = "YES+" + + return params + + +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + if eval_xpath(dom, '//div[@class="v6DsQb"]'): + return [] + + spot = eval_xpath_getindex(dom, '//div[@class="ipRz4"]', 0, None) + if spot is not None: + url = extract_url(eval_xpath(spot, './a[@class="Qfxief"]/@href'), search_url) + title = extract_text(eval_xpath(spot, './/div[@class="vWM94c"]')) + content = extract_text(eval_xpath(spot, './/div[@class="LbQbAe"]')) + img = extract_text(eval_xpath(spot, './/img[@class="T75of bzqKMd"]/@src')) + + results.append({"url": url, "title": title, "content": content, "img_src": img}) + + more = eval_xpath_list(dom, '//c-wiz[@jsrenderer="RBsfwb"]//div[@role="listitem"]', min_len=1) + for result in more: + url = extract_url(eval_xpath(result, ".//a/@href"), search_url) + title = extract_text(eval_xpath(result, './/span[@class="DdYX5"]')) + content = extract_text(eval_xpath(result, './/span[@class="wMUdtb"]')) + img = extract_text( + eval_xpath( + result, + './/img[@class="T75of stzEZd" or @class="T75of etjhNc Q8CSx "]/@src', + ) + ) + + results.append({"url": url, "title": title, "content": content, "img_src": img}) + + for suggestion in eval_xpath_list(dom, '//c-wiz[@jsrenderer="qyd4Kb"]//div[@class="ULeU3b neq64b"]'): + results.append({"suggestion": extract_text(eval_xpath(suggestion, './/div[@class="Epkrse "]'))}) + + return results diff --git a/searx/engines/google_scholar.py b/searx/engines/google_scholar.py index e0700957c..41c62886b 100644 --- a/searx/engines/google_scholar.py +++ b/searx/engines/google_scholar.py @@ -52,6 +52,7 @@ language_support = True use_locale_domain = True time_range_support = True safesearch = False +send_accept_language_header = True def time_range_url(params): @@ -75,7 +76,6 @@ def request(query, params): offset = (params['pageno'] - 1) * 10 lang_info = get_lang_info(params, supported_languages, language_aliases, False) - logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) # subdomain is: scholar.google.xy lang_info['subdomain'] = lang_info['subdomain'].replace("www.", "scholar.") @@ -85,20 +85,13 @@ def request(query, params): + lang_info['subdomain'] + '/scholar' + "?" - + urlencode( - { - 'q': query, - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - 'start': offset, - } - ) + + urlencode({'q': query, **lang_info['params'], 'ie': "utf8", 'oe': "utf8", 'start': offset}) ) query_url += time_range_url(params) params['url'] = query_url + params['cookies']['CONSENT'] = "YES+" params['headers'].update(lang_info['headers']) params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' diff --git a/searx/engines/google_videos.py b/searx/engines/google_videos.py index 06aac8ae1..26dbcdd3c 100644 --- a/searx/engines/google_videos.py +++ b/searx/engines/google_videos.py @@ -60,6 +60,7 @@ language_support = True use_locale_domain = True time_range_support = True safesearch = True +send_accept_language_header = True RE_CACHE = {} @@ -111,22 +112,13 @@ def request(query, params): """Google-Video search request""" lang_info = get_lang_info(params, supported_languages, language_aliases, False) - logger.debug("HTTP header Accept-Language --> %s", lang_info['headers']['Accept-Language']) query_url = ( 'https://' + lang_info['subdomain'] + '/search' + "?" - + urlencode( - { - 'q': query, - 'tbm': "vid", - **lang_info['params'], - 'ie': "utf8", - 'oe': "utf8", - } - ) + + urlencode({'q': query, 'tbm': "vid", **lang_info['params'], 'ie': "utf8", 'oe': "utf8"}) ) if params['time_range'] in time_range_dict: @@ -135,6 +127,7 @@ def request(query, params): query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]}) params['url'] = query_url + params['cookies']['CONSENT'] = "YES+" params['headers'].update(lang_info['headers']) params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' return params diff --git a/searx/engines/json_engine.py b/searx/engines/json_engine.py index f53bc0bf4..2dd3bc55e 100644 --- a/searx/engines/json_engine.py +++ b/searx/engines/json_engine.py @@ -16,6 +16,11 @@ paging = False suggestion_query = '' results_query = '' +cookies = {} +headers = {} +'''Some engines might offer different result based on cookies or headers. +Possible use-case: To set safesearch cookie or header to moderate.''' + # parameters for engines with paging support # # number of results on each page @@ -88,6 +93,9 @@ def request(query, params): if paging and search_url.find('{pageno}') >= 0: fp['pageno'] = (params['pageno'] - 1) * page_size + first_page_num + params['cookies'].update(cookies) + params['headers'].update(headers) + params['url'] = search_url.format(**fp) params['query'] = query diff --git a/searx/engines/lingva.py b/searx/engines/lingva.py new file mode 100644 index 000000000..bf51b705e --- /dev/null +++ b/searx/engines/lingva.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Lingva (alternative Google Translate frontend)""" + +from json import loads + +about = { + "website": 'https://lingva.ml', + "wikidata_id": None, + "official_api_documentation": 'https://github.com/thedaviddelta/lingva-translate#public-apis', + "use_official_api": True, + "require_api_key": False, + "results": 'JSON', +} + +engine_type = 'online_dictionary' +categories = ['general'] + +url = "https://lingva.ml" +search_url = "{url}/api/v1/{from_lang}/{to_lang}/{query}" + + +def request(_query, params): + params['url'] = search_url.format( + url=url, from_lang=params['from_lang'][1], to_lang=params['to_lang'][1], query=params['query'] + ) + return params + + +def response(resp): + results = [] + + result = loads(resp.text) + info = result["info"] + from_to_prefix = "%s-%s " % (resp.search_params['from_lang'][1], resp.search_params['to_lang'][1]) + + if "typo" in info: + results.append({"suggestion": from_to_prefix + info["typo"]}) + + if 'definitions' in info: # pylint: disable=too-many-nested-blocks + for definition in info['definitions']: + if 'list' in definition: + for item in definition['list']: + if 'synonyms' in item: + for synonym in item['synonyms']: + results.append({"suggestion": from_to_prefix + synonym}) + + infobox = "" + + for translation in info["extraTranslations"]: + infobox += f"<b>{translation['type']}</b>" + + for word in translation["list"]: + infobox += f"<dl><dt>{word['word']}</dt>" + + for meaning in word["meanings"]: + infobox += f"<dd>{meaning}</dd>" + + infobox += "</dl>" + + results.append( + { + 'infobox': result["translation"], + 'content': infobox, + } + ) + + return results diff --git a/searx/engines/mongodb.py b/searx/engines/mongodb.py index c833ca9e0..63452bb68 100644 --- a/searx/engines/mongodb.py +++ b/searx/engines/mongodb.py @@ -5,7 +5,7 @@ """ import re -from pymongo import MongoClient # pylint: disable=import-error +from pymongo import MongoClient # pyright: ignore # pylint: disable=import-error engine_type = 'offline' diff --git a/searx/engines/mysql_server.py b/searx/engines/mysql_server.py index c16093fb0..8d0a49565 100644 --- a/searx/engines/mysql_server.py +++ b/searx/engines/mysql_server.py @@ -6,7 +6,7 @@ # import error is ignored because the admin has to install mysql manually to use # the engine -import mysql.connector # pylint: disable=import-error +import mysql.connector # pyright: ignore # pylint: disable=import-error engine_type = 'offline' auth_plugin = 'caching_sha2_password' diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py index c619ce98e..4f799fce7 100644 --- a/searx/engines/openstreetmap.py +++ b/searx/engines/openstreetmap.py @@ -29,6 +29,8 @@ about = { # engine dependent config categories = ['map'] paging = False +language_support = True +send_accept_language_header = True # search-url base_url = 'https://nominatim.openstreetmap.org/' @@ -141,6 +143,8 @@ def request(query, params): params['url'] = base_url + search_string.format(query=urlencode({'q': query})) params['route'] = route_re.match(query) params['headers']['User-Agent'] = searx_useragent() + if 'Accept-Language' not in params['headers']: + params['headers']['Accept-Language'] = 'en' return params @@ -202,7 +206,7 @@ def get_wikipedia_image(raw_value): return get_external_url('wikimedia_image', raw_value) -def fetch_wikidata(nominatim_json, user_langage): +def fetch_wikidata(nominatim_json, user_language): """Update nominatim_json using the result of an unique to wikidata For result in nominatim_json: @@ -223,10 +227,10 @@ def fetch_wikidata(nominatim_json, user_langage): wd_to_results.setdefault(wd_id, []).append(result) if wikidata_ids: - user_langage = 'en' if user_langage == 'all' else user_langage + user_language = 'en' if user_language == 'all' else user_language.split('-')[0] wikidata_ids_str = " ".join(wikidata_ids) query = wikidata_image_sparql.replace('%WIKIDATA_IDS%', sparql_string_escape(wikidata_ids_str)).replace( - '%LANGUAGE%', sparql_string_escape(user_langage) + '%LANGUAGE%', sparql_string_escape(user_language) ) wikidata_json = send_wikidata_query(query) for wd_result in wikidata_json.get('results', {}).get('bindings', {}): @@ -241,7 +245,7 @@ def fetch_wikidata(nominatim_json, user_langage): # overwrite wikipedia link wikipedia_name = wd_result.get('wikipediaName', {}).get('value') if wikipedia_name: - result['extratags']['wikipedia'] = user_langage + ':' + wikipedia_name + result['extratags']['wikipedia'] = user_language + ':' + wikipedia_name # get website if not already defined website = wd_result.get('website', {}).get('value') if ( diff --git a/searx/engines/postgresql.py b/searx/engines/postgresql.py index d8bbabe27..d7ff6a11b 100644 --- a/searx/engines/postgresql.py +++ b/searx/engines/postgresql.py @@ -6,7 +6,7 @@ # import error is ignored because the admin has to install mysql manually to use # the engine -import psycopg2 # pylint: disable=import-error +import psycopg2 # pyright: ignore # pylint: disable=import-error engine_type = 'offline' host = "127.0.0.1" diff --git a/searx/engines/sjp.py b/searx/engines/sjp.py index 8342a2819..6daa46e78 100644 --- a/searx/engines/sjp.py +++ b/searx/engines/sjp.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -"""Słownik Języka Polskiego (general) +# lint: pylint +"""Słownik Języka Polskiego +Dictionary of the polish language from PWN (sjp.pwn) """ from lxml.html import fromstring diff --git a/searx/engines/tineye.py b/searx/engines/tineye.py index fe5b60393..6c5ff134c 100644 --- a/searx/engines/tineye.py +++ b/searx/engines/tineye.py @@ -17,6 +17,7 @@ billion images `[tineye.com] <https://tineye.com/how>`_. from urllib.parse import urlencode from datetime import datetime +from flask_babel import gettext about = { "website": 'https://tineye.com', @@ -28,20 +29,41 @@ about = { } engine_type = 'online_url_search' +""":py:obj:`searx.search.processors.online_url_search`""" + categories = ['general'] paging = True safesearch = False base_url = 'https://tineye.com' search_string = '/result_json/?page={page}&{query}' +FORMAT_NOT_SUPPORTED = gettext( + "Could not read that image url. This may be due to an unsupported file" + " format. TinEye only supports images that are JPEG, PNG, GIF, BMP, TIFF or WebP." +) +"""TinEye error message""" + +NO_SIGNATURE_ERROR = gettext( + "The image is too simple to find matches. TinEye requires a basic level of" + " visual detail to successfully identify matches." +) +"""TinEye error message""" + +DOWNLOAD_ERROR = gettext("The image could not be downloaded.") +"""TinEye error message""" + def request(query, params): + """Build TinEye HTTP request using ``search_urls`` of a :py:obj:`engine_type`.""" + + params['raise_for_httperror'] = False if params['search_urls']['data:image']: query = params['search_urls']['data:image'] elif params['search_urls']['http']: query = params['search_urls']['http'] + logger.debug("query URL: %s", query) query = urlencode({'url': query}) # see https://github.com/TinEye/pytineye/blob/main/pytineye/api.py @@ -59,45 +81,145 @@ def request(query, params): return params +def parse_tineye_match(match_json): + """Takes parsed JSON from the API server and turns it into a :py:obj:`dict` + object. + + Attributes `(class Match) <https://github.com/TinEye/pytineye/blob/main/pytineye/api.py>`__ + + - `image_url`, link to the result image. + - `domain`, domain this result was found on. + - `score`, a number (0 to 100) that indicates how closely the images match. + - `width`, image width in pixels. + - `height`, image height in pixels. + - `size`, image area in pixels. + - `format`, image format. + - `filesize`, image size in bytes. + - `overlay`, overlay URL. + - `tags`, whether this match belongs to a collection or stock domain. + + - `backlinks`, a list of Backlink objects pointing to the original websites + and image URLs. List items are instances of :py:obj:`dict`, (`Backlink + <https://github.com/TinEye/pytineye/blob/main/pytineye/api.py>`__): + + - `url`, the image URL to the image. + - `backlink`, the original website URL. + - `crawl_date`, the date the image was crawled. + + """ + + # HINT: there exists an alternative backlink dict in the domains list / e.g.:: + # + # match_json['domains'][0]['backlinks'] + + backlinks = [] + if "backlinks" in match_json: + + for backlink_json in match_json["backlinks"]: + if not isinstance(backlink_json, dict): + continue + + crawl_date = backlink_json.get("crawl_date") + if crawl_date: + crawl_date = datetime.fromisoformat(crawl_date[:-3]) + else: + crawl_date = datetime.min + + backlinks.append( + { + 'url': backlink_json.get("url"), + 'backlink': backlink_json.get("backlink"), + 'crawl_date': crawl_date, + 'image_name': backlink_json.get("image_name"), + } + ) + + return { + 'image_url': match_json.get("image_url"), + 'domain': match_json.get("domain"), + 'score': match_json.get("score"), + 'width': match_json.get("width"), + 'height': match_json.get("height"), + 'size': match_json.get("size"), + 'image_format': match_json.get("format"), + 'filesize': match_json.get("filesize"), + 'overlay': match_json.get("overlay"), + 'tags': match_json.get("tags"), + 'backlinks': backlinks, + } + + def response(resp): + """Parse HTTP response from TinEye.""" results = [] - # Define wanted results - json_data = resp.json() - number_of_results = json_data['num_matches'] - - for i in json_data['matches']: - image_format = i['format'] - width = i['width'] - height = i['height'] - thumbnail_src = i['image_url'] - backlink = i['domains'][0]['backlinks'][0] - url = backlink['backlink'] - source = backlink['url'] - title = backlink['image_name'] - img_src = backlink['url'] - - # Get and convert published date - api_date = backlink['crawl_date'][:-3] - publishedDate = datetime.fromisoformat(api_date) - - # Append results + try: + json_data = resp.json() + except Exception as exc: # pylint: disable=broad-except + msg = "can't parse JSON response // %s" % exc + logger.error(msg) + json_data = {'error': msg} + + # handle error codes from Tineye + + if resp.is_error: + if resp.status_code in (400, 422): + + message = 'HTTP status: %s' % resp.status_code + error = json_data.get('error') + s_key = json_data.get('suggestions', {}).get('key', '') + + if error and s_key: + message = "%s (%s)" % (error, s_key) + elif error: + message = error + + if s_key == "Invalid image URL": + # test https://docs.searxng.org/_static/searxng-wordmark.svg + message = FORMAT_NOT_SUPPORTED + elif s_key == 'NO_SIGNATURE_ERROR': + # test https://pngimg.com/uploads/dot/dot_PNG4.png + message = NO_SIGNATURE_ERROR + elif s_key == 'Download Error': + # test https://notexists + message = DOWNLOAD_ERROR + + # see https://github.com/searxng/searxng/pull/1456#issuecomment-1193105023 + # results.append({'answer': message}) + logger.error(message) + + return results + + resp.raise_for_status() + + # append results from matches + + for match_json in json_data['matches']: + + tineye_match = parse_tineye_match(match_json) + if not tineye_match['backlinks']: + continue + + backlink = tineye_match['backlinks'][0] results.append( { 'template': 'images.html', - 'url': url, - 'thumbnail_src': thumbnail_src, - 'source': source, - 'title': title, - 'img_src': img_src, - 'format': image_format, - 'widht': width, - 'height': height, - 'publishedDate': publishedDate, + 'url': backlink['backlink'], + 'thumbnail_src': tineye_match['image_url'], + 'source': backlink['url'], + 'title': backlink['image_name'], + 'img_src': backlink['url'], + 'format': tineye_match['image_format'], + 'widht': tineye_match['width'], + 'height': tineye_match['height'], + 'publishedDate': backlink['crawl_date'], } ) - # Append number of results - results.append({'number_of_results': number_of_results}) + # append number of results + + number_of_results = json_data.get('num_matches') + if number_of_results: + results.append({'number_of_results': number_of_results}) return results diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py index b7c318e53..d828f4be8 100644 --- a/searx/engines/wikidata.py +++ b/searx/engines/wikidata.py @@ -65,6 +65,7 @@ WHERE mwapi:language "%LANGUAGE%". ?item wikibase:apiOutputItem mwapi:item. } + hint:Prior hint:runFirst "true". %WHERE% @@ -93,6 +94,12 @@ WHERE { } """ +# see the property "dummy value" of https://www.wikidata.org/wiki/Q2013 (Wikidata) +# hard coded here to avoid to an additional SPARQL request when the server starts +DUMMY_ENTITY_URLS = set( + "http://www.wikidata.org/entity/" + wid for wid in ("Q4115189", "Q13406268", "Q15397819", "Q17339402") +) + # https://www.w3.org/TR/sparql11-query/#rSTRING_LITERAL1 # https://lists.w3.org/Archives/Public/public-rdf-dawg/2011OctDec/0175.html @@ -177,7 +184,7 @@ def response(resp): for result in jsonresponse.get('results', {}).get('bindings', []): attribute_result = {key: value['value'] for key, value in result.items()} entity_url = attribute_result['item'] - if entity_url not in seen_entities: + if entity_url not in seen_entities and entity_url not in DUMMY_ENTITY_URLS: seen_entities.add(entity_url) results += get_results(attribute_result, attributes, language) else: diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py index cc806a8de..52b1053ed 100644 --- a/searx/engines/wikipedia.py +++ b/searx/engines/wikipedia.py @@ -19,6 +19,9 @@ about = { "results": 'JSON', } + +send_accept_language_header = True + # search-url search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}' supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias' @@ -41,9 +44,6 @@ def request(query, params): language = url_lang(params['language']) params['url'] = search_url.format(title=quote(query), language=language) - if params['language'].lower() in language_variants.get(language, []): - params['headers']['Accept-Language'] = params['language'].lower() - params['headers']['User-Agent'] = searx_useragent() params['raise_for_httperror'] = False params['soft_max_redirects'] = 2 diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py index 406314684..7992adf82 100644 --- a/searx/engines/youtube_noapi.py +++ b/searx/engines/youtube_noapi.py @@ -3,7 +3,6 @@ Youtube (Videos) """ -from datetime import datetime from functools import reduce from json import loads, dumps from urllib.parse import quote_plus @@ -37,6 +36,7 @@ base_youtube_url = 'https://www.youtube.com/watch?v=' # do search-request def request(query, params): + params['cookies']['CONSENT'] = "YES+" if not params['engine_data'].get('next_page_token'): params['url'] = search_url.format(query=quote_plus(query), page=params['pageno']) if params['time_range'] in time_range_dict: @@ -52,7 +52,6 @@ def request(query, params): ) params['headers']['Content-Type'] = 'application/json' - params['headers']['Cookie'] = "CONSENT=YES+cb.%s-17-p0.en+F+941;" % datetime.now().strftime("%Y%m%d") return params diff --git a/searx/engines/zlibrary.py b/searx/engines/zlibrary.py index 81d93ac84..7778f69b6 100644 --- a/searx/engines/zlibrary.py +++ b/searx/engines/zlibrary.py @@ -39,7 +39,7 @@ def init(engine_settings=None): resp = http_get('https://z-lib.org', timeout=5.0) if resp.ok: dom = html.fromstring(resp.text) - base_url = "https:" + extract_text( + base_url = extract_text( eval_xpath(dom, './/a[contains(@class, "domain-check-link") and @data-mode="books"]/@href') ) logger.debug("using base_url: %s" % base_url) |