From 6e5f22e5583cfc2a413e0afac66d3c5ea9f628b1 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 29 Sep 2022 20:54:46 +0200 Subject: [mod] replace engines_languages.json by engines_traits.json Implementations of the *traits* of the engines. Engine's traits are fetched from the origin engine and stored in a JSON file in the *data folder*. Most often traits are languages and region codes and their mapping from SearXNG's representation to the representation in the origin search engine. To load traits from the persistence:: searx.enginelib.traits.EngineTraitsMap.from_data() For new traits new properties can be added to the class:: searx.enginelib.traits.EngineTraits .. hint:: Implementation is downward compatible to the deprecated *supported_languages method* from the vintage implementation. The vintage code is tagged as *deprecated* an can be removed when all engines has been ported to the *traits method*. Signed-off-by: Markus Heiser --- searx/autocomplete.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'searx/autocomplete.py') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index aeb697a14..9b8755218 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -12,7 +12,7 @@ from lxml import etree from httpx import HTTPError from searx import settings -from searx.data import ENGINES_LANGUAGES +from searx.engines import engines from searx.network import get as http_get from searx.exceptions import SearxEngineResponseException @@ -111,7 +111,7 @@ def seznam(query, _lang): def startpage(query, lang): # startpage autocompleter - lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english') + lui = engines['startpage'].supported_languages.get(lang, 'english') # vintage / deprecated url = 'https://startpage.com/suggestions?{query}' resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui}))) data = resp.json() @@ -177,12 +177,19 @@ backends = { } -def search_autocomplete(backend_name, query, lang): +def search_autocomplete(backend_name, query, sxng_locale): backend = backends.get(backend_name) if backend is None: return [] + if engines[backend_name].traits.data_type != "traits_v1": + # vintage / deprecated + if not sxng_locale or sxng_locale == 'all': + sxng_locale = 'en' + else: + sxng_locale = sxng_locale.split('-')[0] + try: - return backend(query, lang) + return backend(query, sxng_locale) except (HTTPError, SearxEngineResponseException): return [] -- cgit v1.2.3 From c1ae2ef57c8a7da5df1f0fdacc0e6e745721b2ae Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 3 Oct 2022 22:42:58 +0200 Subject: [mod] qwant: fetch engine traits (data_type: traits_v1) Implements a fetch_traits function for the Qwant engines. .. note:: Includes migration of the request methode from 'supported_languages' to 'traits' (EngineTraits) object! Signed-off-by: Markus Heiser --- searx/autocomplete.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'searx/autocomplete.py') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 9b8755218..848600e57 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -126,16 +126,16 @@ def swisscows(query, _lang): return resp -def qwant(query, lang): - # qwant autocompleter (additional parameter : lang=en_en&count=xxx ) - url = 'https://api.qwant.com/api/suggest?{query}' - - resp = get(url.format(query=urlencode({'q': query, 'lang': lang}))) - +def qwant(query, sxng_locale): + """Autocomplete from Qwant. Supports Qwant's regions.""" results = [] + locale = engines['qwant'].traits.get_region(sxng_locale, 'en_US') + url = 'https://api.qwant.com/v3/suggest?{query}' + resp = get(url.format(query=urlencode({'q': query, 'locale': locale, 'version': '2'}))) + if resp.ok: - data = loads(resp.text) + data = resp.json() if data['status'] == 'success': for item in data['data']['items']: results.append(item['value']) -- cgit v1.2.3 From 858aa3e6043a5102aec1b05e94ef1d65059f8898 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 28 Oct 2022 19:12:59 +0200 Subject: [mod] wikipedia & wikidata: upgrade to data_type: traits_v1 BTW this fix an issue in wikipedia: SearXNG's locales zh-TW and zh-HK are now using language `zh-classical` from wikipedia (and not `zh`). Signed-off-by: Markus Heiser --- searx/autocomplete.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) (limited to 'searx/autocomplete.py') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 848600e57..53e19905c 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -143,14 +143,31 @@ def qwant(query, sxng_locale): return results -def wikipedia(query, lang): - # wikipedia autocompleter - url = 'https://' + lang + '.wikipedia.org/w/api.php?action=opensearch&{0}&limit=10&namespace=0&format=json' +def wikipedia(query, sxng_locale): + """Autocomplete from Wikipedia. Supports Wikipedia's languages (aka netloc).""" + results = [] + eng_traits = engines['wikipedia'].traits + wiki_lang = eng_traits.get_language(sxng_locale, 'en') + wiki_netloc = eng_traits.custom['wiki_netloc'].get(wiki_lang, 'en.wikipedia.org') + + url = 'https://{wiki_netloc}/w/api.php?{args}' + args = urlencode( + { + 'action': 'opensearch', + 'format': 'json', + 'formatversion': '2', + 'search': query, + 'namespace': '0', + 'limit': '10', + } + ) + resp = get(url.format(args=args, wiki_netloc=wiki_netloc)) + if resp.ok: + data = resp.json() + if len(data) > 1: + results = data[1] - resp = loads(get(url.format(urlencode(dict(search=query)))).text) - if len(resp) > 1: - return resp[1] - return [] + return results def yandex(query, _lang): -- cgit v1.2.3 From e9afc4f8ce2df0b02a9b3d8664b66307255b056e Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 30 Oct 2022 11:23:20 +0100 Subject: [mod] Startpage: reversed engineered & upgrade to data_type: traits_v1 One reason for the often seen CAPTCHA of the Startpage requests are the incomplete requests SearXNG sends to startpage.com: this patch is a complete new implementation of the ``request()`` function, reversed engineered from the Startpage's search form. The new implementation: - use traits of data_type: traits_v1 and drop deprecated data_type: supported_languages - adds time-range support - adds save-search support - fix searxng/searxng/issues 1884 - fix searxng/searxng/issues 1081 --> improvements to avoid CAPTCHA In preparation for more categories (News, Images, Videos ..) from Startpage, the variable ``startpage_categ`` was set up. The default value is ``web`` and other categories from Startpage are not yet implemented. Signed-off-by: Markus Heiser --- searx/autocomplete.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'searx/autocomplete.py') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 53e19905c..acea05c32 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -109,9 +109,9 @@ def seznam(query, _lang): ] -def startpage(query, lang): - # startpage autocompleter - lui = engines['startpage'].supported_languages.get(lang, 'english') # vintage / deprecated +def startpage(query, sxng_locale): + """Autocomplete from Startpage. Supports Startpage's languages""" + lui = engines['startpage'].traits.get_language(sxng_locale, 'english') url = 'https://startpage.com/suggestions?{query}' resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui}))) data = resp.json() -- cgit v1.2.3 From c80e82a855fd388c6080066da892b9723d6037c9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 5 Nov 2022 15:10:52 +0100 Subject: [mod] DuckDuckGo: reversed engineered & upgrade to data_type: traits_v1 Partial reverse engineering of the DuckDuckGo (DDG) engines including a improved language and region handling based on the enigne.traits_v1 data. - DDG Lite - DDG Instant Answer API - DDG Images - DDG Weather docs/src/searx.engine.duckduckgo.rst: Online documentation of the DDG engines (make docs.live) searx/data/engine_traits.json Add data type "traits_v1" generated by the fetch_traits() functions from: - "duckduckgo" (WEB), - "duckduckgo images" and - "duckduckgo weather" and remove data from obsolete data type "supported_languages". searx/autocomplete.py: Reversed engineered Autocomplete from DDG. Supports DDG's languages. searx/engines/duckduckgo.py: - fetch_traits(): Fetch languages & regions from DDG. - get_ddg_lang(): Get DDG's language identifier from SearXNG's locale. DDG defines its languages by region codes. DDG-Lite does not offer a language selection to the user, only a region can be selected by the user. - Cache ``vqd`` value: The vqd value depends on the query string and is needed for the follow up pages or the images loaded by a XMLHttpRequest (DDG images). The ``vqd`` value of a search term is stored for 10min in the redis DB. - DDG Lite engine: reversed engineered request method with improved Language and region support and better ``vqd`` handling. searx/engines/duckduckgo_definitions.py: DDG Instant Answer API The *instant answers* API does not support languages, or at least we could not find out how language support should work. It seems that most of the features are based on English terms. searx/engines/duckduckgo_images.py: DDG Images Reversed engineered request method. Improved language and region handling based on cookies and the enigne.traits_v1 data. Response: add image format to the result list searx/engines/duckduckgo_weather.py: DDG Weather Improved language and region handling based on cookies and the enigne.traits_v1 data. Signed-off-by: Markus Heiser --- searx/autocomplete.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'searx/autocomplete.py') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index acea05c32..4eabd880f 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -61,14 +61,24 @@ def dbpedia(query, _lang): return results -def duckduckgo(query, _lang): - # duckduckgo autocompleter - url = 'https://ac.duckduckgo.com/ac/?{0}&type=list' +def duckduckgo(query, sxng_locale): + """Autocomplete from DuckDuckGo. Supports DuckDuckGo's languages""" - resp = loads(get(url.format(urlencode(dict(q=query)))).text) - if len(resp) > 1: - return resp[1] - return [] + traits = engines['duckduckgo'].traits + args = { + 'q': query, + 'kl': traits.get_region(sxng_locale, traits.all_locale), + } + + url = 'https://duckduckgo.com/ac/?type=list&' + urlencode(args) + resp = get(url) + + ret_val = [] + if resp.ok: + j = resp.json() + if len(j) > 1: + ret_val = j[1] + return ret_val def google(query, lang): -- cgit v1.2.3 From 249989955497cd048fa3312d115971282983b269 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 4 Dec 2022 22:57:22 +0100 Subject: [mod] Google: reversed engineered & upgrade to data_type: traits_v1 Partial reverse engineering of the Google engines including a improved language and region handling based on the engine.traits_v1 data. When ever possible the implementations of the Google engines try to make use of the async REST APIs. The get_lang_info() has been generalized to a get_google_info() function / especially the region handling has been improved by adding the cr parameter. searx/data/engine_traits.json Add data type "traits_v1" generated by the fetch_traits() functions from: - Google (WEB), - Google images, - Google news, - Google scholar and - Google videos and remove data from obsolete data type "supported_languages". A traits.custom type that maps region codes to *supported_domains* is fetched from https://www.google.com/supported_domains searx/autocomplete.py: Reversed engineered autocomplete from Google WEB. Supports Google's languages and subdomains. The old API suggestqueries.google.com/complete has been replaced by the async REST API: https://{subdomain}/complete/search?{args} searx/engines/google.py Reverse engineering and extensive testing .. - fetch_traits(): Fetch languages & regions from Google properties. - always use the async REST API (formally known as 'use_mobile_ui') - use *supported_domains* from traits - improved the result list by fetching './/div[@data-content-feature]' and parsing the type of the various *content features* --> thumbnails are added searx/engines/google_images.py Reverse engineering and extensive testing .. - fetch_traits(): Fetch languages & regions from Google properties. - use *supported_domains* from traits - if exists, freshness_date is added to the result - issue 1864: result list has been improved a lot (due to the new cr parameter) searx/engines/google_news.py Reverse engineering and extensive testing .. - fetch_traits(): Fetch languages & regions from Google properties. *supported_domains* is not needed but a ceid list has been added. - different region handling compared to Google WEB - fixed for various languages & regions (due to the new ceid parameter) / avoid CONSENT page - Google News do no longer support time range - result list has been fixed: XPath of pub_date and pub_origin searx/engines/google_videos.py - fetch_traits(): Fetch languages & regions from Google properties. - use *supported_domains* from traits - add paging support - implement a async request ('asearch': 'arc' & 'async': 'use_ac:true,_fmt:html') - simplified code (thanks to '_fmt:html' request) - issue 1359: fixed xpath of video length data searx/engines/google_scholar.py - fetch_traits(): Fetch languages & regions from Google properties. - use *supported_domains* from traits - request(): include patents & citations - response(): fixed CAPTCHA detection (Scholar has its own CATCHA manager) - hardening XPath to iterate over results - fixed XPath of pub_type (has been change from gs_ct1 to gs_cgt2 class) - issue 1769 fixed: new request implementation is no longer incompatible Signed-off-by: Markus Heiser --- searx/autocomplete.py | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) (limited to 'searx/autocomplete.py') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 4eabd880f..d659d110f 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -5,14 +5,17 @@ """ # pylint: disable=use-dict-literal -from json import loads +import json from urllib.parse import urlencode -from lxml import etree +import lxml from httpx import HTTPError from searx import settings -from searx.engines import engines +from searx.engines import ( + engines, + google, +) from searx.network import get as http_get from searx.exceptions import SearxEngineResponseException @@ -55,7 +58,7 @@ def dbpedia(query, _lang): results = [] if response.ok: - dom = etree.fromstring(response.content) + dom = lxml.etree.fromstring(response.content) results = dom.xpath('//Result/Label//text()') return results @@ -81,18 +84,32 @@ def duckduckgo(query, sxng_locale): return ret_val -def google(query, lang): - # google autocompleter - autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&' +def google_complete(query, sxng_locale): + """Autocomplete from Google. Supports Google's languages and subdomains + (:py:obj:`searx.engines.google.get_google_info`) by using the async REST + API:: - response = get(autocomplete_url + urlencode(dict(hl=lang, q=query))) + https://{subdomain}/complete/search?{args} - results = [] + """ - if response.ok: - dom = etree.fromstring(response.text) - results = dom.xpath('//suggestion/@data') + google_info = google.get_google_info({'searxng_locale': sxng_locale}, engines['google'].traits) + url = 'https://{subdomain}/complete/search?{args}' + args = urlencode( + { + 'q': query, + 'client': 'gws-wiz', + 'hl': google_info['params']['hl'], + } + ) + results = [] + resp = get(url.format(subdomain=google_info['subdomain'], args=args)) + if resp.ok: + json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1] + data = json.loads(json_txt) + for item in data[0]: + results.append(lxml.html.fromstring(item[0]).text_content()) return results @@ -132,7 +149,7 @@ def swisscows(query, _lang): # swisscows autocompleter url = 'https://swisscows.ch/api/suggest?{query}&itemsCount=5' - resp = loads(get(url.format(query=urlencode({'query': query}))).text) + resp = json.loads(get(url.format(query=urlencode({'query': query}))).text) return resp @@ -184,7 +201,7 @@ def yandex(query, _lang): # yandex autocompleter url = "https://suggest.yandex.com/suggest-ff.cgi?{0}" - resp = loads(get(url.format(urlencode(dict(part=query)))).text) + resp = json.loads(get(url.format(urlencode(dict(part=query)))).text) if len(resp) > 1: return resp[1] return [] @@ -193,7 +210,7 @@ def yandex(query, _lang): backends = { 'dbpedia': dbpedia, 'duckduckgo': duckduckgo, - 'google': google, + 'google': google_complete, 'seznam': seznam, 'startpage': startpage, 'swisscows': swisscows, -- cgit v1.2.3 From 4d4aa13e1f1d254e5d57c67973a7809d9c1e21f9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 30 Dec 2022 18:28:02 +0100 Subject: [mod] remove obsolete EngineTraits.supported_languages All engines has been migrated from ``supported_languages`` to the ``fetch_traits`` concept. There is no longer a need for the obsolete code that implements the ``supported_languages`` concept. Signed-off-by: Markus Heiser --- searx/autocomplete.py | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'searx/autocomplete.py') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index d659d110f..ad9903f36 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -19,9 +19,6 @@ from searx.engines import ( from searx.network import get as http_get from searx.exceptions import SearxEngineResponseException -# a fetch_supported_languages() for XPath engines isn't available right now -# _brave = ENGINES_LANGUAGES['brave'].keys() - def get(*args, **kwargs): if 'timeout' not in kwargs: @@ -225,14 +222,6 @@ def search_autocomplete(backend_name, query, sxng_locale): backend = backends.get(backend_name) if backend is None: return [] - - if engines[backend_name].traits.data_type != "traits_v1": - # vintage / deprecated - if not sxng_locale or sxng_locale == 'all': - sxng_locale = 'en' - else: - sxng_locale = sxng_locale.split('-')[0] - try: return backend(query, sxng_locale) except (HTTPError, SearxEngineResponseException): -- cgit v1.2.3