diff options
Diffstat (limited to 'searx/engines')
| -rw-r--r-- | searx/engines/acgsou.py | 74 | ||||
| -rw-r--r-- | searx/engines/duckduckgo.py | 3 | ||||
| -rw-r--r-- | searx/engines/ina.py | 4 | ||||
| -rw-r--r-- | searx/engines/microsoft_academic.py | 57 | ||||
| -rw-r--r-- | searx/engines/semantic_scholar.py | 42 | ||||
| -rw-r--r-- | searx/engines/seznam.py | 37 | ||||
| -rw-r--r-- | searx/engines/solr.py | 74 | ||||
| -rw-r--r-- | searx/engines/soundcloud.py | 2 | ||||
| -rw-r--r-- | searx/engines/wikipedia.py | 2 | ||||
| -rw-r--r-- | searx/engines/yahoo_news.py | 7 | ||||
| -rw-r--r-- | searx/engines/youtube_noapi.py | 76 |
11 files changed, 235 insertions, 143 deletions
diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py deleted file mode 100644 index ea9793f10..000000000 --- a/searx/engines/acgsou.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later -""" - Acgsou (Japanese Animation/Music/Comics Bittorrent tracker) -""" - -from urllib.parse import urlencode -from lxml import html -from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex - -# about -about = { - "website": 'https://www.acgsou.com/', - "wikidata_id": None, - "official_api_documentation": None, - "use_official_api": False, - "require_api_key": False, - "results": 'HTML', -} - -# engine dependent config -categories = ['files', 'images', 'videos', 'music'] -paging = True - -# search-url -base_url = 'https://www.acgsou.com/' -search_url = base_url + 'search.php?{query}&page={offset}' -# xpath queries -xpath_results = '//table[contains(@class, "list_style table_fixed")]//tr[not(th)]' -xpath_category = './/td[2]/a[1]' -xpath_title = './/td[3]/a[last()]' -xpath_torrent_links = './/td[3]/a' -xpath_filesize = './/td[4]/text()' - - -def request(query, params): - query = urlencode({'keyword': query}) - params['url'] = search_url.format(query=query, offset=params['pageno']) - return params - - -def response(resp): - results = [] - dom = html.fromstring(resp.text) - for result in eval_xpath_list(dom, xpath_results): - # defaults - filesize = 0 - magnet_link = "magnet:?xt=urn:btih:{}&tr=https://tracker.acgsou.com:2710/announce" - - category = extract_text(eval_xpath_getindex(result, xpath_category, 0, default=[])) - page_a = eval_xpath_getindex(result, xpath_title, 0) - title = extract_text(page_a) - href = base_url + page_a.attrib.get('href') - - magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5]) - - filesize_info = eval_xpath_getindex(result, xpath_filesize, 0, default=None) - if filesize_info: - try: - filesize = filesize_info[:-2] - filesize_multiplier = filesize_info[-2:] - filesize = get_torrent_size(filesize, filesize_multiplier) - except: - pass - # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime - content = 'Category: "{category}".' - content = content.format(category=category) - - results.append({'url': href, - 'title': title, - 'content': content, - 'filesize': filesize, - 'magnetlink': magnet_link, - 'template': 'torrent.html'}) - return results diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 92431b137..ae1e36686 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -39,7 +39,8 @@ url = 'https://html.duckduckgo.com/html' url_ping = 'https://duckduckgo.com/t/sl_h' time_range_dict = {'day': 'd', 'week': 'w', - 'month': 'm'} + 'month': 'm', + 'year': 'y'} # specific xpath variables result_xpath = '//div[@class="result results_links results_links_deep web-result "]' # noqa diff --git a/searx/engines/ina.py b/searx/engines/ina.py index 1a47ca51e..efcc03e39 100644 --- a/searx/engines/ina.py +++ b/searx/engines/ina.py @@ -53,9 +53,7 @@ def response(resp): # we get html in a JSON container... response = loads(resp.text) - if "content" not in response: - return [] - dom = html.fromstring(response["content"]) + dom = html.fromstring(response) # parse results for result in dom.xpath(results_xpath): diff --git a/searx/engines/microsoft_academic.py b/searx/engines/microsoft_academic.py index 14de4ac9a..82a5d3550 100644 --- a/searx/engines/microsoft_academic.py +++ b/searx/engines/microsoft_academic.py @@ -3,10 +3,7 @@ Microsoft Academic (Science) """ -from datetime import datetime -from json import loads -from uuid import uuid4 -from urllib.parse import urlencode +from json import dumps, loads from searx.utils import html_to_text # about @@ -21,26 +18,25 @@ about = { categories = ['images'] paging = True -result_url = 'https://academic.microsoft.com/api/search/GetEntityResults?{query}' +search_url = 'https://academic.microsoft.com/api/search' +_paper_url = 'https://academic.microsoft.com/paper/{id}/reference' def request(query, params): - correlation_id = uuid4() - msacademic = uuid4() - time_now = datetime.now() - - params['url'] = result_url.format(query=urlencode({'correlationId': correlation_id})) - params['cookies']['msacademic'] = str(msacademic) - params['cookies']['ai_user'] = 'vhd0H|{now}'.format(now=str(time_now)) + params['url'] = search_url params['method'] = 'POST' - params['data'] = { - 'Query': '@{query}@'.format(query=query), - 'Limit': 10, - 'Offset': params['pageno'] - 1, - 'Filters': '', - 'OrderBy': '', - 'SortAscending': False, - } + params['headers']['content-type'] = 'application/json; charset=utf-8' + params['data'] = dumps({ + 'query': query, + 'queryExpression': '', + 'filters': [], + 'orderBy': 0, + 'skip': (params['pageno'] - 1) * 10, + 'sortAscending': True, + 'take': 10, + 'includeCitationContexts': False, + 'profileId': '', + }) return params @@ -51,10 +47,13 @@ def response(resp): if not response_data: return results - for result in response_data['results']: - url = _get_url(result) - title = result['e']['dn'] - content = _get_content(result) + for result in response_data['pr']: + if 'dn' not in result['paper']: + continue + + title = result['paper']['dn'] + content = _get_content(result['paper']) + url = _paper_url.format(id=result['paper']['id']) results.append({ 'url': url, 'title': html_to_text(title), @@ -64,15 +63,9 @@ def response(resp): return results -def _get_url(result): - if 's' in result['e']: - return result['e']['s'][0]['u'] - return 'https://academic.microsoft.com/#/detail/{pid}'.format(pid=result['id']) - - def _get_content(result): - if 'd' in result['e']: - content = result['e']['d'] + if 'd' in result: + content = result['d'] if len(content) > 300: return content[:300] + '...' return content diff --git a/searx/engines/semantic_scholar.py b/searx/engines/semantic_scholar.py new file mode 100644 index 000000000..297d0cf71 --- /dev/null +++ b/searx/engines/semantic_scholar.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +""" + Semantic Scholar (Science) +""" + +from json import dumps, loads + + +search_url = 'https://www.semanticscholar.org/api/1/search' + + +def request(query, params): + params['url'] = search_url + params['method'] = 'POST' + params['headers']['content-type'] = 'application/json' + params['data'] = dumps({ + "queryString": query, + "page": params['pageno'], + "pageSize": 10, + "sort": "relevance", + "useFallbackRankerService": False, + "useFallbackSearchCluster": False, + "getQuerySuggestions": False, + "authors": [], + "coAuthors": [], + "venues": [], + "performTitleMatch": True, + }) + return params + + +def response(resp): + res = loads(resp.text) + results = [] + for result in res['results']: + results.append({ + 'url': result['primaryPaperLink']['url'], + 'title': result['title']['text'], + 'content': result['paperAbstractTruncated'] + }) + + return results diff --git a/searx/engines/seznam.py b/searx/engines/seznam.py index 1df92a845..faceb0550 100644 --- a/searx/engines/seznam.py +++ b/searx/engines/seznam.py @@ -7,7 +7,12 @@ from urllib.parse import urlencode, urlparse from lxml import html from searx.poolrequests import get from searx.exceptions import SearxEngineAccessDeniedException -from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex +from searx.utils import ( + extract_text, + eval_xpath_list, + eval_xpath_getindex, + eval_xpath, +) # about about = { @@ -26,7 +31,10 @@ def request(query, params): response_index = get(base_url, headers=params['headers'], raise_for_httperror=True) dom = html.fromstring(response_index.text) - url_params = {'q': query} + url_params = { + 'q': query, + 'oq': query, + } for e in eval_xpath_list(dom, '//input[@type="hidden"]'): name = e.get('name') value = e.get('value') @@ -45,20 +53,15 @@ def response(resp): results = [] dom = html.fromstring(resp.content.decode()) - for result_element in eval_xpath_list(dom, '//div[@id="searchpage-root"]//div[@data-dot="results"]/div'): - dot_data = eval_xpath_getindex(result_element, './div/div[@data-dot-data]/@data-dot-data', 0, default=None) - if dot_data is None: - title_element = eval_xpath_getindex(result_element, './/h3/a', 0) - results.append({ - 'url': title_element.get('href'), - 'title': extract_text(title_element), - 'content': extract_text(eval_xpath_getindex(title_element, '../../div[2]', 0)), - }) - elif dot_data == '{"reporter_name":"hint/related/relates"}': - suggestions_element = eval_xpath_getindex(result_element, - './div/div[@data-dot="main-box"]', 0, default=None) - if suggestions_element is not None: - for suggestion in eval_xpath_list(suggestions_element, './/ul/li'): - results.append({'suggestion': extract_text(suggestion)}) + for result_element in eval_xpath_list(dom, '//div[@data-dot="results"]/div'): + result_data = eval_xpath_getindex(result_element, './/div[contains(@class, "Result")]', 0, default=None) + if result_data is None: + continue + title_element = eval_xpath_getindex(result_element, './/h3/a', 0) + results.append({ + 'url': title_element.get('href'), + 'title': extract_text(title_element), + 'content': extract_text(eval_xpath(result_data, './/p[@class="Result-description"]')), + }) return results diff --git a/searx/engines/solr.py b/searx/engines/solr.py new file mode 100644 index 000000000..0bfcbab36 --- /dev/null +++ b/searx/engines/solr.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +""" + Solr +""" + +# pylint: disable=global-statement, missing-function-docstring + +from json import loads +from urllib.parse import urlencode +from searx.exceptions import SearxEngineAPIException + + +base_url = 'http://localhost:8983' +collection = '' +rows = 10 +sort = '' # sorting: asc or desc +field_list = 'name' # list of field names to display on the UI +default_fields = '' # default field to query +query_fields = '' # query fields +_search_url = '' +paging = True + + +def init(_): + if collection == '': + raise ValueError('collection cannot be empty') + + global _search_url + _search_url = base_url + '/solr/' + collection + '/select?{params}' + + +def request(query, params): + query_params = {'q': query, 'rows': rows} + if field_list != '': + query_params['fl'] = field_list + if query_fields != '': + query_params['qf'] = query_fields + if default_fields != '': + query_params['df'] = default_fields + if sort != '': + query_params['sort'] = sort + + if 'pageno' in params: + query_params['start'] = rows * (params['pageno'] - 1) + + params['url'] = _search_url.format(params=urlencode(query_params)) + + return params + + +def response(resp): + resp_json = __get_response(resp) + + results = [] + for result in resp_json['response']['docs']: + r = {key: str(value) for key, value in result.items()} + if len(r) == 0: + continue + r['template'] = 'key-value.html' + results.append(r) + + return results + + +def __get_response(resp): + try: + resp_json = loads(resp.text) + except Exception as e: + raise SearxEngineAPIException("failed to parse response") from e + + if 'error' in resp_json: + raise SearxEngineAPIException(resp_json['error']['msg']) + + return resp_json diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py index 9e414746f..b3e3383bd 100644 --- a/searx/engines/soundcloud.py +++ b/searx/engines/soundcloud.py @@ -55,7 +55,7 @@ def get_client_id(): app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None] # extracts valid app_js urls from soundcloud.com content - for app_js_url in app_js_urls: + for app_js_url in app_js_urls[::-1]: # gets app_js and searches for the clientid response = http_get(app_js_url) if response.ok: diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py index da867c81e..3ad8748fb 100644 --- a/searx/engines/wikipedia.py +++ b/searx/engines/wikipedia.py @@ -76,7 +76,7 @@ def response(resp): if api_result.get('type') != 'standard': return [] - title = api_result['displaytitle'] + title = api_result['title'] wikipedia_link = api_result['content_urls']['desktop']['page'] results.append({'url': wikipedia_link, 'title': title}) diff --git a/searx/engines/yahoo_news.py b/searx/engines/yahoo_news.py index 5f6734cb3..49b3d1bf8 100644 --- a/searx/engines/yahoo_news.py +++ b/searx/engines/yahoo_news.py @@ -22,13 +22,6 @@ from searx.utils import ( from searx.engines.yahoo import parse_url -# pylint: disable=unused-import -from searx.engines.yahoo import ( - _fetch_supported_languages, - supported_languages_url, -) -# pylint: enable=unused-import - logger = logger.getChild('yahoo_news engine') # about diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py index 90b93f0a4..0933eec68 100644 --- a/searx/engines/youtube_noapi.py +++ b/searx/engines/youtube_noapi.py @@ -4,7 +4,7 @@ """ from functools import reduce -from json import loads +from json import loads, dumps from urllib.parse import quote_plus # about @@ -20,12 +20,15 @@ about = { # engine dependent config categories = ['videos', 'music'] paging = True +language_support = False time_range_support = True # search-url base_url = 'https://www.youtube.com/results' search_url = base_url + '?search_query={query}&page={page}' time_range_url = '&sp=EgII{time_range}%253D%253D' +# the key seems to be constant +next_page_url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8' time_range_dict = {'day': 'Ag', 'week': 'Aw', 'month': 'BA', @@ -40,21 +43,70 @@ base_youtube_url = 'https://www.youtube.com/watch?v=' # do search-request def request(query, params): - params['url'] = search_url.format(query=quote_plus(query), - page=params['pageno']) - if params['time_range'] in time_range_dict: - params['url'] += time_range_url.format(time_range=time_range_dict[params['time_range']]) + if not params['engine_data'].get('next_page_token'): + params['url'] = search_url.format(query=quote_plus(query), page=params['pageno']) + if params['time_range'] in time_range_dict: + params['url'] += time_range_url.format(time_range=time_range_dict[params['time_range']]) + else: + params['url'] = next_page_url + params['method'] = 'POST' + params['data'] = dumps({ + 'context': {"client": {"clientName": "WEB", "clientVersion": "2.20210310.12.01"}}, + 'continuation': params['engine_data']['next_page_token'], + }) + params['headers']['Content-Type'] = 'application/json' return params # get response from search-request def response(resp): + if resp.search_params.get('engine_data'): + return parse_next_page_response(resp.text) + return parse_first_page_response(resp.text) + + +def parse_next_page_response(response_text): results = [] + result_json = loads(response_text) + for section in (result_json['onResponseReceivedCommands'][0] + .get('appendContinuationItemsAction')['continuationItems'][0] + .get('itemSectionRenderer')['contents']): + if 'videoRenderer' not in section: + continue + section = section['videoRenderer'] + content = "-" + if 'descriptionSnippet' in section: + content = ' '.join(x['text'] for x in section['descriptionSnippet']['runs']) + results.append({ + 'url': base_youtube_url + section['videoId'], + 'title': ' '.join(x['text'] for x in section['title']['runs']), + 'content': content, + 'author': section['ownerText']['runs'][0]['text'], + 'length': section['lengthText']['simpleText'], + 'template': 'videos.html', + 'embedded': embedded_url.format(videoid=section['videoId']), + 'thumbnail': section['thumbnail']['thumbnails'][-1]['url'], + }) + try: + token = result_json['onResponseReceivedCommands'][0]\ + .get('appendContinuationItemsAction')['continuationItems'][1]\ + .get('continuationItemRenderer')['continuationEndpoint']\ + .get('continuationCommand')['token'] + results.append({ + "engine_data": token, + "key": "next_page_token", + }) + except: + pass + + return results - results_data = resp.text[resp.text.find('ytInitialData'):] - results_data = results_data[results_data.find('{'):results_data.find(';</script>')] +def parse_first_page_response(response_text): + results = [] + results_data = response_text[response_text.find('ytInitialData'):] + results_data = results_data[results_data.find('{'):results_data.find(';</script>')] results_json = loads(results_data) if results_data else {} sections = results_json.get('contents', {})\ .get('twoColumnSearchResultsRenderer', {})\ @@ -63,6 +115,16 @@ def response(resp): .get('contents', []) for section in sections: + if "continuationItemRenderer" in section: + next_page_token = section["continuationItemRenderer"]\ + .get("continuationEndpoint", {})\ + .get("continuationCommand", {})\ + .get("token", "") + if next_page_token: + results.append({ + "engine_data": next_page_token, + "key": "next_page_token", + }) for video_container in section.get('itemSectionRenderer', {}).get('contents', []): video = video_container.get('videoRenderer', {}) videoid = video.get('videoId') |