diff options
Diffstat (limited to 'searx/engines')
| -rw-r--r-- | searx/engines/duckduckgo.py | 30 | ||||
| -rw-r--r-- | searx/engines/scanr_structures.py | 78 |
2 files changed, 100 insertions, 8 deletions
diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 373ce1b2d..d29e4416a 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -11,13 +11,12 @@ @parse url, title, content @todo rewrite to api - @todo language support - (the current used site does not support language-change) """ from urllib import urlencode from lxml.html import fromstring from searx.engines.xpath import extract_text +from searx.languages import language_codes # engine dependent config categories = ['general'] @@ -39,13 +38,28 @@ def request(query, params): offset = (params['pageno'] - 1) * 30 if params['language'] == 'all': - locale = 'en-us' + locale = None else: - locale = params['language'].replace('_', '-').lower() - - params['url'] = url.format( - query=urlencode({'q': query, 'kl': locale}), - offset=offset) + locale = params['language'].split('_') + if len(locale) == 2: + # country code goes first + locale = locale[1].lower() + '-' + locale[0].lower() + else: + # tries to get a country code from language + locale = locale[0].lower() + lang_codes = [x[0] for x in language_codes] + for lc in lang_codes: + lc = lc.split('_') + if locale == lc[0]: + locale = lc[1].lower() + '-' + lc[0].lower() + break + + if locale: + params['url'] = url.format( + query=urlencode({'q': query, 'kl': locale}), offset=offset) + else: + params['url'] = url.format( + query=urlencode({'q': query}), offset=offset) return params diff --git a/searx/engines/scanr_structures.py b/searx/engines/scanr_structures.py new file mode 100644 index 000000000..ad78155ac --- /dev/null +++ b/searx/engines/scanr_structures.py @@ -0,0 +1,78 @@ +""" + ScanR Structures (Science) + + @website https://scanr.enseignementsup-recherche.gouv.fr + @provide-api yes (https://scanr.enseignementsup-recherche.gouv.fr/api/swagger-ui.html) + + @using-api yes + @results JSON + @stable yes + @parse url, title, content, img_src +""" + +from urllib import urlencode +from json import loads, dumps +from dateutil import parser +from searx.utils import html_to_text + +# engine dependent config +categories = ['science'] +paging = True +page_size = 20 + +# search-url +url = 'https://scanr.enseignementsup-recherche.gouv.fr/' +search_url = url + 'api/structures/search' + + +# do search-request +def request(query, params): + + params['url'] = search_url + params['method'] = 'POST' + params['headers']['Content-type'] = "application/json" + params['data'] = dumps({"query": query, + "searchField": "ALL", + "sortDirection": "ASC", + "sortOrder": "RELEVANCY", + "page": params['pageno'], + "pageSize": page_size}) + + return params + + +# get response from search-request +def response(resp): + results = [] + + search_res = loads(resp.text) + + # return empty array if there are no results + if search_res.get('total') < 1: + return [] + + # parse results + for result in search_res['results']: + if 'id' not in result: + continue + + # is it thumbnail or img_src?? + thumbnail = None + if 'logo' in result: + thumbnail = result['logo'] + if thumbnail[0] == '/': + thumbnail = url + thumbnail + + content = None + if 'highlights' in result: + content = result['highlights'][0]['value'] + + # append result + results.append({'url': url + 'structure/' + result['id'], + 'title': result['label'], + # 'thumbnail': thumbnail, + 'img_src': thumbnail, + 'content': html_to_text(content)}) + + # return results + return results |