diff options
| -rw-r--r-- | searx/data/useragents.json | 10 | ||||
| -rw-r--r-- | searx/engines/faroo.py | 96 | ||||
| -rw-r--r-- | searx/results.py | 4 | ||||
| -rw-r--r-- | searx/search.py | 17 | ||||
| -rw-r--r-- | searx/settings.yml | 5 | ||||
| -rwxr-xr-x | searx/webapp.py | 25 |
6 files changed, 29 insertions, 128 deletions
diff --git a/searx/data/useragents.json b/searx/data/useragents.json index 5ba8c5be8..d464ec5f6 100644 --- a/searx/data/useragents.json +++ b/searx/data/useragents.json @@ -1,12 +1,12 @@ { "versions": [ - "74.0", - "73.0.1", - "73.0" + "75.0", + "74.0.1", + "74.0" ], "os": [ - "Windows NT 10; WOW64", + "Windows NT 10.0; WOW64", "X11; Linux x86_64" ], "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}" -}
\ No newline at end of file +} diff --git a/searx/engines/faroo.py b/searx/engines/faroo.py deleted file mode 100644 index a36ecf778..000000000 --- a/searx/engines/faroo.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - Faroo (Web, News) - - @website http://www.faroo.com - @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key - - @using-api no - @results JSON - @stable yes - @parse url, title, content, publishedDate, img_src -""" - -from json import loads -import datetime -from searx.utils import searx_useragent -from searx.url_utils import urlencode - -# engine dependent config -categories = ['general', 'news'] -paging = True -language_support = True -number_of_results = 10 - -# search-url -url = 'http://www.faroo.com/' -search_url = url + 'instant.json?{query}'\ - '&start={offset}'\ - '&length={number_of_results}'\ - '&l={language}'\ - '&src={categorie}'\ - '&i=false'\ - '&c=false' - -search_category = {'general': 'web', - 'news': 'news'} - - -# do search-request -def request(query, params): - offset = (params['pageno'] - 1) * number_of_results + 1 - categorie = search_category.get(params['category'], 'web') - - if params['language'] == 'all': - language = 'en' - else: - language = params['language'].split('-')[0] - - # if language is not supported, put it in english - if language != 'en' and\ - language != 'de' and\ - language != 'zh': - language = 'en' - - params['url'] = search_url.format(offset=offset, - number_of_results=number_of_results, - query=urlencode({'q': query}), - language=language, - categorie=categorie) - - params['headers']['Referer'] = url - - return params - - -# get response from search-request -def response(resp): - # HTTP-Code 429: rate limit exceeded - if resp.status_code == 429: - raise Exception("rate limit has been exceeded!") - - results = [] - - search_res = loads(resp.text) - - # return empty array if there are no results - if not search_res.get('results', {}): - return [] - - # parse results - for result in search_res['results']: - publishedDate = None - result_json = {'url': result['url'], 'title': result['title'], - 'content': result['kwic']} - if result['news']: - result_json['publishedDate'] = \ - datetime.datetime.fromtimestamp(result['date'] / 1000.0) - - # append image result if image url is set - if result['iurl']: - result_json['template'] = 'videos.html' - result_json['thumbnail'] = result['iurl'] - - results.append(result_json) - - # return results - return results diff --git a/searx/results.py b/searx/results.py index 3b1e4bd62..02ab9efb1 100644 --- a/searx/results.py +++ b/searx/results.py @@ -345,8 +345,8 @@ class ResultContainer(object): return 0 return resultnum_sum / len(self._number_of_results) - def add_unresponsive_engine(self, engine_error): - self.unresponsive_engines.add(engine_error) + def add_unresponsive_engine(self, engine_name, error_type, error_message=None): + self.unresponsive_engines.add((engine_name, error_type, error_message)) def add_timing(self, engine_name, engine_time, page_load_time): self.timings.append({ diff --git a/searx/search.py b/searx/search.py index 2dcc4c8f7..661add7ca 100644 --- a/searx/search.py +++ b/searx/search.py @@ -127,11 +127,7 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e)) except Exception as e: record_offline_engine_stats_on_error(engine, result_container, start_time) - - result_container.add_unresponsive_engine(( - engine_name, - u'{0}: {1}'.format(gettext('unexpected crash'), e), - )) + result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e)) logger.exception('engine {0} : exception : {1}'.format(engine_name, e)) @@ -186,24 +182,21 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont engine.stats['errors'] += 1 if (issubclass(e.__class__, requests.exceptions.Timeout)): - result_container.add_unresponsive_engine((engine_name, gettext('timeout'))) + result_container.add_unresponsive_engine(engine_name, 'timeout') # requests timeout (connect or read) logger.error("engine {0} : HTTP requests timeout" "(search duration : {1} s, timeout: {2} s) : {3}" .format(engine_name, engine_time, timeout_limit, e.__class__.__name__)) requests_exception = True elif (issubclass(e.__class__, requests.exceptions.RequestException)): - result_container.add_unresponsive_engine((engine_name, gettext('request exception'))) + result_container.add_unresponsive_engine(engine_name, 'request exception') # other requests exception logger.exception("engine {0} : requests exception" "(search duration : {1} s, timeout: {2} s) : {3}" .format(engine_name, engine_time, timeout_limit, e)) requests_exception = True else: - result_container.add_unresponsive_engine(( - engine_name, - u'{0}: {1}'.format(gettext('unexpected crash'), e), - )) + result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e)) # others errors logger.exception('engine {0} : exception : {1}'.format(engine_name, e)) @@ -238,7 +231,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim remaining_time = max(0.0, timeout_limit - (time() - start_time)) th.join(remaining_time) if th.isAlive(): - result_container.add_unresponsive_engine((th._engine_name, gettext('timeout'))) + result_container.add_unresponsive_engine(th._engine_name, 'timeout') logger.warning('engine timeout: {0}'.format(th._engine_name)) diff --git a/searx/settings.yml b/searx/settings.yml index b51db0ed2..b0dee9fed 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -219,11 +219,6 @@ engines: shortcut : et disabled : True - - name : faroo - engine : faroo - shortcut : fa - disabled : True - - name : 1x engine : www1x shortcut : 1x diff --git a/searx/webapp.py b/searx/webapp.py index c910230ab..8c3531069 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -56,6 +56,7 @@ from flask import ( from babel.support import Translations import flask_babel from flask_babel import Babel, gettext, format_date, format_decimal +from flask.ctx import has_request_context from flask.json import jsonify from searx import brand from searx import settings, searx_dir, searx_debug @@ -165,13 +166,11 @@ _flask_babel_get_translations = flask_babel.get_translations # monkey patch for flask_babel.get_translations def _get_translations(): - translation_locale = request.form.get('use-translation') - if translation_locale: + if has_request_context() and request.form.get('use-translation') == 'oc': babel_ext = flask_babel.current_app.extensions['babel'] - translation = Translations.load(next(babel_ext.translation_directories), 'oc') - else: - translation = _flask_babel_get_translations() - return translation + return Translations.load(next(babel_ext.translation_directories), 'oc') + + return _flask_babel_get_translations() flask_babel.get_translations = _get_translations @@ -627,7 +626,7 @@ def index(): 'corrections': list(result_container.corrections), 'infoboxes': result_container.infoboxes, 'suggestions': list(result_container.suggestions), - 'unresponsive_engines': list(result_container.unresponsive_engines)}, + 'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)}, # noqa default=lambda item: list(item) if isinstance(item, set) else item), mimetype='application/json') elif output_format == 'csv': @@ -695,7 +694,7 @@ def index(): corrections=correction_urls, infoboxes=result_container.infoboxes, paging=result_container.paging, - unresponsive_engines=result_container.unresponsive_engines, + unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines), current_language=match_language(search_query.lang, LANGUAGE_CODES, fallback=request.preferences.get_value("language")), @@ -706,6 +705,16 @@ def index(): ) +def __get_translated_errors(unresponsive_engines): + translated_errors = [] + for unresponsive_engine in unresponsive_engines: + error_msg = gettext(unresponsive_engine[1]) + if unresponsive_engine[2]: + error_msg = "{} {}".format(error_msg, unresponsive_engine[2]) + translated_errors.append((unresponsive_engine[0], error_msg)) + return translated_errors + + @app.route('/about', methods=['GET']) def about(): """Render about page""" |