diff options
Diffstat (limited to 'searx/webapp.py')
| -rwxr-xr-x[-rw-r--r--] | searx/webapp.py | 314 |
1 files changed, 246 insertions, 68 deletions
diff --git a/searx/webapp.py b/searx/webapp.py index 3afdbefd8..f3f5f21b8 100644..100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -41,15 +41,24 @@ except: logger.critical("cannot import dependency: pygments") from sys import exit exit(1) -from cgi import escape +try: + from cgi import escape +except: + from html import escape +from six import next from datetime import datetime, timedelta -from werkzeug.contrib.fixers import ProxyFix +from time import time +from werkzeug.middleware.proxy_fix import ProxyFix from flask import ( Flask, request, render_template, url_for, Response, make_response, redirect, send_from_directory ) +from babel.support import Translations +import flask_babel from flask_babel import Babel, gettext, format_date, format_decimal +from flask.ctx import has_request_context from flask.json import jsonify +from searx import brand from searx import settings, searx_dir, searx_debug from searx.exceptions import SearxParameterException from searx.engines import ( @@ -58,17 +67,19 @@ from searx.engines import ( from searx.utils import ( UnicodeWriter, highlight_content, html_to_text, get_resources_directory, get_static_files, get_result_templates, get_themes, gen_useragent, - dict_subset, prettify_url + dict_subset, prettify_url, match_language ) from searx.version import VERSION_STRING -from searx.languages import language_codes +from searx.languages import language_codes as languages from searx.search import SearchWithPlugins, get_search_query_from_webapp from searx.query import RawTextQuery from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.plugins import plugins -from searx.preferences import Preferences, ValidationException +from searx.plugins.oa_doi_rewrite import get_doi_resolver +from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES from searx.answerers import answerers from searx.url_utils import urlencode, urlparse, urljoin +from searx.utils import new_hmac # check if the pyopenssl package is installed. # It is needed for SSL connection without trouble, see #298 @@ -86,6 +97,11 @@ except: if sys.version_info[0] == 3: unicode = str + PY3 = True +else: + PY3 = False + logger.warning('\033[1;31m *** Deprecation Warning ***\033[0m') + logger.warning('\033[1;31m Python2 is deprecated\033[0m') # serve pages with HTTP/1.1 from werkzeug.serving import WSGIRequestHandler @@ -118,6 +134,7 @@ app = Flask( app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True +app.jinja_env.add_extension('jinja2.ext.loopcontrols') app.secret_key = settings['server']['secret_key'] if not searx_debug \ @@ -127,8 +144,8 @@ if not searx_debug \ babel = Babel(app) -rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he', - 'ku', 'mzn', 'pnb'', ''ps', 'sd', 'ug', 'ur', 'yi'] +rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'fa_IR', 'glk', 'he', + 'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi'] # used when translating category names _category_names = (gettext('files'), @@ -144,22 +161,53 @@ _category_names = (gettext('files'), outgoing_proxies = settings['outgoing'].get('proxies') or None +_flask_babel_get_translations = flask_babel.get_translations + + +# monkey patch for flask_babel.get_translations +def _get_translations(): + if has_request_context() and request.form.get('use-translation') == 'oc': + babel_ext = flask_babel.current_app.extensions['babel'] + return Translations.load(next(babel_ext.translation_directories), 'oc') + + return _flask_babel_get_translations() + + +flask_babel.get_translations = _get_translations + + +def _get_browser_language(request, lang_list): + for lang in request.headers.get("Accept-Language", "en").split(","): + if ';' in lang: + lang = lang.split(';')[0] + locale = match_language(lang, lang_list, fallback=None) + if locale is not None: + return locale + return settings['search']['default_lang'] or 'en' + @babel.localeselector def get_locale(): - locale = request.accept_languages.best_match(settings['locales'].keys()) + locale = _get_browser_language(request, settings['locales'].keys()) + + logger.debug("default locale from browser info is `%s`", locale) if request.preferences.get_value('locale') != '': locale = request.preferences.get_value('locale') - if 'locale' in request.args\ - and request.args['locale'] in settings['locales']: - locale = request.args['locale'] - if 'locale' in request.form\ and request.form['locale'] in settings['locales']: locale = request.form['locale'] + if locale == 'zh_TW': + locale = 'zh_Hant_TW' + + if locale == 'oc': + request.form['use-translation'] = 'oc' + locale = 'fr_FR' + + logger.debug("selected locale is `%s`", locale) + return locale @@ -287,10 +335,13 @@ def image_proxify(url): if not request.preferences.get_value('image_proxy'): return url + if url.startswith('data:image/jpeg;base64,'): + return url + if settings.get('result_proxy'): return proxify(url) - h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest() + h = new_hmac(settings['server']['secret_key'], url.encode('utf-8')) return '{0}?{1}'.format(url_for('image_proxy'), urlencode(dict(url=url.encode('utf-8'), h=h))) @@ -335,7 +386,9 @@ def render(template_name, override_theme=None, **kwargs): if 'autocomplete' not in kwargs: kwargs['autocomplete'] = request.preferences.get_value('autocomplete') - if get_locale() in rtl_locales and 'rtl' not in kwargs: + locale = request.preferences.get_value('locale') + + if locale in rtl_locales and 'rtl' not in kwargs: kwargs['rtl'] = True kwargs['searx_version'] = VERSION_STRING @@ -344,9 +397,10 @@ def render(template_name, override_theme=None, **kwargs): kwargs['safesearch'] = str(request.preferences.get_value('safesearch')) - kwargs['language_codes'] = language_codes + kwargs['language_codes'] = languages if 'current_language' not in kwargs: - kwargs['current_language'] = request.preferences.get_value('language') + kwargs['current_language'] = match_language(request.preferences.get_value('language'), + LANGUAGE_CODES) # override url_for function in templates kwargs['url_for'] = url_for_theme @@ -371,6 +425,10 @@ def render(template_name, override_theme=None, **kwargs): kwargs['unicode'] = unicode + kwargs['preferences'] = request.preferences + + kwargs['brand'] = brand + kwargs['scripts'] = set() for plugin in request.user_plugins: for script in plugin.js_dependencies: @@ -387,6 +445,8 @@ def render(template_name, override_theme=None, **kwargs): @app.before_request def pre_request(): + request.start_time = time() + request.timings = [] request.errors = [] preferences = Preferences(themes, list(categories.keys()), engines, plugins) @@ -402,11 +462,21 @@ def pre_request(): for k, v in request.args.items(): if k not in request.form: request.form[k] = v - try: - preferences.parse_dict(request.form) - except Exception as e: - logger.exception('invalid settings') - request.errors.append(gettext('Invalid settings')) + + if request.form.get('preferences'): + preferences.parse_encoded_data(request.form['preferences']) + else: + try: + preferences.parse_dict(request.form) + except Exception as e: + logger.exception('invalid settings') + request.errors.append(gettext('Invalid settings')) + + # init search language and locale + if not preferences.get_value("language"): + preferences.parse_dict({"language": _get_browser_language(request, LANGUAGE_CODES)}) + if not preferences.get_value("locale"): + preferences.parse_dict({"locale": get_locale()}) # request.user_plugins request.user_plugins = [] @@ -418,6 +488,21 @@ def pre_request(): request.user_plugins.append(plugin) +@app.after_request +def post_request(response): + total_time = time() - request.start_time + timings_all = ['total;dur=' + str(round(total_time * 1000, 3))] + if len(request.timings) > 0: + timings = sorted(request.timings, key=lambda v: v['total']) + timings_total = ['total_' + str(i) + '_' + v['engine'] + + ';dur=' + str(round(v['total'] * 1000, 3)) for i, v in enumerate(timings)] + timings_load = ['load_' + str(i) + '_' + v['engine'] + + ';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings)] + timings_all = timings_all + timings_total + timings_load + response.headers.add('Server-Timing', ', '.join(timings_all)) + return response + + def index_error(output_format, error_message): if output_format == 'json': return Response(json.dumps({'error': error_message}), @@ -470,9 +555,10 @@ def index(): # search search_query = None + raw_text_query = None result_container = None try: - search_query = get_search_query_from_webapp(request.preferences, request.form) + search_query, raw_text_query = get_search_query_from_webapp(request.preferences, request.form) # search = Search(search_query) # without plugins search = SearchWithPlugins(search_query, request.user_plugins, request) result_container = search.search() @@ -495,19 +581,24 @@ def index(): # UI advanced_search = request.form.get('advanced_search', None) + # Server-Timing header + request.timings = result_container.get_timings() + # output for result in results: if output_format == 'html': if 'content' in result and result['content']: result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query) - result['title'] = highlight_content(escape(result['title'] or u''), search_query.query) + if 'title' in result and result['title']: + result['title'] = highlight_content(escape(result['title'] or u''), search_query.query) else: if result.get('content'): result['content'] = html_to_text(result['content']).strip() # removing html content and whitespace duplications result['title'] = ' '.join(html_to_text(result['title']).strip().split()) - result['pretty_url'] = prettify_url(result['url']) + if 'url' in result: + result['pretty_url'] = prettify_url(result['url']) # TODO, check if timezone is calculated right if 'publishedDate' in result: @@ -535,24 +626,38 @@ def index(): 'corrections': list(result_container.corrections), 'infoboxes': result_container.infoboxes, 'suggestions': list(result_container.suggestions), - 'unresponsive_engines': list(result_container.unresponsive_engines)}), + 'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)}, # noqa + default=lambda item: list(item) if isinstance(item, set) else item), mimetype='application/json') elif output_format == 'csv': csv = UnicodeWriter(StringIO()) - keys = ('title', 'url', 'content', 'host', 'engine', 'score') + keys = ('title', 'url', 'content', 'host', 'engine', 'score', 'type') csv.writerow(keys) for row in results: row['host'] = row['parsed_url'].netloc + row['type'] = 'result' + csv.writerow([row.get(key, '') for key in keys]) + for a in result_container.answers: + row = {'title': a, 'type': 'answer'} + csv.writerow([row.get(key, '') for key in keys]) + for a in result_container.suggestions: + row = {'title': a, 'type': 'suggestion'} + csv.writerow([row.get(key, '') for key in keys]) + for a in result_container.corrections: + row = {'title': a, 'type': 'correction'} csv.writerow([row.get(key, '') for key in keys]) csv.stream.seek(0) response = Response(csv.stream.read(), mimetype='application/csv') - cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query) + cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query.decode('utf-8')) response.headers.add('Content-Disposition', cont_disp) return response elif output_format == 'rss': response_rss = render( 'opensearch_response_rss.xml', results=results, + answers=result_container.answers, + corrections=result_container.corrections, + suggestions=result_container.suggestions, q=request.form['q'], number_of_results=number_of_results, base_url=get_base_url(), @@ -560,6 +665,21 @@ def index(): ) return Response(response_rss, mimetype='text/xml') + # HTML output format + + # suggestions: use RawTextQuery to get the suggestion URLs with the same bang + suggestion_urls = list(map(lambda suggestion: { + 'url': raw_text_query.changeSearchQuery(suggestion).getFullQuery(), + 'title': suggestion + }, + result_container.suggestions)) + + correction_urls = list(map(lambda correction: { + 'url': raw_text_query.changeSearchQuery(correction).getFullQuery(), + 'title': correction + }, + result_container.corrections)) + # return render( 'results.html', results=results, @@ -569,19 +689,32 @@ def index(): time_range=search_query.time_range, number_of_results=format_decimal(number_of_results), advanced_search=advanced_search, - suggestions=result_container.suggestions, + suggestions=suggestion_urls, answers=result_container.answers, - corrections=result_container.corrections, + corrections=correction_urls, infoboxes=result_container.infoboxes, paging=result_container.paging, - unresponsive_engines=result_container.unresponsive_engines, - current_language=search_query.lang, + unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines), + current_language=match_language(search_query.lang, + LANGUAGE_CODES, + fallback=request.preferences.get_value("language")), base_url=get_base_url(), theme=get_current_theme_name(), - favicons=global_favicons[themes.index(get_current_theme_name())] + favicons=global_favicons[themes.index(get_current_theme_name())], + timeout_limit=request.form.get('timeout_limit', None) ) +def __get_translated_errors(unresponsive_engines): + translated_errors = [] + for unresponsive_engine in unresponsive_engines: + error_msg = gettext(unresponsive_engine[1]) + if unresponsive_engine[2]: + error_msg = "{} {}".format(error_msg, unresponsive_engine[2]) + translated_errors.append((unresponsive_engine[0], error_msg)) + return translated_errors + + @app.route('/about', methods=['GET']) def about(): """Render about page""" @@ -598,7 +731,10 @@ def autocompleter(): disabled_engines = request.preferences.engines.get_disabled() # parse query - raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines) + if PY3: + raw_text_query = RawTextQuery(request.form.get('q', b''), disabled_engines) + else: + raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines) raw_text_query.parse_query() # check if search query is set @@ -611,8 +747,11 @@ def autocompleter(): # parse searx specific autocompleter results like !bang raw_results = searx_bang(raw_text_query) - # normal autocompletion results only appear if max 3 inner results returned - if len(raw_results) <= 3 and completer: + # normal autocompletion results only appear if no inner results returned + # and there is a query part besides the engine and language bangs + if len(raw_results) == 0 and completer and (len(raw_text_query.query_parts) > 1 or + (len(raw_text_query.languages) == 0 and + not raw_text_query.specific)): # get language from cookie language = request.preferences.get_value('language') if not language or language == 'all': @@ -662,17 +801,25 @@ def preferences(): # stats for preferences page stats = {} + engines_by_category = {} for c in categories: + engines_by_category[c] = [] for e in categories[c]: + if not request.preferences.validate_token(e): + continue + stats[e.name] = {'time': None, 'warn_timeout': False, 'warn_time': False} if e.timeout > settings['outgoing']['request_timeout']: stats[e.name]['warn_timeout'] = True + stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences) + + engines_by_category[c].append(e) # get first element [0], the engine time, # and then the second element [1] : the time (the first one is the label) - for engine_stat in get_engines_stats()[0][1]: + for engine_stat in get_engines_stats(request.preferences)[0][1]: stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3) if engine_stat.get('avg') > settings['outgoing']['request_timeout']: stats[engine_stat.get('name')]['warn_time'] = True @@ -680,9 +827,9 @@ def preferences(): return render('preferences.html', locales=settings['locales'], - current_locale=get_locale(), + current_locale=request.preferences.get_value("locale"), image_proxy=image_proxy, - engines_by_category=categories, + engines_by_category=engines_by_category, stats=stats, answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers], disabled_engines=disabled_engines, @@ -690,6 +837,8 @@ def preferences(): shortcuts={y: x for x, y in engine_shortcuts.items()}, themes=themes, plugins=plugins, + doi_resolvers=settings['doi_resolvers'], + current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')), allowed_plugins=allowed_plugins, theme=get_current_theme_name(), preferences_url_params=request.preferences.get_as_url_params(), @@ -697,6 +846,14 @@ def preferences(): preferences=True) +def _is_selected_language_supported(engine, preferences): + language = preferences.get_value('language') + return (language == 'all' + or match_language(language, + getattr(engine, 'supported_languages', []), + getattr(engine, 'language_aliases', {}), None)) + + @app.route('/image_proxy', methods=['GET']) def image_proxy(): url = request.args.get('url').encode('utf-8') @@ -704,7 +861,7 @@ def image_proxy(): if not url: return '', 400 - h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest() + h = new_hmac(settings['server']['secret_key'], url) if h != request.args.get('h'): return '', 400 @@ -731,7 +888,7 @@ def image_proxy(): logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type'))) return '', 400 - img = '' + img = b'' chunk_counter = 0 for chunk in resp.iter_content(1024 * 1024): @@ -748,7 +905,7 @@ def image_proxy(): @app.route('/stats', methods=['GET']) def stats(): """Render engine statistics page.""" - stats = get_engines_stats() + stats = get_engines_stats(request.preferences) return render( 'stats.html', stats=stats, @@ -792,7 +949,8 @@ def opensearch(): @app.route('/favicon.ico') def favicon(): return send_from_directory(os.path.join(app.root_path, - 'static/themes', + static_path, + 'themes', get_current_theme_name(), 'img'), 'favicon.png', @@ -809,31 +967,51 @@ def clear_cookies(): @app.route('/config') def config(): - return jsonify({'categories': categories.keys(), - 'engines': [{'name': engine_name, - 'categories': engine.categories, - 'shortcut': engine.shortcut, - 'enabled': not engine.disabled, - 'paging': engine.paging, - 'language_support': engine.language_support, - 'supported_languages': - engine.supported_languages.keys() - if isinstance(engine.supported_languages, dict) - else engine.supported_languages, - 'safesearch': engine.safesearch, - 'time_range_support': engine.time_range_support, - 'timeout': engine.timeout} - for engine_name, engine in engines.items()], - 'plugins': [{'name': plugin.name, - 'enabled': plugin.default_on} - for plugin in plugins], - 'instance_name': settings['general']['instance_name'], - 'locales': settings['locales'], - 'default_locale': settings['ui']['default_locale'], - 'autocomplete': settings['search']['autocomplete'], - 'safe_search': settings['search']['safe_search'], - 'default_theme': settings['ui']['default_theme'], - 'version': VERSION_STRING}) + """Return configuration in JSON format.""" + _engines = [] + for name, engine in engines.items(): + if not request.preferences.validate_token(engine): + continue + + supported_languages = engine.supported_languages + if isinstance(engine.supported_languages, dict): + supported_languages = list(engine.supported_languages.keys()) + + _engines.append({ + 'name': name, + 'categories': engine.categories, + 'shortcut': engine.shortcut, + 'enabled': not engine.disabled, + 'paging': engine.paging, + 'language_support': engine.language_support, + 'supported_languages': supported_languages, + 'safesearch': engine.safesearch, + 'time_range_support': engine.time_range_support, + 'timeout': engine.timeout + }) + + _plugins = [] + for _ in plugins: + _plugins.append({'name': _.name, 'enabled': _.default_on}) + + return jsonify({ + 'categories': list(categories.keys()), + 'engines': _engines, + 'plugins': _plugins, + 'instance_name': settings['general']['instance_name'], + 'locales': settings['locales'], + 'default_locale': settings['ui']['default_locale'], + 'autocomplete': settings['search']['autocomplete'], + 'safe_search': settings['search']['safe_search'], + 'default_theme': settings['ui']['default_theme'], + 'version': VERSION_STRING, + 'brand': { + 'GIT_URL': brand.GIT_URL, + 'DOCS_URL': brand.DOCS_URL + }, + 'doi_resolvers': [r for r in settings['doi_resolvers']], + 'default_doi_resolver': settings['default_doi_resolver'], + }) @app.errorhandler(404) @@ -842,7 +1020,7 @@ def page_not_found(e): def run(): - logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address']) + logger.debug('starting webserver on %s:%s', settings['server']['bind_address'], settings['server']['port']) app.run( debug=searx_debug, use_debugger=searx_debug, |