diff options
Diffstat (limited to 'searx')
| -rw-r--r-- | searx/engines/__init__.py | 18 | ||||
| -rw-r--r-- | searx/engines/openstreetmap.py | 5 | ||||
| -rw-r--r-- | searx/network/__init__.py | 15 | ||||
| -rw-r--r-- | searx/plugins/__init__.py | 2 | ||||
| -rw-r--r-- | searx/plugins/hostname_replace.py | 32 | ||||
| -rw-r--r-- | searx/settings.yml | 14 | ||||
| -rw-r--r-- | searx/templates/oscar/base.html | 1 | ||||
| -rw-r--r-- | searx/templates/oscar/stats.html | 2 | ||||
| -rw-r--r-- | searx/templates/simple/base.html | 1 | ||||
| -rw-r--r-- | searx/templates/simple/stats.html | 2 |
10 files changed, 79 insertions, 13 deletions
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 154cd605f..e0b5796e4 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -110,10 +110,26 @@ def load_engine(engine_data): if is_missing_required_attributes(engine): return None - engine.logger = logger.getChild(engine_name) + set_loggers(engine, engine_name) + return engine +def set_loggers(engine, engine_name): + # set the logger for engine + engine.logger = logger.getChild(engine_name) + # the engine may have load some other engines + # may sure the logger is initialized + for module_name, module in sys.modules.items(): + if ( + module_name.startswith("searx.engines") + and module_name != "searx.engines.__init__" + and not hasattr(module, "logger") + ): + module_engine_name = module_name.split(".")[-1] + module.logger = logger.getChild(module_engine_name) + + def update_engine_attributes(engine, engine_data): # set engine attributes from engine_data for param_name, param_value in engine_data.items(): diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py index 78c15320a..9f1fe94c9 100644 --- a/searx/engines/openstreetmap.py +++ b/searx/engines/openstreetmap.py @@ -438,8 +438,3 @@ def get_key_label(key_name, lang): if labels is None: return None return get_label(labels, lang) - - -def init(_): - import searx.engines.wikidata # pylint: disable=import-outside-toplevel - searx.engines.wikidata.logger = logger diff --git a/searx/network/__init__.py b/searx/network/__init__.py index 7b0396a12..21c4c27b5 100644 --- a/searx/network/__init__.py +++ b/searx/network/__init__.py @@ -173,9 +173,17 @@ async def stream_chunk_to_queue(network, queue, method, url, **kwargs): if len(chunk) > 0: queue.put(chunk) except httpx.ResponseClosed: - # the response was closed + # the response was queued before the exception. + # the exception was raised on aiter_raw. + # we do nothing here: in the finally block, None will be queued + # so stream(method, url, **kwargs) generator can stop pass - except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e: + except Exception as e: # pylint: disable=broad-except + # broad except to avoid this scenario: + # exception in network.stream(method, url, **kwargs) + # -> the exception is not catch here + # -> queue None (in finally) + # -> the function below steam(method, url, **kwargs) has nothing to return queue.put(e) finally: queue.put(None) @@ -201,8 +209,9 @@ def stream(method, url, **kwargs): the httpx.AsyncHTTPTransport declared above. """ queue = SimpleQueue() + network = get_context_network() future = asyncio.run_coroutine_threadsafe( - stream_chunk_to_queue(get_network(), queue, method, url, **kwargs), + stream_chunk_to_queue(network, queue, method, url, **kwargs), get_loop() ) diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 3a35f7025..1153c9ed1 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -31,6 +31,7 @@ from searx.plugins import (oa_doi_rewrite, hash_plugin, infinite_scroll, self_info, + hostname_replace, search_on_category_select, tracker_url_remover, vim_hotkeys) @@ -182,6 +183,7 @@ plugins.register(oa_doi_rewrite) plugins.register(hash_plugin) plugins.register(infinite_scroll) plugins.register(self_info) +plugins.register(hostname_replace) plugins.register(search_on_category_select) plugins.register(tracker_url_remover) plugins.register(vim_hotkeys) diff --git a/searx/plugins/hostname_replace.py b/searx/plugins/hostname_replace.py new file mode 100644 index 000000000..778b84615 --- /dev/null +++ b/searx/plugins/hostname_replace.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import re +from urllib.parse import urlunparse +from searx import settings +from searx.plugins import logger +from flask_babel import gettext + +name = gettext('Hostname replace') +description = gettext('Rewrite result hostnames or remove results based on the hostname') +default_on = False +preference_section = 'general' + +plugin_id = 'hostname_replace' + +replacements = {re.compile(p): r for (p, r) in settings[plugin_id].items()} if plugin_id in settings else {} + +logger = logger.getChild(plugin_id) +parsed = 'parsed_url' + + +def on_result(request, search, result): + if parsed not in result: + return True + for (pattern, replacement) in replacements.items(): + if pattern.search(result[parsed].netloc): + if not replacement: + return False + result[parsed] = result[parsed]._replace(netloc=pattern.sub(replacement, result[parsed].netloc)) + result['url'] = urlunparse(result[parsed]) + + return True diff --git a/searx/settings.yml b/searx/settings.yml index e5eb9dd65..55d8603a5 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1,6 +1,6 @@ general: debug: false # Debug mode, only for development - instance_name: "searxng" # displayed name + instance_name: "SearXNG" # displayed name contact_url: false # mailto:contact@example.com brand: @@ -150,7 +150,17 @@ outgoing: # # enabled_plugins: # - "HTTPS rewrite" -# - ... +# - "Hostname replace" # see configuration below + +# "Hostname replace" plugin configuration example: +# hostname_replace: +# '(.*\.)?youtube\.com$': 'invidious.example.com' +# '(.*\.)?youtu\.be$': 'invidious.example.com' +# '(.*\.)?youtube-noocookie\.com$': 'yotter.example.com' +# '(.*\.)?reddit\.com$': 'teddit.example.com' +# '(.*\.)?redd\.it$': 'teddit.example.com' +# '(www\.)?twitter\.com$': 'nitter.example.com' +# 'spam\.example\.com': false # remove results from spam.example.com checker: # disable checker when in debug mode diff --git a/searx/templates/oscar/base.html b/searx/templates/oscar/base.html index 7cd38a25c..df06763fa 100644 --- a/searx/templates/oscar/base.html +++ b/searx/templates/oscar/base.html @@ -85,6 +85,7 @@ {{ _('Powered by') }} <a href="{{ get_setting('brand.docs_url') }}">SearXNG</a> - {{ searx_version }} - {{ _('a privacy-respecting, hackable metasearch engine') }}<br/> <a href="{{ searx_git_url }}">{{ _('Source code') }}</a> | <a href="{{ get_setting('brand.issue_url') }}">{{ _('Issue tracker') }}</a> | + <a href="{{ url_for('stats') }}">{{ _('Engine stats') }}</a> | <a href="{{ get_setting('brand.public_instances') }}">{{ _('Public instances') }}</a>{% if get_setting('general.contact_url') %} | <a href="{{ get_setting('general.contact_url') }}">{{ _('Contact instance maintainer') }}</a>{% endif %} </small> diff --git a/searx/templates/oscar/stats.html b/searx/templates/oscar/stats.html index 4be8043ff..646cb9923 100644 --- a/searx/templates/oscar/stats.html +++ b/searx/templates/oscar/stats.html @@ -15,7 +15,7 @@ {% block content %} <div class="container-fluid"> - <h1>{{ _('Engine stats') }}{% if selected_engine_name %} - {{ selected_engine_name }}{% endif %}</h1> + <h1>{% if selected_engine_name %}<a href="{{ url_for('stats') }}">{% endif %}{{ _('Engine stats') }}{% if selected_engine_name %}</a> - {{ selected_engine_name }}{% endif %}</h1> <div class="row"> <div class="col-xs-12 col-sm-12 col-md-12"> <div class="table-responsive"> diff --git a/searx/templates/simple/base.html b/searx/templates/simple/base.html index 7020de756..27c4b5a4c 100644 --- a/searx/templates/simple/base.html +++ b/searx/templates/simple/base.html @@ -53,6 +53,7 @@ {{ _('Powered by') }} <a href="{{ url_for('about') }}">searxng</a> - {{ searx_version }} — {{ _('a privacy-respecting, hackable metasearch engine') }}<br/> <a href="{{ searx_git_url }}">{{ _('Source code') }}</a> | <a href="{{ get_setting('brand.issue_url') }}">{{ _('Issue tracker') }}</a> | + <a href="{{ url_for('stats') }}">{{ _('Engine stats') }}</a> | <a href="{{ get_setting('brand.public_instances') }}">{{ _('Public instances') }}</a>{% if get_setting('general.contact_url') %} | <a href="{{ get_setting('general.contact_url') }}">{{ _('Contact instance maintainer') }}</a>{% endif %} </p> diff --git a/searx/templates/simple/stats.html b/searx/templates/simple/stats.html index f423b6861..889b5d05a 100644 --- a/searx/templates/simple/stats.html +++ b/searx/templates/simple/stats.html @@ -18,7 +18,7 @@ <a href="{{ url_for('index') }}"><h1><span>searx</span></h1></a> -<h2>{{ _('Engine stats') }}{% if selected_engine_name %} - {{ selected_engine_name }}{% endif %}</h2> +<h2>{% if selected_engine_name %}<a href="{{ url_for('stats') }}">{% endif %}{{ _('Engine stats') }}{% if selected_engine_name %}</a> - {{ selected_engine_name }}{% endif %}</h2> {% if not engine_stats.get('time') %} {{ _('There is currently no data available. ') }} |