diff options
Diffstat (limited to 'searxng_extra/update/update_engine_descriptions.py')
| -rwxr-xr-x | searxng_extra/update/update_engine_descriptions.py | 29 |
1 files changed, 9 insertions, 20 deletions
diff --git a/searxng_extra/update/update_engine_descriptions.py b/searxng_extra/update/update_engine_descriptions.py index 2e16e1029..51cfc7cc2 100755 --- a/searxng_extra/update/update_engine_descriptions.py +++ b/searxng_extra/update/update_engine_descriptions.py @@ -56,7 +56,8 @@ NOT_A_DESCRIPTION = [ SKIP_ENGINE_SOURCE = [ # fmt: off - ('gitlab', 'wikidata') # descriptions are about wikipedia disambiguation pages + ('gitlab', 'wikidata') + # descriptions are about wikipedia disambiguation pages # fmt: on ] @@ -94,10 +95,7 @@ def update_description(engine_name, lang, description, source, replace=True): def get_wikipedia_summary(lang, pageid): - params = { - 'language': lang.replace('_','-'), - 'headers': {} - } + params = {'language': lang.replace('_', '-'), 'headers': {}} searx.engines.engines['wikipedia'].request(pageid, params) try: response = searx.network.get(params['url'], headers=params['headers'], timeout=10) @@ -162,10 +160,7 @@ def initialize(): global IDS, WIKIPEDIA_LANGUAGES, LANGUAGES_SPARQL searx.search.initialize() wikipedia_engine = searx.engines.engines['wikipedia'] - WIKIPEDIA_LANGUAGES = { - language: wikipedia_engine.url_lang(language.replace('_', '-')) - for language in LANGUAGES - } + WIKIPEDIA_LANGUAGES = {language: wikipedia_engine.url_lang(language.replace('_', '-')) for language in LANGUAGES} WIKIPEDIA_LANGUAGES['nb_NO'] = 'no' LANGUAGES_SPARQL = ', '.join(f"'{l}'" for l in set(WIKIPEDIA_LANGUAGES.values())) for engine_name, engine in searx.engines.engines.items(): @@ -180,9 +175,7 @@ def initialize(): def fetch_wikidata_descriptions(): searx.network.set_timeout_for_thread(60) result = wikidata.send_wikidata_query( - SPARQL_DESCRIPTION - .replace('%IDS%', IDS) - .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL) + SPARQL_DESCRIPTION.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL) ) if result is not None: for binding in result['results']['bindings']: @@ -197,9 +190,7 @@ def fetch_wikidata_descriptions(): def fetch_wikipedia_descriptions(): result = wikidata.send_wikidata_query( - SPARQL_WIKIPEDIA_ARTICLE - .replace('%IDS%', IDS) - .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL) + SPARQL_WIKIPEDIA_ARTICLE.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL) ) if result is not None: for binding in result['results']['bindings']: @@ -226,9 +217,9 @@ def fetch_website_description(engine_name, website): # the front page can't be fetched: skip this engine return - wikipedia_languages_r = { V: K for K, V in WIKIPEDIA_LANGUAGES.items() } + wikipedia_languages_r = {V: K for K, V in WIKIPEDIA_LANGUAGES.items()} languages = ['en', 'es', 'pt', 'ru', 'tr', 'fr'] - languages = languages + [ l for l in LANGUAGES if l not in languages] + languages = languages + [l for l in LANGUAGES if l not in languages] previous_matched_lang = None previous_count = 0 @@ -281,9 +272,7 @@ def get_output(): * description (if source = "wikipedia") * [f"engine:lang", "ref"] (reference to another existing description) """ - output = { - locale: {} for locale in LOCALE_NAMES - } + output = {locale: {} for locale in LOCALE_NAMES} seen_descriptions = {} |