summaryrefslogtreecommitdiff
path: root/searxng_extra/update/update_engine_descriptions.py
diff options
context:
space:
mode:
Diffstat (limited to 'searxng_extra/update/update_engine_descriptions.py')
-rwxr-xr-xsearxng_extra/update/update_engine_descriptions.py29
1 files changed, 9 insertions, 20 deletions
diff --git a/searxng_extra/update/update_engine_descriptions.py b/searxng_extra/update/update_engine_descriptions.py
index 2e16e1029..51cfc7cc2 100755
--- a/searxng_extra/update/update_engine_descriptions.py
+++ b/searxng_extra/update/update_engine_descriptions.py
@@ -56,7 +56,8 @@ NOT_A_DESCRIPTION = [
SKIP_ENGINE_SOURCE = [
# fmt: off
- ('gitlab', 'wikidata') # descriptions are about wikipedia disambiguation pages
+ ('gitlab', 'wikidata')
+ # descriptions are about wikipedia disambiguation pages
# fmt: on
]
@@ -94,10 +95,7 @@ def update_description(engine_name, lang, description, source, replace=True):
def get_wikipedia_summary(lang, pageid):
- params = {
- 'language': lang.replace('_','-'),
- 'headers': {}
- }
+ params = {'language': lang.replace('_', '-'), 'headers': {}}
searx.engines.engines['wikipedia'].request(pageid, params)
try:
response = searx.network.get(params['url'], headers=params['headers'], timeout=10)
@@ -162,10 +160,7 @@ def initialize():
global IDS, WIKIPEDIA_LANGUAGES, LANGUAGES_SPARQL
searx.search.initialize()
wikipedia_engine = searx.engines.engines['wikipedia']
- WIKIPEDIA_LANGUAGES = {
- language: wikipedia_engine.url_lang(language.replace('_', '-'))
- for language in LANGUAGES
- }
+ WIKIPEDIA_LANGUAGES = {language: wikipedia_engine.url_lang(language.replace('_', '-')) for language in LANGUAGES}
WIKIPEDIA_LANGUAGES['nb_NO'] = 'no'
LANGUAGES_SPARQL = ', '.join(f"'{l}'" for l in set(WIKIPEDIA_LANGUAGES.values()))
for engine_name, engine in searx.engines.engines.items():
@@ -180,9 +175,7 @@ def initialize():
def fetch_wikidata_descriptions():
searx.network.set_timeout_for_thread(60)
result = wikidata.send_wikidata_query(
- SPARQL_DESCRIPTION
- .replace('%IDS%', IDS)
- .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL)
+ SPARQL_DESCRIPTION.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL)
)
if result is not None:
for binding in result['results']['bindings']:
@@ -197,9 +190,7 @@ def fetch_wikidata_descriptions():
def fetch_wikipedia_descriptions():
result = wikidata.send_wikidata_query(
- SPARQL_WIKIPEDIA_ARTICLE
- .replace('%IDS%', IDS)
- .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL)
+ SPARQL_WIKIPEDIA_ARTICLE.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL)
)
if result is not None:
for binding in result['results']['bindings']:
@@ -226,9 +217,9 @@ def fetch_website_description(engine_name, website):
# the front page can't be fetched: skip this engine
return
- wikipedia_languages_r = { V: K for K, V in WIKIPEDIA_LANGUAGES.items() }
+ wikipedia_languages_r = {V: K for K, V in WIKIPEDIA_LANGUAGES.items()}
languages = ['en', 'es', 'pt', 'ru', 'tr', 'fr']
- languages = languages + [ l for l in LANGUAGES if l not in languages]
+ languages = languages + [l for l in LANGUAGES if l not in languages]
previous_matched_lang = None
previous_count = 0
@@ -281,9 +272,7 @@ def get_output():
* description (if source = "wikipedia")
* [f"engine:lang", "ref"] (reference to another existing description)
"""
- output = {
- locale: {} for locale in LOCALE_NAMES
- }
+ output = {locale: {} for locale in LOCALE_NAMES}
seen_descriptions = {}