From 149802c56926bf48520c98932c4c36b8152b3d2d Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 5 Aug 2016 23:34:56 -0500 Subject: [enh] add supported_languages on engines and auto-generate languages.py --- utils/update_languages.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 utils/update_languages.py (limited to 'utils') diff --git a/utils/update_languages.py b/utils/update_languages.py new file mode 100644 index 000000000..5c340b01e --- /dev/null +++ b/utils/update_languages.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +# This script generates languages.py from +# intersecting each engine's supported languages. +# +# The language's native names are obtained from +# Wikipedia's supported languages. +# +# Output file (languages.py) is written in current directory +# to avoid overwriting in case something goes wrong. + +from requests import get +from re import sub +from lxml.html import fromstring +from json import loads +from sys import path +path.append('../searx') +from searx.engines import engines + +# list of language names +wiki_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias' +google_languages_url = 'https://www.google.com/preferences?#languages' + +google_json_name = 'google.preferences.langMap' + +languages = {} + +# Get language names from Wikipedia. +def get_wikipedia_languages(): + response = get(wiki_languages_url) + dom = fromstring(response.text) + tables = dom.xpath('//table[contains(@class,"sortable")]') + for table in tables: + # exclude header row + trs = table.xpath('.//tr')[1:] + for tr in trs: + td = tr.xpath('./td') + code = td[3].xpath('./a')[0].text + name = td[2].xpath('./a')[0].text + english_name = td[1].xpath('./a')[0].text + + if code not in languages: + languages[code] = (name, '', english_name) + +# Get language names from Google. +def get_google_languages(): + response = get(google_languages_url) + dom = fromstring(response.text) + options = dom.xpath('//select[@name="hl"]/option') + for option in options: + code = option.xpath('./@value')[0] + name = option.text[:-1] + + if code not in languages: + languages[code] = (name, '', '') + +# Join all language lists. +# iterate all languages supported by each engine +def join_language_lists(): + for engine_name in engines: + for locale in engines[engine_name].supported_languages: + locale = locale.replace('_', '-') + if locale not in languages: + # try to get language name + language = languages.get(locale.split('-')[0], None) + if language == None: + print engine_name + ": " + locale + continue + + (name, country, english) = language + languages[locale] = (name, country, english) + +# Write languages.py. +def write_languages_file(): + new_file = open('languages.py', 'w') + file_content = '# -*- coding: utf-8 -*-\n' + file_content += '# list of language codes\n' + file_content += '# this file is generated automatically by utils/update_search_languages.py\n' + file_content += '\nlanguage_codes = (' + for code in languages: + (name, country, english) = languages[code] + file_content += '\n (u"' + code + '"'\ + + ', u"' + name + '"'\ + + ', u"' + country[1:-1] + '"'\ + + ', u"' + english + '"),' + # remove last comma + file_content = file_content[:-1] + file_content += '\n)\n' + new_file.write(file_content.encode('utf8')) + new_file.close() + +def main(): + get_wikipedia_languages() + get_google_languages() + join_language_lists() + write_languages_file() + +if __name__ == "__main__": + main() -- cgit v1.2.3 From c677aee58a4eca1015262eb24530620a333ddcef Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 6 Aug 2016 22:19:21 -0500 Subject: filter langauges --- utils/update_languages.py | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) (limited to 'utils') diff --git a/utils/update_languages.py b/utils/update_languages.py index 5c340b01e..cb230c210 100644 --- a/utils/update_languages.py +++ b/utils/update_languages.py @@ -11,7 +11,7 @@ from requests import get from re import sub -from lxml.html import fromstring +from lxml.html import fromstring, tostring from json import loads from sys import path path.append('../searx') @@ -25,6 +25,28 @@ google_json_name = 'google.preferences.langMap' languages = {} + +# To filter out invalid codes and dialects. +def valid_code(lang_code): + # filter invalid codes + if lang_code[:2] == 'xx'\ + or lang_code == 'jw'\ + or lang_code[-2:] == 'UK'\ + or lang_code[-2:] == 'XA'\ + or lang_code[-2:] == 'XL': + return False + + # filter dialects + lang_code = lang_code.split('-') + if len(lang_code) > 2 or len(lang_code[0]) > 3: + return False + if len(lang_code) == 2 and len(lang_code[1]) > 2: + print lang_code + return False + + return True + + # Get language names from Wikipedia. def get_wikipedia_languages(): response = get(wiki_languages_url) @@ -38,10 +60,13 @@ def get_wikipedia_languages(): code = td[3].xpath('./a')[0].text name = td[2].xpath('./a')[0].text english_name = td[1].xpath('./a')[0].text + articles = int(td[4].xpath('./a/b')[0].text.replace(',','')) - if code not in languages: + # exclude languages with few articles and language variants + if code not in languages and articles >= 100 and valid_code(code): languages[code] = (name, '', english_name) + # Get language names from Google. def get_google_languages(): response = get(google_languages_url) @@ -51,25 +76,27 @@ def get_google_languages(): code = option.xpath('./@value')[0] name = option.text[:-1] - if code not in languages: + if code not in languages and valid_code(code): languages[code] = (name, '', '') + # Join all language lists. # iterate all languages supported by each engine def join_language_lists(): for engine_name in engines: for locale in engines[engine_name].supported_languages: locale = locale.replace('_', '-') - if locale not in languages: + if locale not in languages and valid_code(locale): # try to get language name language = languages.get(locale.split('-')[0], None) if language == None: - print engine_name + ": " + locale + # print engine_name + ": " + locale continue (name, country, english) = language languages[locale] = (name, country, english) + # Write languages.py. def write_languages_file(): new_file = open('languages.py', 'w') @@ -81,7 +108,7 @@ def write_languages_file(): (name, country, english) = languages[code] file_content += '\n (u"' + code + '"'\ + ', u"' + name + '"'\ - + ', u"' + country[1:-1] + '"'\ + + ', u"' + country + '"'\ + ', u"' + english + '"),' # remove last comma file_content = file_content[:-1] @@ -89,11 +116,13 @@ def write_languages_file(): new_file.write(file_content.encode('utf8')) new_file.close() + def main(): get_wikipedia_languages() get_google_languages() join_language_lists() write_languages_file() + if __name__ == "__main__": main() -- cgit v1.2.3 From a11948c71bfe7b2aac6e50e7634874d5073c7d84 Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 29 Oct 2016 21:04:01 -0500 Subject: Add language support for more engines. --- utils/update_languages.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'utils') diff --git a/utils/update_languages.py b/utils/update_languages.py index cb230c210..6f86742c6 100644 --- a/utils/update_languages.py +++ b/utils/update_languages.py @@ -41,7 +41,6 @@ def valid_code(lang_code): if len(lang_code) > 2 or len(lang_code[0]) > 3: return False if len(lang_code) == 2 and len(lang_code[1]) > 2: - print lang_code return False return True @@ -62,8 +61,8 @@ def get_wikipedia_languages(): english_name = td[1].xpath('./a')[0].text articles = int(td[4].xpath('./a/b')[0].text.replace(',','')) - # exclude languages with few articles and language variants - if code not in languages and articles >= 100 and valid_code(code): + # exclude language variants and languages with few articles + if code not in languages and articles >= 1000 and valid_code(code): languages[code] = (name, '', english_name) @@ -90,7 +89,7 @@ def join_language_lists(): # try to get language name language = languages.get(locale.split('-')[0], None) if language == None: - # print engine_name + ": " + locale + print engine_name + ": " + locale continue (name, country, english) = language @@ -117,12 +116,8 @@ def write_languages_file(): new_file.close() -def main(): +if __name__ == "__main__": get_wikipedia_languages() get_google_languages() join_language_lists() write_languages_file() - - -if __name__ == "__main__": - main() -- cgit v1.2.3 From 922d51bf0c992cee71a69496efbfafee5c81afcf Mon Sep 17 00:00:00 2001 From: marc Date: Sun, 30 Oct 2016 19:12:38 -0600 Subject: Add country names in language list. --- utils/update_languages.py | 64 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 10 deletions(-) (limited to 'utils') diff --git a/utils/update_languages.py b/utils/update_languages.py index 6f86742c6..193c19278 100644 --- a/utils/update_languages.py +++ b/utils/update_languages.py @@ -4,22 +4,28 @@ # intersecting each engine's supported languages. # # The language's native names are obtained from -# Wikipedia's supported languages. +# Wikipedia and Google's supported languages. +# +# The country names are obtained from http://api.geonames.org +# which requires registering as a user. # # Output file (languages.py) is written in current directory # to avoid overwriting in case something goes wrong. from requests import get -from re import sub -from lxml.html import fromstring, tostring +from urllib import urlencode +from lxml.html import fromstring from json import loads from sys import path path.append('../searx') from searx.engines import engines -# list of language names +# list of names wiki_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias' google_languages_url = 'https://www.google.com/preferences?#languages' +country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}' + +geonames_user = '' # add user name here google_json_name = 'google.preferences.langMap' @@ -46,6 +52,29 @@ def valid_code(lang_code): return True +# Get country name in specified language. +def get_country_name(locale): + if geonames_user is '': + return '' + + locale = locale.split('-') + if len(locale) != 2: + return '' + + url = country_names_url.format(parameters=urlencode({'lang': locale[0], + 'country': locale[1], + 'username': geonames_user})) + response = get(url) + json = loads(response.text) + content = json.get('geonames', None) + if content is None or len(content) != 1: + print "No country name found for " + locale[0] + "-" + locale[1] + print json + return '' + + return content[0].get('countryName', '') + + # Get language names from Wikipedia. def get_wikipedia_languages(): response = get(wiki_languages_url) @@ -62,7 +91,7 @@ def get_wikipedia_languages(): articles = int(td[4].xpath('./a/b')[0].text.replace(',','')) # exclude language variants and languages with few articles - if code not in languages and articles >= 1000 and valid_code(code): + if code not in languages and articles >= 10000 and valid_code(code): languages[code] = (name, '', english_name) @@ -72,8 +101,8 @@ def get_google_languages(): dom = fromstring(response.text) options = dom.xpath('//select[@name="hl"]/option') for option in options: - code = option.xpath('./@value')[0] - name = option.text[:-1] + code = option.xpath('./@value')[0].split('-')[0] + name = option.text[:-1].title() if code not in languages and valid_code(code): languages[code] = (name, '', '') @@ -92,8 +121,22 @@ def join_language_lists(): print engine_name + ": " + locale continue - (name, country, english) = language - languages[locale] = (name, country, english) + country = get_country_name(locale) + languages[locale] = (language[0], country, language[2]) + + +# Remove countryless language if language is featured in only one country. +def filter_single_country_languages(): + prev_lang = None + for code in sorted(languages): + lang = code.split('-')[0] + if lang == prev_lang: + countries += 1 + else: + if prev_lang is not None and countries == 1: + del languages[prev_lang] + countries = 0 + prev_lang = lang # Write languages.py. @@ -103,7 +146,7 @@ def write_languages_file(): file_content += '# list of language codes\n' file_content += '# this file is generated automatically by utils/update_search_languages.py\n' file_content += '\nlanguage_codes = (' - for code in languages: + for code in sorted(languages): (name, country, english) = languages[code] file_content += '\n (u"' + code + '"'\ + ', u"' + name + '"'\ @@ -120,4 +163,5 @@ if __name__ == "__main__": get_wikipedia_languages() get_google_languages() join_language_lists() + filter_single_country_languages() write_languages_file() -- cgit v1.2.3 From 92c6e88ad3e5ba57bd6e2ba64d0c38e8fd72ea09 Mon Sep 17 00:00:00 2001 From: marc Date: Mon, 31 Oct 2016 23:52:08 -0600 Subject: small fixes --- utils/update_languages.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'utils') diff --git a/utils/update_languages.py b/utils/update_languages.py index 193c19278..cc3fa29cc 100644 --- a/utils/update_languages.py +++ b/utils/update_languages.py @@ -35,7 +35,9 @@ languages = {} # To filter out invalid codes and dialects. def valid_code(lang_code): # filter invalid codes + # sl-SL is technically not invalid, but still a mistake if lang_code[:2] == 'xx'\ + or lang_code == 'sl-SL'\ or lang_code == 'jw'\ or lang_code[-2:] == 'UK'\ or lang_code[-2:] == 'XA'\ -- cgit v1.2.3 From f62ce21f50b540315a708ebfbf36878ddec9d1c4 Mon Sep 17 00:00:00 2001 From: marc Date: Sat, 5 Nov 2016 20:51:38 -0600 Subject: [mod] fetch supported languages for several engines utils/fetch_languages.py gets languages supported by each engine and generates engines_languages.json with each engine's supported language. --- utils/fetch_languages.py | 164 ++++++++++++++++++++++++++++++++++++++++++++ utils/update_languages.py | 169 ---------------------------------------------- 2 files changed, 164 insertions(+), 169 deletions(-) create mode 100644 utils/fetch_languages.py delete mode 100644 utils/update_languages.py (limited to 'utils') diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py new file mode 100644 index 000000000..ae4a2def9 --- /dev/null +++ b/utils/fetch_languages.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- + +# This script generates languages.py from intersecting each engine's supported languages. +# +# The country names are obtained from http://api.geonames.org which requires registering as a user. +# +# Output files (engines_languages.json and languages.py) +# are written in current directory to avoid overwriting in case something goes wrong. + +from requests import get +from urllib import urlencode +from lxml.html import fromstring +from json import loads, dumps +import io +from sys import path +path.append('../searx') # noqa +from searx.engines import engines + +# Geonames API for country names. +geonames_user = '' # ADD USER NAME HERE +country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}' + +# Output files. +engines_languages_file = 'engines_languages.json' +languages_file = 'languages.py' + +engines_languages = {} +languages = {} + + +# To filter out invalid codes and dialects. +def valid_code(lang_code): + # filter invalid codes + # sl-SL is technically not invalid, but still a mistake + if lang_code[:2] == 'xx'\ + or lang_code == 'sl-SL'\ + or lang_code == 'wt-WT'\ + or lang_code == 'jw'\ + or lang_code[-2:] == 'UK'\ + or lang_code[-2:] == 'XA'\ + or lang_code[-2:] == 'XL': + return False + + # filter dialects + lang_code = lang_code.split('-') + if len(lang_code) > 2 or len(lang_code[0]) > 3: + return False + if len(lang_code) == 2 and len(lang_code[1]) > 2: + return False + + return True + + +# Get country name in specified language. +def get_country_name(locale): + if geonames_user is '': + return '' + + locale = locale.split('-') + if len(locale) != 2: + return '' + + url = country_names_url.format(parameters=urlencode({'lang': locale[0], + 'country': locale[1], + 'username': geonames_user})) + response = get(url) + json = loads(response.text) + content = json.get('geonames', None) + if content is None or len(content) != 1: + print "No country name found for " + locale[0] + "-" + locale[1] + return '' + + return content[0].get('countryName', '') + + +# Fetchs supported languages for each engine and writes json file with those. +def fetch_supported_languages(): + for engine_name in engines: + if hasattr(engines[engine_name], 'fetch_supported_languages'): + try: + engines_languages[engine_name] = engines[engine_name].fetch_supported_languages() + except Exception as e: + print e + + # write json file + f = io.open(engines_languages_file, "w", encoding="utf-8") + f.write(unicode(dumps(engines_languages, indent=4, ensure_ascii=False, encoding="utf-8"))) + f.close() + + +# Join all language lists. +# Iterate all languages supported by each engine. +def join_language_lists(): + # include wikipedia first for more accurate language names + # exclude languages with too few articles + languages.update({code: lang for code, lang + in engines_languages['wikipedia'].iteritems() + if valid_code(code) and lang['articles'] >= 100000}) + + for engine_name in engines_languages: + for locale in engines_languages[engine_name]: + if not valid_code(locale): + continue + + # if language is not on list or if it has no name yet + if locale not in languages or not languages[locale].get('name'): + if isinstance(engines_languages[engine_name], dict) \ + and engines_languages[engine_name][locale].get('articles', float('inf')) >= 100000: + languages[locale] = engines_languages[engine_name][locale] + else: + languages[locale] = {} + + # get locales that have no name yet + for locale in languages.keys(): + if not languages[locale].get('name'): + # try to get language and country names + name = languages.get(locale.split('-')[0], {}).get('name', None) + if name: + languages[locale]['name'] = name + languages[locale]['country'] = get_country_name(locale) or '' + languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '') + else: + # filter out locales with no name + del languages[locale] + + +# Remove countryless language if language is featured in only one country. +def filter_single_country_languages(): + prev_lang = None + for code in sorted(languages): + lang = code.split('-')[0] + if lang == prev_lang: + countries += 1 + else: + if prev_lang is not None and countries == 1: + del languages[prev_lang] + countries = 0 + prev_lang = lang + + +# Write languages.py. +def write_languages_file(): + new_file = open(languages_file, 'w') + file_content = '# -*- coding: utf-8 -*-\n' + file_content += '# list of language codes\n' + file_content += '# this file is generated automatically by utils/update_search_languages.py\n' + file_content += '\nlanguage_codes = (' + for code in sorted(languages): + file_content += '\n (u"' + code + '"'\ + + ', u"' + languages[code]['name'].split(' (')[0] + '"'\ + + ', u"' + languages[code].get('country', '') + '"'\ + + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),' + # remove last comma + file_content = file_content[:-1] + file_content += '\n)\n' + new_file.write(file_content.encode('utf8')) + new_file.close() + + +if __name__ == "__main__": + fetch_supported_languages() + join_language_lists() + filter_single_country_languages() + write_languages_file() diff --git a/utils/update_languages.py b/utils/update_languages.py deleted file mode 100644 index cc3fa29cc..000000000 --- a/utils/update_languages.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- - -# This script generates languages.py from -# intersecting each engine's supported languages. -# -# The language's native names are obtained from -# Wikipedia and Google's supported languages. -# -# The country names are obtained from http://api.geonames.org -# which requires registering as a user. -# -# Output file (languages.py) is written in current directory -# to avoid overwriting in case something goes wrong. - -from requests import get -from urllib import urlencode -from lxml.html import fromstring -from json import loads -from sys import path -path.append('../searx') -from searx.engines import engines - -# list of names -wiki_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias' -google_languages_url = 'https://www.google.com/preferences?#languages' -country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}' - -geonames_user = '' # add user name here - -google_json_name = 'google.preferences.langMap' - -languages = {} - - -# To filter out invalid codes and dialects. -def valid_code(lang_code): - # filter invalid codes - # sl-SL is technically not invalid, but still a mistake - if lang_code[:2] == 'xx'\ - or lang_code == 'sl-SL'\ - or lang_code == 'jw'\ - or lang_code[-2:] == 'UK'\ - or lang_code[-2:] == 'XA'\ - or lang_code[-2:] == 'XL': - return False - - # filter dialects - lang_code = lang_code.split('-') - if len(lang_code) > 2 or len(lang_code[0]) > 3: - return False - if len(lang_code) == 2 and len(lang_code[1]) > 2: - return False - - return True - - -# Get country name in specified language. -def get_country_name(locale): - if geonames_user is '': - return '' - - locale = locale.split('-') - if len(locale) != 2: - return '' - - url = country_names_url.format(parameters=urlencode({'lang': locale[0], - 'country': locale[1], - 'username': geonames_user})) - response = get(url) - json = loads(response.text) - content = json.get('geonames', None) - if content is None or len(content) != 1: - print "No country name found for " + locale[0] + "-" + locale[1] - print json - return '' - - return content[0].get('countryName', '') - - -# Get language names from Wikipedia. -def get_wikipedia_languages(): - response = get(wiki_languages_url) - dom = fromstring(response.text) - tables = dom.xpath('//table[contains(@class,"sortable")]') - for table in tables: - # exclude header row - trs = table.xpath('.//tr')[1:] - for tr in trs: - td = tr.xpath('./td') - code = td[3].xpath('./a')[0].text - name = td[2].xpath('./a')[0].text - english_name = td[1].xpath('./a')[0].text - articles = int(td[4].xpath('./a/b')[0].text.replace(',','')) - - # exclude language variants and languages with few articles - if code not in languages and articles >= 10000 and valid_code(code): - languages[code] = (name, '', english_name) - - -# Get language names from Google. -def get_google_languages(): - response = get(google_languages_url) - dom = fromstring(response.text) - options = dom.xpath('//select[@name="hl"]/option') - for option in options: - code = option.xpath('./@value')[0].split('-')[0] - name = option.text[:-1].title() - - if code not in languages and valid_code(code): - languages[code] = (name, '', '') - - -# Join all language lists. -# iterate all languages supported by each engine -def join_language_lists(): - for engine_name in engines: - for locale in engines[engine_name].supported_languages: - locale = locale.replace('_', '-') - if locale not in languages and valid_code(locale): - # try to get language name - language = languages.get(locale.split('-')[0], None) - if language == None: - print engine_name + ": " + locale - continue - - country = get_country_name(locale) - languages[locale] = (language[0], country, language[2]) - - -# Remove countryless language if language is featured in only one country. -def filter_single_country_languages(): - prev_lang = None - for code in sorted(languages): - lang = code.split('-')[0] - if lang == prev_lang: - countries += 1 - else: - if prev_lang is not None and countries == 1: - del languages[prev_lang] - countries = 0 - prev_lang = lang - - -# Write languages.py. -def write_languages_file(): - new_file = open('languages.py', 'w') - file_content = '# -*- coding: utf-8 -*-\n' - file_content += '# list of language codes\n' - file_content += '# this file is generated automatically by utils/update_search_languages.py\n' - file_content += '\nlanguage_codes = (' - for code in sorted(languages): - (name, country, english) = languages[code] - file_content += '\n (u"' + code + '"'\ - + ', u"' + name + '"'\ - + ', u"' + country + '"'\ - + ', u"' + english + '"),' - # remove last comma - file_content = file_content[:-1] - file_content += '\n)\n' - new_file.write(file_content.encode('utf8')) - new_file.close() - - -if __name__ == "__main__": - get_wikipedia_languages() - get_google_languages() - join_language_lists() - filter_single_country_languages() - write_languages_file() -- cgit v1.2.3 From af35eee10b98940c51c6e5e18629de514b4bd48d Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 15 Dec 2016 00:34:43 -0600 Subject: tests for _fetch_supported_languages in engines and refactor method to make it testable without making requests --- utils/fetch_languages.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'utils') diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py index ae4a2def9..aed875daf 100644 --- a/utils/fetch_languages.py +++ b/utils/fetch_languages.py @@ -84,7 +84,7 @@ def fetch_supported_languages(): # write json file f = io.open(engines_languages_file, "w", encoding="utf-8") - f.write(unicode(dumps(engines_languages, indent=4, ensure_ascii=False, encoding="utf-8"))) + f.write(unicode(dumps(engines_languages, ensure_ascii=False, encoding="utf-8"))) f.close() @@ -110,18 +110,22 @@ def join_language_lists(): else: languages[locale] = {} - # get locales that have no name yet + # get locales that have no name or country yet for locale in languages.keys(): if not languages[locale].get('name'): - # try to get language and country names + # try to get language names name = languages.get(locale.split('-')[0], {}).get('name', None) if name: languages[locale]['name'] = name - languages[locale]['country'] = get_country_name(locale) or '' languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '') else: # filter out locales with no name del languages[locale] + continue + + # try to get country name + if locale.find('-') > 0 and not languages[locale].get('country'): + languages[locale]['country'] = get_country_name(locale) or '' # Remove countryless language if language is featured in only one country. -- cgit v1.2.3 From 4a1ff56389d6ad560594ba82b448aef1d70bbbf4 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 16 Dec 2016 22:14:14 -0600 Subject: minor fixes in utils/fetch_languages.py --- utils/fetch_languages.py | 49 +++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) (limited to 'utils') diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py index aed875daf..3510a3234 100644 --- a/utils/fetch_languages.py +++ b/utils/fetch_languages.py @@ -32,23 +32,26 @@ languages = {} def valid_code(lang_code): # filter invalid codes # sl-SL is technically not invalid, but still a mistake + invalid_codes = ['sl-SL', 'wt-WT', 'jw'] + invalid_countries = ['UK', 'XA', 'XL'] if lang_code[:2] == 'xx'\ - or lang_code == 'sl-SL'\ - or lang_code == 'wt-WT'\ - or lang_code == 'jw'\ - or lang_code[-2:] == 'UK'\ - or lang_code[-2:] == 'XA'\ - or lang_code[-2:] == 'XL': + or lang_code in invalid_codes\ + or lang_code[-2:] in invalid_countries\ + or is_dialect(lang_code): return False - # filter dialects + return True + + +# Language codes with any additional tags other than language and country. +def is_dialect(lang_code): lang_code = lang_code.split('-') if len(lang_code) > 2 or len(lang_code[0]) > 3: - return False + return True if len(lang_code) == 2 and len(lang_code[1]) > 2: - return False + return True - return True + return False # Get country name in specified language. @@ -83,19 +86,17 @@ def fetch_supported_languages(): print e # write json file - f = io.open(engines_languages_file, "w", encoding="utf-8") - f.write(unicode(dumps(engines_languages, ensure_ascii=False, encoding="utf-8"))) - f.close() + with io.open(engines_languages_file, "w", encoding="utf-8") as f: + f.write(unicode(dumps(engines_languages, ensure_ascii=False, encoding="utf-8"))) # Join all language lists. # Iterate all languages supported by each engine. def join_language_lists(): # include wikipedia first for more accurate language names - # exclude languages with too few articles languages.update({code: lang for code, lang in engines_languages['wikipedia'].iteritems() - if valid_code(code) and lang['articles'] >= 100000}) + if valid_code(code)}) for engine_name in engines_languages: for locale in engines_languages[engine_name]: @@ -104,25 +105,27 @@ def join_language_lists(): # if language is not on list or if it has no name yet if locale not in languages or not languages[locale].get('name'): - if isinstance(engines_languages[engine_name], dict) \ - and engines_languages[engine_name][locale].get('articles', float('inf')) >= 100000: + if isinstance(engines_languages[engine_name], dict): languages[locale] = engines_languages[engine_name][locale] else: languages[locale] = {} # get locales that have no name or country yet for locale in languages.keys(): + # try to get language names if not languages[locale].get('name'): - # try to get language names name = languages.get(locale.split('-')[0], {}).get('name', None) if name: languages[locale]['name'] = name - languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '') else: # filter out locales with no name del languages[locale] continue + # try to get language name in english + if not languages[locale].get('english_name'): + languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '') + # try to get country name if locale.find('-') > 0 and not languages[locale].get('country'): languages[locale]['country'] = get_country_name(locale) or '' @@ -145,10 +148,10 @@ def filter_single_country_languages(): # Write languages.py. def write_languages_file(): new_file = open(languages_file, 'w') - file_content = '# -*- coding: utf-8 -*-\n' - file_content += '# list of language codes\n' - file_content += '# this file is generated automatically by utils/update_search_languages.py\n' - file_content += '\nlanguage_codes = (' + file_content = '# -*- coding: utf-8 -*-\n'\ + + '# list of language codes\n'\ + + '# this file is generated automatically by utils/update_search_languages.py\n'\ + + '\nlanguage_codes = (' for code in sorted(languages): file_content += '\n (u"' + code + '"'\ + ', u"' + languages[code]['name'].split(' (')[0] + '"'\ -- cgit v1.2.3