diff options
Diffstat (limited to 'searx/utils.py')
| -rw-r--r-- | searx/utils.py | 213 |
1 files changed, 183 insertions, 30 deletions
diff --git a/searx/utils.py b/searx/utils.py index 3df571160..5ea9dc89c 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -1,14 +1,23 @@ +# -*- coding: utf-8 -*- import csv +import hashlib +import hmac import os import re +from babel.core import get_global from babel.dates import format_date from codecs import getincrementalencoder from imp import load_source +from numbers import Number from os.path import splitext, join +from io import open from random import choice +from lxml.etree import XPath import sys +import json +from searx import settings from searx.version import VERSION_STRING from searx.languages import language_codes from searx import settings @@ -27,31 +36,24 @@ except: if sys.version_info[0] == 3: unichr = chr unicode = str + IS_PY2 = False + basestring = str +else: + IS_PY2 = True logger = logger.getChild('utils') -ua_versions = ('40.0', - '41.0', - '42.0', - '43.0', - '44.0', - '45.0', - '46.0', - '47.0') - -ua_os = ('Windows NT 6.3; WOW64', - 'X11; Linux x86_64', - 'X11; Linux x86') - -ua = "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}" - blocked_tags = ('script', 'style') +ecma_unescape4_re = re.compile(r'%u([0-9a-fA-F]{4})', re.UNICODE) +ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE) + +useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__)) + + "/data/useragents.json", 'r', encoding='utf-8').read()) -def gen_useragent(): - # TODO - return ua.format(os=choice(ua_os), version=choice(ua_versions)) +xpath_cache = dict() +lang_to_lc_cache = dict() def searx_useragent(): @@ -60,6 +62,10 @@ def searx_useragent(): suffix=settings['outgoing'].get('useragent_suffix', '')) +def gen_useragent(os=None): + return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions']))) + + def highlight_content(content, query): if not content: @@ -157,19 +163,22 @@ class UnicodeWriter: self.encoder = getincrementalencoder(encoding)() def writerow(self, row): - unicode_row = [] - for col in row: - if type(col) == str or type(col) == unicode: - unicode_row.append(col.encode('utf-8').strip()) - else: - unicode_row.append(col) - self.writer.writerow([x.decode('utf-8') if hasattr(x, 'decode') else x for x in unicode_row]) + if IS_PY2: + row = [s.encode("utf-8") if hasattr(s, 'encode') else s for s in row] + self.writer.writerow(row) # Fetch UTF-8 output from the queue ... - data = self.queue.getvalue().strip('\x00') + data = self.queue.getvalue() + if IS_PY2: + data = data.decode("utf-8") + else: + data = data.strip('\x00') # ... and reencode it into the target encoding data = self.encoder.encode(data) # write to the target stream - self.stream.write(data.decode('utf-8')) + if IS_PY2: + self.stream.write(data) + else: + self.stream.write(data.decode("utf-8")) # empty queue self.queue.truncate(0) @@ -182,7 +191,7 @@ def get_resources_directory(searx_directory, subdirectory, resources_directory): if not resources_directory: resources_directory = os.path.join(searx_directory, subdirectory) if not os.path.isdir(resources_directory): - raise Exception(directory + " is not a directory") + raise Exception(resources_directory + " is not a directory") return resources_directory @@ -290,20 +299,96 @@ def convert_str_to_int(number_str): return 0 +# convert a variable to integer or return 0 if it's not a number +def int_or_zero(num): + if isinstance(num, list): + if len(num) < 1: + return 0 + num = num[0] + return convert_str_to_int(num) + + def is_valid_lang(lang): is_abbr = (len(lang) == 2) + lang = lang.lower().decode('utf-8') if is_abbr: for l in language_codes: - if l[0][:2] == lang.lower(): + if l[0][:2] == lang: return (True, l[0][:2], l[3].lower()) return False else: for l in language_codes: - if l[1].lower() == lang.lower(): + if l[1].lower() == lang or l[3].lower() == lang: return (True, l[0][:2], l[3].lower()) return False +def _get_lang_to_lc_dict(lang_list): + key = str(lang_list) + value = lang_to_lc_cache.get(key, None) + if value is None: + value = dict() + for lc in lang_list: + value.setdefault(lc.split('-')[0], lc) + lang_to_lc_cache[key] = value + return value + + +# auxiliary function to match lang_code in lang_list +def _match_language(lang_code, lang_list=[], custom_aliases={}): + # replace language code with a custom alias if necessary + if lang_code in custom_aliases: + lang_code = custom_aliases[lang_code] + + if lang_code in lang_list: + return lang_code + + # try to get the most likely country for this language + subtags = get_global('likely_subtags').get(lang_code) + if subtags: + subtag_parts = subtags.split('_') + new_code = subtag_parts[0] + '-' + subtag_parts[-1] + if new_code in custom_aliases: + new_code = custom_aliases[new_code] + if new_code in lang_list: + return new_code + + # try to get the any supported country for this language + return _get_lang_to_lc_dict(lang_list).get(lang_code, None) + + +# get the language code from lang_list that best matches locale_code +def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'): + # try to get language from given locale_code + language = _match_language(locale_code, lang_list, custom_aliases) + if language: + return language + + locale_parts = locale_code.split('-') + lang_code = locale_parts[0] + + # try to get language using an equivalent country code + if len(locale_parts) > 1: + country_alias = get_global('territory_aliases').get(locale_parts[-1]) + if country_alias: + language = _match_language(lang_code + '-' + country_alias[0], lang_list, custom_aliases) + if language: + return language + + # try to get language using an equivalent language code + alias = get_global('language_aliases').get(lang_code) + if alias: + language = _match_language(alias, lang_list, custom_aliases) + if language: + return language + + if lang_code != locale_code: + # try to get language from given language without giving the country + language = _match_language(lang_code, lang_list, custom_aliases) + + return language or fallback + + def load_module(filename, module_dir): modname = splitext(filename)[0] if modname in sys.modules: @@ -312,3 +397,71 @@ def load_module(filename, module_dir): module = load_source(modname, filepath) module.name = modname return module + + +def new_hmac(secret_key, url): + try: + secret_key_bytes = bytes(secret_key, 'utf-8') + except TypeError as err: + if isinstance(secret_key, bytes): + secret_key_bytes = secret_key + else: + raise err + if sys.version_info[0] == 2: + return hmac.new(bytes(secret_key), url, hashlib.sha256).hexdigest() + else: + return hmac.new(secret_key_bytes, url, hashlib.sha256).hexdigest() + + +def to_string(obj): + if isinstance(obj, basestring): + return obj + if isinstance(obj, Number): + return unicode(obj) + if hasattr(obj, '__str__'): + return obj.__str__() + if hasattr(obj, '__repr__'): + return obj.__repr__() + + +def ecma_unescape(s): + """ + python implementation of the unescape javascript function + + https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string + https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape + """ + # s = unicode(s) + # "%u5409" becomes "吉" + s = ecma_unescape4_re.sub(lambda e: unichr(int(e.group(1), 16)), s) + # "%20" becomes " ", "%F3" becomes "ó" + s = ecma_unescape2_re.sub(lambda e: unichr(int(e.group(1), 16)), s) + return s + + +def get_engine_from_settings(name): + """Return engine configuration from settings.yml of a given engine name""" + + if 'engines' not in settings: + return {} + + for engine in settings['engines']: + if 'name' not in engine: + continue + if name == engine['name']: + return engine + + return {} + + +def get_xpath(xpath_str): + result = xpath_cache.get(xpath_str, None) + if result is None: + result = XPath(xpath_str) + xpath_cache[xpath_str] = result + return result + + +def eval_xpath(element, xpath_str): + xpath = get_xpath(xpath_str) + return xpath(element) |