diff options
70 files changed, 2586 insertions, 591 deletions
diff --git a/Dockerfile b/Dockerfile index 831a429e2..543c74d0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,22 @@ -FROM debian:stable +FROM python:2.7-slim + +WORKDIR /app + +RUN useradd searx + +EXPOSE 5000 +CMD ["/usr/local/bin/uwsgi", "--uid", "searx", "--gid", "searx", "--http", ":5000", "-w", "searx.webapp"] RUN apt-get update && \ apt-get install -y --no-install-recommends \ - python-dev python2.7-minimal python-virtualenv \ - python-pybabel python-pip zlib1g-dev \ - libxml2-dev libxslt1-dev build-essential \ - openssl + zlib1g-dev libxml2-dev libxslt1-dev libffi-dev build-essential \ + libssl-dev openssl && \ + rm -rf /var/lib/apt/lists/* -RUN useradd searx +RUN pip install --no-cache uwsgi -WORKDIR /app -RUN pip install uwsgi COPY requirements.txt /app/requirements.txt -RUN pip install -r requirements.txt +RUN pip install --no-cache -r requirements.txt COPY . /app RUN sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml - -EXPOSE 5000 -CMD ["/usr/local/bin/uwsgi", "--uid", "searx", "--gid", "searx", "--http", ":5000", "-w", "searx.webapp"] diff --git a/searx/autocomplete.py b/searx/autocomplete.py index f5775bc63..1a324b8a9 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -57,17 +57,17 @@ def searx_bang(full_query): # check if query starts with categorie name for categorie in categories: if categorie.startswith(engine_query): - results.append(first_char+'{categorie}'.format(categorie=categorie)) + results.append(first_char + '{categorie}'.format(categorie=categorie)) # check if query starts with engine name for engine in engines: if engine.startswith(engine_query.replace('_', ' ')): - results.append(first_char+'{engine}'.format(engine=engine.replace(' ', '_'))) + results.append(first_char + '{engine}'.format(engine=engine.replace(' ', '_'))) # check if query starts with engine shortcut for engine_shortcut in engine_shortcuts: if engine_shortcut.startswith(engine_query): - results.append(first_char+'{engine_shortcut}'.format(engine_shortcut=engine_shortcut)) + results.append(first_char + '{engine_shortcut}'.format(engine_shortcut=engine_shortcut)) # check if current query stats with :bang elif first_char == ':': @@ -112,7 +112,7 @@ def searx_bang(full_query): def dbpedia(query): # dbpedia autocompleter, no HTTPS - autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' # noqa + autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' response = get(autocomplete_url + urlencode(dict(QueryString=query))) @@ -139,7 +139,7 @@ def duckduckgo(query): def google(query): # google autocompleter - autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&' # noqa + autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&' response = get(autocomplete_url + urlencode(dict(q=query))) @@ -153,9 +153,19 @@ def google(query): return results +def startpage(query): + # wikipedia autocompleter + url = 'https://startpage.com/do/suggest?{query}' + + resp = get(url.format(query=urlencode({'query': query}))).text.split('\n') + if len(resp) > 1: + return resp + return [] + + def wikipedia(query): # wikipedia autocompleter - url = 'https://en.wikipedia.org/w/api.php?action=opensearch&{0}&limit=10&namespace=0&format=json' # noqa + url = 'https://en.wikipedia.org/w/api.php?action=opensearch&{0}&limit=10&namespace=0&format=json' resp = loads(get(url.format(urlencode(dict(search=query)))).text) if len(resp) > 1: @@ -166,5 +176,6 @@ def wikipedia(query): backends = {'dbpedia': dbpedia, 'duckduckgo': duckduckgo, 'google': google, + 'startpage': startpage, 'wikipedia': wikipedia } diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 18a45d851..42e1f08bc 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -71,6 +71,9 @@ def load_engine(engine_data): if not hasattr(engine, 'language_support'): engine.language_support = True + if not hasattr(engine, 'safesearch'): + engine.safesearch = False + if not hasattr(engine, 'timeout'): engine.timeout = settings['server']['request_timeout'] diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index b06a57edc..839b8e5be 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -28,7 +28,7 @@ safesearch = True # search-url base_url = 'https://www.bing.com/' search_string = 'images/search?{query}&count=10&first={offset}' -thumb_url = "http://ts1.mm.bing.net/th?id={ihk}" # no https, bad certificate +thumb_url = "https://www.bing.com/th?id={ihk}" # safesearch definitions safesearch_types = {2: 'STRICT', diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 1e5d361c1..a2397c48e 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -6,18 +6,17 @@ max. 5000 query/month @using-api no (because of query limit) - @results HTML (using search portal) - @stable no (HTML can change) - @parse url, title, content, publishedDate + @results RSS (using search portal) + @stable yes (except perhaps for the images) + @parse url, title, content, publishedDate, thumbnail """ from urllib import urlencode -from cgi import escape -from lxml import html -from datetime import datetime, timedelta +from urlparse import urlparse, parse_qsl +from datetime import datetime from dateutil import parser -import re -from searx.engines.xpath import extract_text +from lxml import etree +from searx.utils import list_get # engine dependent config categories = ['news'] @@ -26,7 +25,25 @@ language_support = True # search-url base_url = 'https://www.bing.com/' -search_string = 'news/search?{query}&first={offset}' +search_string = 'news/search?{query}&first={offset}&format=RSS' + + +# remove click +def url_cleanup(url_string): + parsed_url = urlparse(url_string) + if parsed_url.netloc == 'www.bing.com' and parsed_url.path == '/news/apiclick.aspx': + query = dict(parse_qsl(parsed_url.query)) + return query.get('url', None) + return url_string + + +# replace the http://*bing4.com/th?id=... by https://www.bing.com/th?id=... +def image_url_cleanup(url_string): + parsed_url = urlparse(url_string) + if parsed_url.netloc.endswith('bing4.com') and parsed_url.path == '/th': + query = dict(parse_qsl(parsed_url.query)) + return "https://www.bing.com/th?id=" + query.get('id') + return url_string # do search-request @@ -42,8 +59,6 @@ def request(query, params): query=urlencode({'q': query, 'setmkt': language}), offset=offset) - params['cookies']['_FP'] = "ui=en-US" - params['url'] = base_url + search_path return params @@ -53,50 +68,44 @@ def request(query, params): def response(resp): results = [] - dom = html.fromstring(resp.content) + rss = etree.fromstring(resp.content) + + ns = rss.nsmap # parse results - for result in dom.xpath('//div[@class="sn_r"]'): - link = result.xpath('.//div[@class="newstitle"]/a')[0] - url = link.attrib.get('href') - title = extract_text(link) - contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]') - content = escape(extract_text(contentXPath)) - - # parse publishedDate - publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div' - '//span[contains(@class,"sn_ST")]' - '//span[contains(@class,"sn_tm")]') - - publishedDate = escape(extract_text(publishedDateXPath)) - - if re.match("^[0-9]+ minute(s|) ago$", publishedDate): - timeNumbers = re.findall(r'\d+', publishedDate) - publishedDate = datetime.now() - timedelta(minutes=int(timeNumbers[0])) - elif re.match("^[0-9]+ hour(s|) ago$", publishedDate): - timeNumbers = re.findall(r'\d+', publishedDate) - publishedDate = datetime.now() - timedelta(hours=int(timeNumbers[0])) - elif re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$", publishedDate): - timeNumbers = re.findall(r'\d+', publishedDate) - publishedDate = datetime.now()\ - - timedelta(hours=int(timeNumbers[0]))\ - - timedelta(minutes=int(timeNumbers[1])) - elif re.match("^[0-9]+ day(s|) ago$", publishedDate): - timeNumbers = re.findall(r'\d+', publishedDate) - publishedDate = datetime.now() - timedelta(days=int(timeNumbers[0])) - else: - try: - publishedDate = parser.parse(publishedDate, dayfirst=False) - except TypeError: - publishedDate = datetime.now() - except ValueError: - publishedDate = datetime.now() + for item in rss.xpath('./channel/item'): + # url / title / content + url = url_cleanup(item.xpath('./link/text()')[0]) + title = list_get(item.xpath('./title/text()'), 0, url) + content = list_get(item.xpath('./description/text()'), 0, '') + + # publishedDate + publishedDate = list_get(item.xpath('./pubDate/text()'), 0) + try: + publishedDate = parser.parse(publishedDate, dayfirst=False) + except TypeError: + publishedDate = datetime.now() + except ValueError: + publishedDate = datetime.now() + + # thumbnail + thumbnail = list_get(item.xpath('./News:Image/text()', namespaces=ns), 0) + if thumbnail is not None: + thumbnail = image_url_cleanup(thumbnail) # append result - results.append({'url': url, - 'title': title, - 'publishedDate': publishedDate, - 'content': content}) + if thumbnail is not None: + results.append({'template': 'videos.html', + 'url': url, + 'title': title, + 'publishedDate': publishedDate, + 'content': content, + 'thumbnail': thumbnail}) + else: + results.append({'url': url, + 'title': title, + 'publishedDate': publishedDate, + 'content': content}) # return results return results diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py index 1ba4575c5..26830a167 100644 --- a/searx/engines/currency_convert.py +++ b/searx/engines/currency_convert.py @@ -9,7 +9,7 @@ categories = [] url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X' weight = 100 -parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([^.0-9].+)\W*in?\W*([^\.]+)\W*$', re.I) # noqa +parser_re = re.compile(u'^\W*(\d+(?:\.\d+)?)\W*([^.0-9].+)\W+in?\W+([^\.]+)\W*$', re.I) # noqa db = 1 @@ -17,7 +17,7 @@ db = 1 def normalize_name(name): name = name.lower().replace('-', ' ') name = re.sub(' +', ' ', name) - return unicodedata.normalize('NFKD', u"" + name).lower() + return unicodedata.normalize('NFKD', name).lower() def name_to_iso4217(name): @@ -35,7 +35,7 @@ def iso4217_to_name(iso4217, language): def request(query, params): - m = parser_re.match(query) + m = parser_re.match(unicode(query, 'utf8')) if not m: # wrong query return params diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py index 2071b8e36..87b912eb3 100644 --- a/searx/engines/flickr_noapi.py +++ b/searx/engines/flickr_noapi.py @@ -25,7 +25,7 @@ categories = ['images'] url = 'https://www.flickr.com/' search_url = url + 'search?{query}&page={page}' photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' -regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) +regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') paging = True @@ -38,6 +38,7 @@ def build_flickr_url(user_id, photo_id): def request(query, params): params['url'] = search_url.format(query=urlencode({'text': query}), page=params['pageno']) + return params @@ -75,10 +76,10 @@ def response(resp): logger.debug('cannot find valid image size: {0}'.format(repr(photo))) continue - if 'id' not in photo['owner']: + if 'ownerNsid' not in photo: continue -# For a bigger thumbnail, keep only the url_z, not the url_n + # For a bigger thumbnail, keep only the url_z, not the url_n if 'n' in photo['sizes']: thumbnail_src = photo['sizes']['n']['url'] elif 'z' in photo['sizes']: @@ -86,20 +87,14 @@ def response(resp): else: thumbnail_src = img_src - url = build_flickr_url(photo['owner']['id'], photo['id']) + url = build_flickr_url(photo['ownerNsid'], photo['id']) title = photo.get('title', '') content = '<span class="photo-author">' +\ - photo['owner']['username'] +\ + photo['username'] +\ '</span><br />' - if 'description' in photo: - content = content +\ - '<span class="description">' +\ - photo['description'] +\ - '</span>' - # append result results.append({'url': url, 'title': title, diff --git a/searx/engines/google.py b/searx/engines/google.py index 807c58ed5..0e78a9e2c 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -8,39 +8,126 @@ # @stable no (HTML can change) # @parse url, title, content, suggestion +import re from urllib import urlencode from urlparse import urlparse, parse_qsl from lxml import html from searx.poolrequests import get from searx.engines.xpath import extract_text, extract_url + # engine dependent config categories = ['general'] paging = True language_support = True +use_locale_domain = True + +# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests +default_hostname = 'www.google.com' + +country_to_hostname = { + 'BG': 'www.google.bg', # Bulgaria + 'CZ': 'www.google.cz', # Czech Republic + 'DE': 'www.google.de', # Germany + 'DK': 'www.google.dk', # Denmark + 'AT': 'www.google.at', # Austria + 'CH': 'www.google.ch', # Switzerland + 'GR': 'www.google.gr', # Greece + 'AU': 'www.google.com.au', # Australia + 'CA': 'www.google.ca', # Canada + 'GB': 'www.google.co.uk', # United Kingdom + 'ID': 'www.google.co.id', # Indonesia + 'IE': 'www.google.ie', # Ireland + 'IN': 'www.google.co.in', # India + 'MY': 'www.google.com.my', # Malaysia + 'NZ': 'www.google.co.nz', # New Zealand + 'PH': 'www.google.com.ph', # Philippines + 'SG': 'www.google.com.sg', # Singapore + # 'US': 'www.google.us', # United State, redirect to .com + 'ZA': 'www.google.co.za', # South Africa + 'AR': 'www.google.com.ar', # Argentina + 'CL': 'www.google.cl', # Chile + 'ES': 'www.google.es', # Span + 'MX': 'www.google.com.mx', # Mexico + 'EE': 'www.google.ee', # Estonia + 'FI': 'www.google.fi', # Finland + 'BE': 'www.google.be', # Belgium + 'FR': 'www.google.fr', # France + 'IL': 'www.google.co.il', # Israel + 'HR': 'www.google.hr', # Croatia + 'HU': 'www.google.hu', # Hungary + 'IT': 'www.google.it', # Italy + 'JP': 'www.google.co.jp', # Japan + 'KR': 'www.google.co.kr', # South Korean + 'LT': 'www.google.lt', # Lithuania + 'LV': 'www.google.lv', # Latvia + 'NO': 'www.google.no', # Norway + 'NL': 'www.google.nl', # Netherlands + 'PL': 'www.google.pl', # Poland + 'BR': 'www.google.com.br', # Brazil + 'PT': 'www.google.pt', # Portugal + 'RO': 'www.google.ro', # Romania + 'RU': 'www.google.ru', # Russia + 'SK': 'www.google.sk', # Slovakia + 'SL': 'www.google.si', # Slovenia (SL -> si) + 'SE': 'www.google.se', # Sweden + 'TH': 'www.google.co.th', # Thailand + 'TR': 'www.google.com.tr', # Turkey + 'UA': 'www.google.com.ua', # Ikraine + # 'CN': 'www.google.cn', # China, only from china ? + 'HK': 'www.google.com.hk', # Hong kong + 'TW': 'www.google.com.tw' # Taiwan +} + +# osm +url_map = 'https://www.openstreetmap.org/'\ + + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M' # search-url -google_hostname = 'www.google.com' search_path = '/search' -redirect_path = '/url' -images_path = '/images' -search_url = ('https://' + - google_hostname + +search_url = ('https://{hostname}' + search_path + '?{query}&start={offset}&gbv=1') +# other URLs +map_hostname_start = 'maps.google.' +maps_path = '/maps' +redirect_path = '/url' +images_path = '/images' + # specific xpath variables results_xpath = '//li[@class="g"]' url_xpath = './/h3/a/@href' title_xpath = './/h3' content_xpath = './/span[@class="st"]' +content_misc_xpath = './/div[@class="f slp"]' suggestion_xpath = '//p[@class="_Bmc"]' +# map : detail location +map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()' +map_phone_xpath = './/div[@class="s"]//table//td[2]/span/span' +map_website_url_xpath = 'h3[2]/a/@href' +map_website_title_xpath = 'h3[2]' + +# map : near the location +map_near = 'table[@class="ts"]//tr' +map_near_title = './/h4' +map_near_url = './/h4/a/@href' +map_near_phone = './/span[@class="nobr"]' + +# images images_xpath = './/div/a' image_url_xpath = './@href' image_img_src_xpath = './img/@src' +# property names +# FIXME : no translation +property_address = "Address" +property_phone = "Phone number" + +# cookies pref_cookie = '' +nid_cookie = {} # see https://support.google.com/websearch/answer/873?hl=en @@ -52,8 +139,21 @@ def get_google_pref_cookie(): return pref_cookie +def get_google_nid_cookie(google_hostname): + global nid_cookie + if google_hostname not in nid_cookie: + resp = get('https://' + google_hostname) + nid_cookie[google_hostname] = resp.cookies.get("NID", None) + return nid_cookie[google_hostname] + + # remove google-specific tracking-url -def parse_url(url_string): +def parse_url(url_string, google_hostname): + # sanity check + if url_string is None: + return url_string + + # normal case parsed_url = urlparse(url_string) if (parsed_url.netloc in [google_hostname, ''] and parsed_url.path == redirect_path): @@ -63,21 +163,45 @@ def parse_url(url_string): return url_string +# returns extract_text on the first result selected by the xpath or None +def extract_text_from_dom(result, xpath): + r = result.xpath(xpath) + if len(r) > 0: + return extract_text(r[0]) + return None + + # do search-request def request(query, params): offset = (params['pageno'] - 1) * 10 if params['language'] == 'all': language = 'en' + country = 'US' else: - language = params['language'].replace('_', '-').lower() + language_array = params['language'].lower().split('_') + if len(language_array) == 2: + country = language_array[1] + else: + country = 'US' + language = language_array[0] + ',' + language_array[0] + '-' + country + + if use_locale_domain: + google_hostname = country_to_hostname.get(country.upper(), default_hostname) + else: + google_hostname = default_hostname params['url'] = search_url.format(offset=offset, - query=urlencode({'q': query})) + query=urlencode({'q': query}), + hostname=google_hostname) params['headers']['Accept-Language'] = language - if language.startswith('en'): + params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + if google_hostname == default_hostname: params['cookies']['PREF'] = get_google_pref_cookie() + params['cookies']['NID'] = get_google_nid_cookie(google_hostname) + + params['google_hostname'] = google_hostname return params @@ -86,33 +210,63 @@ def request(query, params): def response(resp): results = [] + # detect google sorry + resp_url = urlparse(resp.url) + if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect': + raise RuntimeWarning('sorry.google.com') + + # which hostname ? + google_hostname = resp.search_params.get('google_hostname') + google_url = "https://" + google_hostname + + # convert the text to dom dom = html.fromstring(resp.text) # parse results for result in dom.xpath(results_xpath): title = extract_text(result.xpath(title_xpath)[0]) try: - url = parse_url(extract_url(result.xpath(url_xpath), search_url)) - parsed_url = urlparse(url) - if (parsed_url.netloc == google_hostname - and parsed_url.path == search_path): - # remove the link to google news - continue + url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname) + parsed_url = urlparse(url, google_hostname) + + # map result + if ((parsed_url.netloc == google_hostname and parsed_url.path.startswith(maps_path)) + or (parsed_url.netloc.startswith(map_hostname_start))): + x = result.xpath(map_near) + if len(x) > 0: + # map : near the location + results = results + parse_map_near(parsed_url, x, google_hostname) + else: + # map : detail about a location + results = results + parse_map_detail(parsed_url, result, google_hostname) + + # google news + elif (parsed_url.netloc == google_hostname + and parsed_url.path == search_path): + # skipping news results + pass # images result - if (parsed_url.netloc == google_hostname - and parsed_url.path == images_path): + elif (parsed_url.netloc == google_hostname + and parsed_url.path == images_path): # only thumbnail image provided, # so skipping image results - # results = results + parse_images(result) + # results = results + parse_images(result, google_hostname) pass + else: # normal result - content = extract_text(result.xpath(content_xpath)[0]) + content = extract_text_from_dom(result, content_xpath) + if content is None: + continue + content_misc = extract_text_from_dom(result, content_misc_xpath) + if content_misc is not None: + content = content_misc + "<br />" + content # append result results.append({'url': url, 'title': title, - 'content': content}) + 'content': content + }) except: continue @@ -125,10 +279,10 @@ def response(resp): return results -def parse_images(result): +def parse_images(result, google_hostname): results = [] for image in result.xpath(images_xpath): - url = parse_url(extract_text(image.xpath(image_url_xpath)[0])) + url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname) img_src = extract_text(image.xpath(image_img_src_xpath)[0]) # append result @@ -136,6 +290,77 @@ def parse_images(result): 'title': '', 'content': '', 'img_src': img_src, - 'template': 'images.html'}) + 'template': 'images.html' + }) + + return results + + +def parse_map_near(parsed_url, x, google_hostname): + results = [] + + for result in x: + title = extract_text_from_dom(result, map_near_title) + url = parse_url(extract_text_from_dom(result, map_near_url), google_hostname) + attributes = [] + phone = extract_text_from_dom(result, map_near_phone) + add_attributes(attributes, property_phone, phone, 'tel:' + phone) + results.append({'title': title, + 'url': url, + 'content': attributes_to_html(attributes) + }) return results + + +def parse_map_detail(parsed_url, result, google_hostname): + results = [] + + # try to parse the geoloc + m = re.search('@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path) + if m is None: + m = re.search('ll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query) + + if m is not None: + # geoloc found (ignored) + lon = float(m.group(2)) # noqa + lat = float(m.group(1)) # noqa + zoom = int(m.group(3)) # noqa + + # attributes + attributes = [] + address = extract_text_from_dom(result, map_address_xpath) + phone = extract_text_from_dom(result, map_phone_xpath) + add_attributes(attributes, property_address, address, 'geo:' + str(lat) + ',' + str(lon)) + add_attributes(attributes, property_phone, phone, 'tel:' + phone) + + # title / content / url + website_title = extract_text_from_dom(result, map_website_title_xpath) + content = extract_text_from_dom(result, content_xpath) + website_url = parse_url(extract_text_from_dom(result, map_website_url_xpath), google_hostname) + + # add a result if there is a website + if website_url is not None: + results.append({'title': website_title, + 'content': (content + '<br />' if content is not None else '') + + attributes_to_html(attributes), + 'url': website_url + }) + + return results + + +def add_attributes(attributes, name, value, url): + if value is not None and len(value) > 0: + attributes.append({'label': name, 'value': value, 'url': url}) + + +def attributes_to_html(attributes): + retval = '<table class="table table-striped">' + for a in attributes: + value = a.get('value') + if 'url' in a: + value = '<a href="' + a.get('url') + '">' + value + '</a>' + retval = retval + '<tr><th>' + a.get('label') + '</th><td>' + value + '</td></tr>' + retval = retval + '</table>' + return retval diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py new file mode 100644 index 000000000..872bd4e95 --- /dev/null +++ b/searx/engines/qwant.py @@ -0,0 +1,98 @@ +""" + Qwant (Web, Images, News, Social) + + @website https://qwant.com/ + @provide-api not officially (https://api.qwant.com/api/search/) + + @using-api yes + @results JSON + @stable yes + @parse url, title, content +""" + +from urllib import urlencode +from json import loads +from datetime import datetime + +# engine dependent config +categories = None +paging = True +language_support = True + +category_to_keyword = {'general': 'web', + 'images': 'images', + 'news': 'news', + 'social media': 'social'} + +# search-url +url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}' + + +# do search-request +def request(query, params): + offset = (params['pageno'] - 1) * 10 + + if categories[0] and categories[0] in category_to_keyword: + + params['url'] = url.format(keyword=category_to_keyword[categories[0]], + query=urlencode({'q': query}), + offset=offset) + else: + params['url'] = url.format(keyword='web', + query=urlencode({'q': query}), + offset=offset) + + # add language tag if specified + if params['language'] != 'all': + params['url'] += '&locale=' + params['language'].lower() + + return params + + +# get response from search-request +def response(resp): + results = [] + + search_results = loads(resp.text) + + # return empty array if there are no results + if 'data' not in search_results: + return [] + + data = search_results.get('data', {}) + + res = data.get('result', {}) + + # parse results + for result in res.get('items', {}): + + title = result['title'] + res_url = result['url'] + content = result['desc'] + + if category_to_keyword.get(categories[0], '') == 'web': + results.append({'title': title, + 'content': content, + 'url': res_url}) + + elif category_to_keyword.get(categories[0], '') == 'images': + thumbnail_src = result['thumbnail'] + img_src = result['media'] + results.append({'template': 'images.html', + 'url': res_url, + 'title': title, + 'content': '', + 'thumbnail_src': thumbnail_src, + 'img_src': img_src}) + + elif (category_to_keyword.get(categories[0], '') == 'news' or + category_to_keyword.get(categories[0], '') == 'social'): + published_date = datetime.fromtimestamp(result['date'], None) + + results.append({'url': res_url, + 'title': title, + 'publishedDate': published_date, + 'content': content}) + + # return results + return results diff --git a/searx/engines/swisscows.py b/searx/engines/swisscows.py new file mode 100644 index 000000000..2d31264ca --- /dev/null +++ b/searx/engines/swisscows.py @@ -0,0 +1,108 @@ +""" + Swisscows (Web, Images) + + @website https://swisscows.ch + @provide-api no + + @using-api no + @results HTML (using search portal) + @stable no (HTML can change) + @parse url, title, content +""" + +from json import loads +from urllib import urlencode, unquote +import re + +# engine dependent config +categories = ['general', 'images'] +paging = True +language_support = True + +# search-url +base_url = 'https://swisscows.ch/' +search_string = '?{query}&page={page}' + +# regex +regex_json = re.compile('initialData: {"Request":(.|\n)*},\s*environment') +regex_json_remove_start = re.compile('^initialData:\s*') +regex_json_remove_end = re.compile(',\s*environment$') +regex_img_url_remove_start = re.compile('^https?://i\.swisscows\.ch/\?link=') + + +# do search-request +def request(query, params): + if params['language'] == 'all': + ui_language = 'browser' + region = 'browser' + else: + region = params['language'].replace('_', '-') + ui_language = params['language'].split('_')[0] + + search_path = search_string.format( + query=urlencode({'query': query, + 'uiLanguage': ui_language, + 'region': region}), + page=params['pageno']) + + # image search query is something like 'image?{query}&page={page}' + if params['category'] == 'images': + search_path = 'image' + search_path + + params['url'] = base_url + search_path + + return params + + +# get response from search-request +def response(resp): + results = [] + + json_regex = regex_json.search(resp.content) + + # check if results are returned + if not json_regex: + return [] + + json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group())) + json = loads(json_raw) + + # parse results + for result in json['Results'].get('items', []): + result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '') + + # parse image results + if result.get('ContentType', '').startswith('image'): + img_url = unquote(regex_img_url_remove_start.sub('', result['Url'])) + + # append result + results.append({'url': result['SourceUrl'], + 'title': result['Title'], + 'content': '', + 'img_src': img_url, + 'template': 'images.html'}) + + # parse general results + else: + result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '') + result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '') + + # append result + results.append({'url': result_url, + 'title': result_title, + 'content': result_content}) + + # parse images + for result in json.get('Images', []): + # decode image url + img_url = unquote(regex_img_url_remove_start.sub('', result['Url'])) + + # append result + results.append({'url': result['SourceUrl'], + 'title': result['Title'], + 'content': '', + 'img_src': img_url, + 'template': 'images.html'}) + + # return results + return results diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py index 0dcc65b7c..517ac1c44 100644 --- a/searx/engines/vimeo.py +++ b/searx/engines/vimeo.py @@ -27,11 +27,11 @@ base_url = 'https://vimeo.com' search_url = base_url + '/search/page:{pageno}?{query}' # specific xpath variables -results_xpath = '//div[@id="browse_content"]/ol/li' -url_xpath = './a/@href' -title_xpath = './a/div[@class="data"]/p[@class="title"]' -content_xpath = './a/img/@src' -publishedDate_xpath = './/p[@class="meta"]//attribute::datetime' +results_xpath = '//div[contains(@class,"results_grid")]/ul/li' +url_xpath = './/a/@href' +title_xpath = './/span[@class="title"]' +thumbnail_xpath = './/img[@class="js-clip_thumbnail_image"]/@src' +publishedDate_xpath = './/time/attribute::datetime' embedded_url = '<iframe data-src="//player.vimeo.com/video{videoid}" ' +\ 'width="540" height="304" frameborder="0" ' +\ @@ -58,7 +58,7 @@ def response(resp): videoid = result.xpath(url_xpath)[0] url = base_url + videoid title = p.unescape(extract_text(result.xpath(title_xpath))) - thumbnail = extract_text(result.xpath(content_xpath)[0]) + thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) publishedDate = parser.parse(extract_text(result.xpath(publishedDate_xpath)[0])) embedded = embedded_url.format(videoid=videoid) diff --git a/searx/engines/www1x.py b/searx/engines/www1x.py index 12868ad22..ddb79bfea 100644 --- a/searx/engines/www1x.py +++ b/searx/engines/www1x.py @@ -20,8 +20,8 @@ import re categories = ['images'] paging = False -# search-url, no HTTPS (there is a valid certificate for https://api2.1x.com/ ) -base_url = 'http://1x.com' +# search-url +base_url = 'https://1x.com' search_url = base_url+'/backend/search.php?{query}' diff --git a/searx/engines/youtube_api.py b/searx/engines/youtube_api.py new file mode 100644 index 000000000..8fd939a25 --- /dev/null +++ b/searx/engines/youtube_api.py @@ -0,0 +1,83 @@ +# Youtube (Videos) +# +# @website https://www.youtube.com/ +# @provide-api yes (https://developers.google.com/apis-explorer/#p/youtube/v3/youtube.search.list) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, content, publishedDate, thumbnail, embedded + +from json import loads +from urllib import urlencode +from dateutil import parser + +# engine dependent config +categories = ['videos', 'music'] +paging = False +language_support = True +api_key = None + +# search-url +base_url = 'https://www.googleapis.com/youtube/v3/search' +search_url = base_url + '?part=snippet&{query}&maxResults=20&key={api_key}' + +embedded_url = '<iframe width="540" height="304" ' +\ + 'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\ + 'frameborder="0" allowfullscreen></iframe>' + +base_youtube_url = 'https://www.youtube.com/watch?v=' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(query=urlencode({'q': query}), + api_key=api_key) + + # add language tag if specified + if params['language'] != 'all': + params['url'] += '&relevanceLanguage=' + params['language'].split('_')[0] + + return params + + +# get response from search-request +def response(resp): + results = [] + + search_results = loads(resp.text) + + # return empty array if there are no results + if 'items' not in search_results: + return [] + + # parse results + for result in search_results['items']: + videoid = result['id']['videoId'] + + title = result['snippet']['title'] + content = '' + thumbnail = '' + + pubdate = result['snippet']['publishedAt'] + publishedDate = parser.parse(pubdate) + + thumbnail = result['snippet']['thumbnails']['high']['url'] + + content = result['snippet']['description'] + + url = base_youtube_url + videoid + + embedded = embedded_url.format(videoid=videoid) + + # append result + results.append({'url': url, + 'title': title, + 'content': content, + 'template': 'videos.html', + 'publishedDate': publishedDate, + 'embedded': embedded, + 'thumbnail': thumbnail}) + + # return results + return results diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py new file mode 100644 index 000000000..401fca4c9 --- /dev/null +++ b/searx/engines/youtube_noapi.py @@ -0,0 +1,81 @@ +# Youtube (Videos) +# +# @website https://www.youtube.com/ +# @provide-api yes (https://developers.google.com/apis-explorer/#p/youtube/v3/youtube.search.list) +# +# @using-api no +# @results HTML +# @stable no +# @parse url, title, content, publishedDate, thumbnail, embedded + +from urllib import quote_plus +from lxml import html +from searx.engines.xpath import extract_text +from searx.utils import list_get + +# engine dependent config +categories = ['videos', 'music'] +paging = True +language_support = False + +# search-url +base_url = 'https://www.youtube.com/results' +search_url = base_url + '?search_query={query}&page={page}' + +embedded_url = '<iframe width="540" height="304" ' +\ + 'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\ + 'frameborder="0" allowfullscreen></iframe>' + +base_youtube_url = 'https://www.youtube.com/watch?v=' + +# specific xpath variables +results_xpath = "//ol/li/div[contains(@class, 'yt-lockup yt-lockup-tile yt-lockup-video vve-check')]" +url_xpath = './/h3/a/@href' +title_xpath = './/div[@class="yt-lockup-content"]/h3/a' +content_xpath = './/div[@class="yt-lockup-content"]/div[@class="yt-lockup-description yt-ui-ellipsis yt-ui-ellipsis-2"]' + + +# returns extract_text on the first result selected by the xpath or None +def extract_text_from_dom(result, xpath): + r = result.xpath(xpath) + if len(r) > 0: + return extract_text(r[0]) + return None + + +# do search-request +def request(query, params): + params['url'] = search_url.format(query=quote_plus(query), + page=params['pageno']) + + return params + + +# get response from search-request +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + # parse results + for result in dom.xpath(results_xpath): + videoid = list_get(result.xpath('@data-context-item-id'), 0) + if videoid is not None: + url = base_youtube_url + videoid + thumbnail = 'https://i.ytimg.com/vi/' + videoid + '/hqdefault.jpg' + + title = extract_text_from_dom(result, title_xpath) or videoid + content = extract_text_from_dom(result, content_xpath) + + embedded = embedded_url.format(videoid=videoid) + + # append result + results.append({'url': url, + 'title': title, + 'content': content, + 'template': 'videos.html', + 'embedded': embedded, + 'thumbnail': thumbnail}) + + # return results + return results diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py index 5ac3f447c..a4d7ad8a8 100644 --- a/searx/plugins/__init__.py +++ b/searx/plugins/__init__.py @@ -20,8 +20,9 @@ from searx import logger logger = logger.getChild('plugins') from searx.plugins import (https_rewrite, - self_ip, - search_on_category_select) + self_info, + search_on_category_select, + tracker_url_remover) required_attrs = (('name', str), ('description', str), @@ -71,5 +72,6 @@ class PluginStore(): plugins = PluginStore() plugins.register(https_rewrite) -plugins.register(self_ip) +plugins.register(self_info) plugins.register(search_on_category_select) +plugins.register(tracker_url_remover) diff --git a/searx/plugins/self_ip.py b/searx/plugins/self_info.py index 5184ea4cf..5ca994526 100644 --- a/searx/plugins/self_ip.py +++ b/searx/plugins/self_info.py @@ -15,11 +15,16 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2015 by Adam Tauber, <asciimoo@gmail.com> ''' from flask.ext.babel import gettext -name = "Self IP" -description = gettext('Display your source IP address if the query expression is "ip"') +import re +name = "Self Informations" +description = gettext('Displays your IP if the query is "ip" and your user agent if the query contains "user agent".') default_on = True +# Self User Agent regex +p = re.compile('.*user[ -]agent.*', re.IGNORECASE) + + # attach callback to the post search hook # request: flask request object # ctx: the whole local context of the pre search hook @@ -32,4 +37,8 @@ def post_search(request, ctx): ip = request.remote_addr ctx['search'].answers.clear() ctx['search'].answers.add(ip) + elif p.match(ctx['search'].query): + ua = request.user_agent + ctx['search'].answers.clear() + ctx['search'].answers.add(ua) return True diff --git a/searx/plugins/tracker_url_remover.py b/searx/plugins/tracker_url_remover.py new file mode 100644 index 000000000..ed71c94d3 --- /dev/null +++ b/searx/plugins/tracker_url_remover.py @@ -0,0 +1,44 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2015 by Adam Tauber, <asciimoo@gmail.com> +''' + +from flask.ext.babel import gettext +import re +from urlparse import urlunparse + +regexes = {re.compile(r'utm_[^&]+&?'), + re.compile(r'(wkey|wemail)[^&]+&?'), + re.compile(r'&$')} + +name = gettext('Tracker URL remover') +description = gettext('Remove trackers arguments from the returned URL') +default_on = True + + +def on_result(request, ctx): + query = ctx['result']['parsed_url'].query + + if query == "": + return True + + for reg in regexes: + query = reg.sub('', query) + + if query != ctx['result']['parsed_url'].query: + ctx['result']['parsed_url'] = ctx['result']['parsed_url']._replace(query=query) + ctx['result']['url'] = urlunparse(ctx['result']['parsed_url']) + + return True diff --git a/searx/search.py b/searx/search.py index e7ac7bb66..bb440352b 100644 --- a/searx/search.py +++ b/searx/search.py @@ -237,7 +237,7 @@ def score_results(results): for k in categoryPositions: v = categoryPositions[k]['index'] if v >= index: - categoryPositions[k]['index'] = v+1 + categoryPositions[k]['index'] = v + 1 # update this category current['count'] -= 1 @@ -306,7 +306,7 @@ def merge_infoboxes(infoboxes): if add_infobox: results.append(infobox) - infoboxes_id[infobox_id] = len(results)-1 + infoboxes_id[infobox_id] = len(results) - 1 return results @@ -472,7 +472,12 @@ class Search(object): request_params['category'] = selected_engine['category'] request_params['started'] = time() request_params['pageno'] = self.pageno - request_params['language'] = self.lang + + if hasattr(engine, 'language'): + request_params['language'] = engine.language + else: + request_params['language'] = self.lang + try: # 0 = None, 1 = Moderate, 2 = Strict request_params['safesearch'] = int(request.cookies.get('safesearch', 1)) diff --git a/searx/settings.yml b/searx/settings.yml index 98a2ed786..03d895363 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -169,6 +169,27 @@ engines: shortcut : tpb disabled : True + - name : qwant + engine : qwant + shortcut : qw + categories : general + disabled : True + + - name : qwant images + engine : qwant + shortcut : qwi + categories : images + + - name : qwant news + engine : qwant + shortcut : qwn + categories : news + + - name : qwant social + engine : qwant + shortcut : qws + categories : social media + - name : kickass engine : kickass shortcut : ka @@ -204,12 +225,21 @@ engines: - name : startpage engine : startpage shortcut : sp + timeout : 6.0 + disabled : True + + - name : ixquick + engine : startpage + base_url : 'https://www.ixquick.com/' + search_url : 'https://www.ixquick.com/do/search' + shortcut : iq + timeout : 6.0 + disabled : True -# +30% page load time -# - name : ixquick -# engine : startpage -# base_url : 'https://www.ixquick.com/' -# search_url : 'https://www.ixquick.com/do/search' + - name : swisscows + engine : swisscows + shortcut : sw + disabled : True - name : twitter engine : twitter @@ -240,8 +270,13 @@ engines: shortcut : yhn - name : youtube - engine : youtube shortcut : yt + # You can use the engine using the official stable API, but you need an API key + # See : https://console.developers.google.com/project + # engine : youtube_api + # api_key: 'apikey' # required! + # Or you can use the html non-stable engine, activated by default + engine : youtube_noapi - name : dailymotion engine : dailymotion @@ -275,5 +310,6 @@ locales: nl : Nederlands ja : 日本語 (Japanese) tr : Türkçe + pt: Português ru : Russian ro : Romanian diff --git a/searx/static/js/search_on_category_select.js b/searx/static/js/search_on_category_select.js index 6156ca4e8..5ecc2cdb9 100644 --- a/searx/static/js/search_on_category_select.js +++ b/searx/static/js/search_on_category_select.js @@ -1,5 +1,5 @@ $(document).ready(function() { - if($('#q')) { + if($('#q').length) { $('#categories label').click(function(e) { $('#categories input[type="checkbox"]').each(function(i, checkbox) { $(checkbox).prop('checked', false); @@ -7,7 +7,9 @@ $(document).ready(function() { $('#categories label').removeClass('btn-primary').removeClass('active').addClass('btn-default'); $(this).removeClass('btn-default').addClass('btn-primary').addClass('active'); $($(this).children()[0]).prop('checked', 'checked'); - $('#search_form').submit(); + if($('#q').val()) { + $('#search_form').submit(); + } return false; }); } diff --git a/searx/static/themes/courgette/img/favicon.png b/searx/static/themes/courgette/img/favicon.png Binary files differindex cefbac496..1a43d7fa6 100644 --- a/searx/static/themes/courgette/img/favicon.png +++ b/searx/static/themes/courgette/img/favicon.png diff --git a/searx/static/themes/courgette/img/github_ribbon.png b/searx/static/themes/courgette/img/github_ribbon.png Binary files differindex 146ef8a80..3799c2ea1 100644 --- a/searx/static/themes/courgette/img/github_ribbon.png +++ b/searx/static/themes/courgette/img/github_ribbon.png diff --git a/searx/static/themes/courgette/img/preference-icon.png b/searx/static/themes/courgette/img/preference-icon.png Binary files differindex 039db04a8..57e991cc6 100644 --- a/searx/static/themes/courgette/img/preference-icon.png +++ b/searx/static/themes/courgette/img/preference-icon.png diff --git a/searx/static/themes/courgette/img/search-icon.png b/searx/static/themes/courgette/img/search-icon.png Binary files differindex 52c267842..9bc7a222c 100644 --- a/searx/static/themes/courgette/img/search-icon.png +++ b/searx/static/themes/courgette/img/search-icon.png diff --git a/searx/static/themes/courgette/img/searx-mobile.png b/searx/static/themes/courgette/img/searx-mobile.png Binary files differindex 2b9383a5d..b5af3865c 100644 --- a/searx/static/themes/courgette/img/searx-mobile.png +++ b/searx/static/themes/courgette/img/searx-mobile.png diff --git a/searx/static/themes/courgette/img/searx.png b/searx/static/themes/courgette/img/searx.png Binary files differindex e162da502..68c2e4ffd 100644 --- a/searx/static/themes/courgette/img/searx.png +++ b/searx/static/themes/courgette/img/searx.png diff --git a/searx/static/themes/default/img/favicon.png b/searx/static/themes/default/img/favicon.png Binary files differindex 28afb0111..1a43d7fa6 100644 --- a/searx/static/themes/default/img/favicon.png +++ b/searx/static/themes/default/img/favicon.png diff --git a/searx/static/themes/default/img/github_ribbon.png b/searx/static/themes/default/img/github_ribbon.png Binary files differindex 146ef8a80..3799c2ea1 100644 --- a/searx/static/themes/default/img/github_ribbon.png +++ b/searx/static/themes/default/img/github_ribbon.png diff --git a/searx/static/themes/default/img/preference-icon.png b/searx/static/themes/default/img/preference-icon.png Binary files differindex 300279d24..8bdee641d 100644 --- a/searx/static/themes/default/img/preference-icon.png +++ b/searx/static/themes/default/img/preference-icon.png diff --git a/searx/static/themes/default/img/searx.png b/searx/static/themes/default/img/searx.png Binary files differindex e69e9eef9..a98f12a1d 100644 --- a/searx/static/themes/default/img/searx.png +++ b/searx/static/themes/default/img/searx.png diff --git a/searx/static/themes/oscar/img/favicon.png b/searx/static/themes/oscar/img/favicon.png Binary files differindex cefbac496..1a43d7fa6 100644 --- a/searx/static/themes/oscar/img/favicon.png +++ b/searx/static/themes/oscar/img/favicon.png diff --git a/searx/static/themes/oscar/img/icons/github.png b/searx/static/themes/oscar/img/icons/github.png Binary files differindex bf09bae55..e6439715b 100644 --- a/searx/static/themes/oscar/img/icons/github.png +++ b/searx/static/themes/oscar/img/icons/github.png diff --git a/searx/static/themes/oscar/img/icons/searchcode code.png b/searx/static/themes/oscar/img/icons/searchcode code.png Binary files differindex 884c2660d..968a7ce27 100644 --- a/searx/static/themes/oscar/img/icons/searchcode code.png +++ b/searx/static/themes/oscar/img/icons/searchcode code.png diff --git a/searx/static/themes/oscar/img/icons/searchcode doc.png b/searx/static/themes/oscar/img/icons/searchcode doc.png Binary files differindex 884c2660d..968a7ce27 100644 --- a/searx/static/themes/oscar/img/icons/searchcode doc.png +++ b/searx/static/themes/oscar/img/icons/searchcode doc.png diff --git a/searx/static/themes/oscar/img/icons/wikipedia.png b/searx/static/themes/oscar/img/icons/wikipedia.png Binary files differindex 37997fbc7..f77168382 100644 --- a/searx/static/themes/oscar/img/icons/wikipedia.png +++ b/searx/static/themes/oscar/img/icons/wikipedia.png diff --git a/searx/static/themes/oscar/img/icons/youtube.png b/searx/static/themes/oscar/img/icons/youtube.png Binary files differindex ba2484d0f..de64dff7b 100644 --- a/searx/static/themes/oscar/img/icons/youtube.png +++ b/searx/static/themes/oscar/img/icons/youtube.png diff --git a/searx/static/themes/oscar/img/map/layers-2x.png b/searx/static/themes/oscar/img/map/layers-2x.png Binary files differindex a2cf7f9ef..0b30da678 100644 --- a/searx/static/themes/oscar/img/map/layers-2x.png +++ b/searx/static/themes/oscar/img/map/layers-2x.png diff --git a/searx/static/themes/oscar/img/map/layers.png b/searx/static/themes/oscar/img/map/layers.png Binary files differindex bca0a0e42..4297fd9ee 100644 --- a/searx/static/themes/oscar/img/map/layers.png +++ b/searx/static/themes/oscar/img/map/layers.png diff --git a/searx/static/themes/oscar/img/map/marker-icon-2x-green.png b/searx/static/themes/oscar/img/map/marker-icon-2x-green.png Binary files differindex c359abb6c..7446bb031 100644 --- a/searx/static/themes/oscar/img/map/marker-icon-2x-green.png +++ b/searx/static/themes/oscar/img/map/marker-icon-2x-green.png diff --git a/searx/static/themes/oscar/img/map/marker-icon-2x-orange.png b/searx/static/themes/oscar/img/map/marker-icon-2x-orange.png Binary files differindex c3c863211..ecd67736f 100644 --- a/searx/static/themes/oscar/img/map/marker-icon-2x-orange.png +++ b/searx/static/themes/oscar/img/map/marker-icon-2x-orange.png diff --git a/searx/static/themes/oscar/img/map/marker-icon-2x-red.png b/searx/static/themes/oscar/img/map/marker-icon-2x-red.png Binary files differindex 1c26e9fc2..1d2e197c6 100644 --- a/searx/static/themes/oscar/img/map/marker-icon-2x-red.png +++ b/searx/static/themes/oscar/img/map/marker-icon-2x-red.png diff --git a/searx/static/themes/oscar/img/map/marker-icon-green.png b/searx/static/themes/oscar/img/map/marker-icon-green.png Binary files differindex 56db5ea9f..f48ef41df 100644 --- a/searx/static/themes/oscar/img/map/marker-icon-green.png +++ b/searx/static/themes/oscar/img/map/marker-icon-green.png diff --git a/searx/static/themes/oscar/img/map/marker-icon-orange.png b/searx/static/themes/oscar/img/map/marker-icon-orange.png Binary files differindex fbbce7b2a..d0d22205c 100644 --- a/searx/static/themes/oscar/img/map/marker-icon-orange.png +++ b/searx/static/themes/oscar/img/map/marker-icon-orange.png diff --git a/searx/static/themes/oscar/img/map/marker-icon-red.png b/searx/static/themes/oscar/img/map/marker-icon-red.png Binary files differindex 3e64e06d1..7a92b9e04 100644 --- a/searx/static/themes/oscar/img/map/marker-icon-red.png +++ b/searx/static/themes/oscar/img/map/marker-icon-red.png diff --git a/searx/static/themes/oscar/img/searx_logo.png b/searx/static/themes/oscar/img/searx_logo.png Binary files differindex 26be6def7..ea4837b05 100644 --- a/searx/static/themes/oscar/img/searx_logo.png +++ b/searx/static/themes/oscar/img/searx_logo.png diff --git a/searx/static/themes/pix-art/img/favicon.png b/searx/static/themes/pix-art/img/favicon.png Binary files differindex 28afb0111..1a43d7fa6 100644 --- a/searx/static/themes/pix-art/img/favicon.png +++ b/searx/static/themes/pix-art/img/favicon.png diff --git a/searx/static/themes/pix-art/img/searx-pixel-small.png b/searx/static/themes/pix-art/img/searx-pixel-small.png Binary files differindex 76f381c5c..75b476c7f 100644 --- a/searx/static/themes/pix-art/img/searx-pixel-small.png +++ b/searx/static/themes/pix-art/img/searx-pixel-small.png diff --git a/searx/templates/courgette/preferences.html b/searx/templates/courgette/preferences.html index 2afb74d11..f89915d8d 100644 --- a/searx/templates/courgette/preferences.html +++ b/searx/templates/courgette/preferences.html @@ -101,7 +101,7 @@ <th>{{ _('Category') }}</th> <th>{{ _('Allow') }} / {{ _('Block') }}</th> </tr> - {% for categ in categories %} + {% for categ in all_categories %} {% for search_engine in engines_by_category[categ] %} {% if not search_engine.private %} diff --git a/searx/templates/default/preferences.html b/searx/templates/default/preferences.html index 0afe9f7d0..90006c029 100644 --- a/searx/templates/default/preferences.html +++ b/searx/templates/default/preferences.html @@ -89,7 +89,7 @@ <th>{{ _('Category') }}</th> <th>{{ _('Allow') }} / {{ _('Block') }}</th> </tr> - {% for categ in categories %} + {% for categ in all_categories %} {% for search_engine in engines_by_category[categ] %} {% if not search_engine.private %} diff --git a/searx/templates/oscar/messages/no_cookies.html b/searx/templates/oscar/messages/no_cookies.html new file mode 100644 index 000000000..9bebc8ad1 --- /dev/null +++ b/searx/templates/oscar/messages/no_cookies.html @@ -0,0 +1,5 @@ +{% from 'oscar/macros.html' import icon %} +<div class="alert alert-info fade in" role="alert"> + <strong class="lead">{{ icon('info-sign') }} {{ _('Information!') }}</strong> + {{ _('currently, there are no cookies defined.') }} +</div> diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html index 693167807..ea36a14b9 100644 --- a/searx/templates/oscar/preferences.html +++ b/searx/templates/oscar/preferences.html @@ -117,7 +117,7 @@ <!-- Nav tabs --> <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;"> - {% for categ in categories %} + {% for categ in all_categories %} <li{% if loop.first %} class="active"{% endif %}><a href="#tab_engine_{{ categ|replace(' ', '_') }}" role="tab" data-toggle="tab">{{ _(categ) }}</a></li> {% endfor %} </ul> @@ -128,27 +128,54 @@ <!-- Tab panes --> <div class="tab-content"> - {% for categ in categories %} + {% for categ in all_categories %} <noscript><label>{{ _(categ) }}</label> </noscript> <div class="tab-pane{% if loop.first %} active{% endif %} active_if_nojs" id="tab_engine_{{ categ|replace(' ', '_') }}"> <div class="container-fluid"> <fieldset> + <div class="table-responsive"> + <table class="table table-hover table-condensed table-striped"> + <tr> + {% if not rtl %} + <th>{{ _("Allow") }}</th> + <th>{{ _("Engine name") }}</th> + <th>{{ _("Shortcut") }}</th> + <th>{{ _("SafeSearch") }}</th> + <th>{{ _("Avg. time") }}</th> + <th>{{ _("Max time") }}</th> + {% else %} + <th>{{ _("Max time") }}</th> + <th>{{ _("Avg. time") }}</th> + <th>{{ _("SafeSearch") }}</th> + <th>{{ _("Shortcut") }}</th> + <th>{{ _("Engine name") }}</th> + <th>{{ _("Allow") }}</th> + {% endif %} + </tr> {% for search_engine in engines_by_category[categ] %} {% if not search_engine.private %} - <div class="row"> + <tr> {% if not rtl %} - <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div> + <td>{{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}</td> + <th>{{ search_engine.name }}</th> + <td>{{ shortcuts[search_engine.name] }}</td> + <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td> + <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td> + <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td> + {% else %} + <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td> + <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td> + <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td> + <td>{{ shortcuts[search_engine.name] }}</td> + <th>{{ search_engine.name }}</th> + <td>{{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}</td> {% endif %} - <div class="col-xs-6 col-sm-4 col-md-4"> - {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }} - </div> - {% if rtl %} - <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})‎</div> - {% endif %} - </div> + </tr> {% endif %} {% endfor %} + </table> + </div> </fieldset> </div> </div> @@ -186,21 +213,23 @@ {{ _('This is the list of cookies and their values searx is storing on your computer.') }}<br /> {{ _('With that list, you can assess searx transparency.') }}<br /> </p> - <div class="container-fluid"> - <fieldset> - <div class="row"> - <div class="col-xs-6 col-sm-4 col-md-4 text-muted"><label>{{ _('Cookie name') }}</label></div> - <div class="col-xs-6 col-sm-4 col-md-4 text-muted"><label>{{ _('Value') }}</label></div> - </div> + {% if cookies %} + <table class="table table-striped"> + <tr> + <th class="text-muted" style="padding-right:40px;">{{ _('Cookie name') }}</th> + <th class="text-muted">{{ _('Value') }}</th> + </tr> {% for cookie in cookies %} - <div class="row"> - <div class="col-xs-6 col-sm-4 col-md-4 text-muted">{{ cookie }}</div> - <div class="col-xs-6 col-sm-4 col-md-4 text-muted">{{ cookies[cookie] }}</div> - </div> + <tr> + <td class="text-muted" style="padding-right:40px;">{{ cookie }}</td> + <td class="text-muted">{{ cookies[cookie] }}</td> + </tr> {% endfor %} - </fieldset> - </div> + </table> + {% else %} + {% include 'oscar/messages/no_cookies.html' %} + {% endif %} </div> </div> <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }} diff --git a/searx/tests/engines/test_bing_images.py b/searx/tests/engines/test_bing_images.py index a1d96b06e..f869da79d 100644 --- a/searx/tests/engines/test_bing_images.py +++ b/searx/tests/engines/test_bing_images.py @@ -59,7 +59,7 @@ oh:"238",tft:"0",oi:"http://www.image.url/Images/Test%2 self.assertEqual(results[0]['title'], 'Test Query') self.assertEqual(results[0]['url'], 'http://www.page.url/') self.assertEqual(results[0]['content'], '') - self.assertEqual(results[0]['thumbnail_src'], 'http://ts1.mm.bing.net/th?id=HN.608003696942779811') + self.assertEqual(results[0]['thumbnail_src'], 'https://www.bing.com/th?id=HN.608003696942779811') self.assertEqual(results[0]['img_src'], 'http://test.url/Test%20Query.jpg') html = """ diff --git a/searx/tests/engines/test_bing_news.py b/searx/tests/engines/test_bing_news.py index f22b80e87..a64d59b7b 100644 --- a/searx/tests/engines/test_bing_news.py +++ b/searx/tests/engines/test_bing_news.py @@ -2,6 +2,7 @@ from collections import defaultdict import mock from searx.engines import bing_news from searx.testing import SearxTestCase +import lxml class TestBingNewsEngine(SearxTestCase): @@ -16,14 +17,10 @@ class TestBingNewsEngine(SearxTestCase): self.assertIn(query, params['url']) self.assertIn('bing.com', params['url']) self.assertIn('fr', params['url']) - self.assertIn('_FP', params['cookies']) - self.assertIn('en', params['cookies']['_FP']) dicto['language'] = 'all' params = bing_news.request(query, dicto) self.assertIn('en', params['url']) - self.assertIn('_FP', params['cookies']) - self.assertIn('en', params['cookies']['_FP']) def test_response(self): self.assertRaises(AttributeError, bing_news.response, None) @@ -37,200 +34,105 @@ class TestBingNewsEngine(SearxTestCase): response = mock.Mock(content='<html></html>') self.assertEqual(bing_news.response(response), []) - html = """ - <div class="sn_r"> - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">44 minutes ago</span> - </span> - </div> - </div> - </div> - """ + html = """<?xml version="1.0" encoding="utf-8" ?> +<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS"> + <channel> + <title>python - Bing News</title> + <link>https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS</link> + <description>Search results</description> + <image> + <url>http://10.53.64.9/rsslogo.gif</url> + <title>test</title> + <link>https://www.bing.com:443/news/search?q=test&setmkt=en-US&first=1&format=RSS</link> + </image> + <copyright>Copyright</copyright> + <item> + <title>Title</title> + <link>https://www.bing.com/news/apiclick.aspx?ref=FexRss&aid=&tid=c237eccc50bd4758b106a5e3c94fce09&url=http%3a%2f%2furl.of.article%2f&c=xxxxxxxxx&mkt=en-us</link> + <description>Article Content</description> + <pubDate>Tue, 02 Jun 2015 13:37:00 GMT</pubDate> + <News:Source>Infoworld</News:Source> + <News:Image>http://a1.bing4.com/th?id=ON.13371337133713371337133713371337&pid=News</News:Image> + <News:ImageSize>w={0}&h={1}&c=7</News:ImageSize> + <News:ImageKeepOriginalRatio></News:ImageKeepOriginalRatio> + <News:ImageMaxWidth>620</News:ImageMaxWidth> + <News:ImageMaxHeight>413</News:ImageMaxHeight> + </item> + <item> + <title>Another Title</title> + <link>https://www.bing.com/news/apiclick.aspx?ref=FexRss&aid=&tid=c237eccc50bd4758b106a5e3c94fce09&url=http%3a%2f%2fanother.url.of.article%2f&c=xxxxxxxxx&mkt=en-us</link> + <description>Another Article Content</description> + <pubDate>Tue, 02 Jun 2015 13:37:00 GMT</pubDate> + </item> + </channel> +</rss>""" # noqa response = mock.Mock(content=html) results = bing_news.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'Title') self.assertEqual(results[0]['url'], 'http://url.of.article/') self.assertEqual(results[0]['content'], 'Article Content') + self.assertEqual(results[0]['thumbnail'], 'https://www.bing.com/th?id=ON.13371337133713371337133713371337') + self.assertEqual(results[1]['title'], 'Another Title') + self.assertEqual(results[1]['url'], 'http://another.url.of.article/') + self.assertEqual(results[1]['content'], 'Another Article Content') + self.assertNotIn('thumbnail', results[1]) - html = """ - <div class="sn_r"> - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">44 minutes ago</span> - </span> - </div> - </div> - </div> - <div class="sn_r"> - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">3 hours, 44 minutes ago</span> - </span> - </div> - </div> - </div> - <div class="sn_r"> - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">44 hours ago</span> - </span> - </div> - </div> - </div> - <div class="sn_r"> - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">2 days ago</span> - </span> - </div> - </div> - </div> - <div class="sn_r"> - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">27/01/2015</span> - </span> - </div> - </div> - </div> - <div class="sn_r"> - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">Il y a 3 heures</span> - </span> - </div> - </div> - </div> - """ + html = """<?xml version="1.0" encoding="utf-8" ?> +<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS"> + <channel> + <title>python - Bing News</title> + <link>https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS</link> + <description>Search results</description> + <image> + <url>http://10.53.64.9/rsslogo.gif</url> + <title>test</title> + <link>https://www.bing.com:443/news/search?q=test&setmkt=en-US&first=1&format=RSS</link> + </image> + <copyright>Copyright</copyright> + <item> + <title>Title</title> + <link>http://another.url.of.article/</link> + <description>Article Content</description> + <pubDate>garbage</pubDate> + <News:Source>Infoworld</News:Source> + <News:Image>http://another.bing.com/image</News:Image> + <News:ImageSize>w={0}&h={1}&c=7</News:ImageSize> + <News:ImageKeepOriginalRatio></News:ImageKeepOriginalRatio> + <News:ImageMaxWidth>620</News:ImageMaxWidth> + <News:ImageMaxHeight>413</News:ImageMaxHeight> + </item> + </channel> +</rss>""" # noqa response = mock.Mock(content=html) results = bing_news.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'http://another.url.of.article/') + self.assertEqual(results[0]['content'], 'Article Content') + self.assertEqual(results[0]['thumbnail'], 'http://another.bing.com/image') + + html = """<?xml version="1.0" encoding="utf-8" ?> +<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS"> + <channel> + <title>python - Bing News</title> + <link>https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS</link> + <description>Search results</description> + <image> + <url>http://10.53.64.9/rsslogo.gif</url> + <title>test</title> + <link>https://www.bing.com:443/news/search?q=test&setmkt=en-US&first=1&format=RSS</link> + </image> + </channel> +</rss>""" # noqa - html = """ - <div class="newstitle"> - <a href="http://url.of.article/" target="_blank" h="ID=news,5022.1"> - Title - </a> - </div> - <div class="sn_img"> - <a href="http://url.of.article2/" target="_blank" h="ID=news,5024.1"> - <img class="rms_img" height="80" id="emb1" src="/image.src" title="Title" width="80" /> - </a> - </div> - <div class="sn_txt"> - <div class="sn_oi"> - <span class="sn_snip">Article Content</span> - <span class="sn_ST"> - <cite class="sn_src">metronews.fr</cite> - ·  - <span class="sn_tm">44 minutes ago</span> - </span> - </div> - </div> - """ response = mock.Mock(content=html) results = bing_news.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + + html = """<?xml version="1.0" encoding="utf-8" ?>gabarge""" + response = mock.Mock(content=html) + self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response) diff --git a/searx/tests/engines/test_flickr_noapi.py b/searx/tests/engines/test_flickr_noapi.py index a1de3a5e4..3b337a2d8 100644 --- a/searx/tests/engines/test_flickr_noapi.py +++ b/searx/tests/engines/test_flickr_noapi.py @@ -26,19 +26,29 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertRaises(AttributeError, flickr_noapi.response, '') self.assertRaises(AttributeError, flickr_noapi.response, '[]') - response = mock.Mock(text='"search-photos-models","photos":{},"totalItems":') + response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":') self.assertEqual(flickr_noapi.response(response), []) - response = mock.Mock(text='search-photos-models","photos":{"data": []},"totalItems":') + response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":') self.assertEqual(flickr_noapi.response(response), []) + # everthing is ok test json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "c": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg", @@ -117,40 +127,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", "key": "z" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -168,15 +145,24 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertIn('k.jpg', results[0]['img_src']) self.assertIn('n.jpg', results[0]['thumbnail_src']) self.assertIn('Owner', results[0]['content']) - self.assertIn('Description', results[0]['content']) + # no n size, only the z size json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "z": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", @@ -185,40 +171,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", "key": "z" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -235,15 +188,24 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertIn('z.jpg', results[0]['img_src']) self.assertIn('z.jpg', results[0]['thumbnail_src']) self.assertIn('Owner', results[0]['content']) - self.assertIn('Description', results[0]['content']) + # no z or n size json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "o": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", @@ -252,39 +214,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", "key": "o" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -302,48 +232,25 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertIn('o.jpg', results[0]['thumbnail_src']) self.assertIn('Owner', results[0]['content']) + # no image test json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", - "sizes": { - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true, - "id": "59729010@N00" - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", + "sizes": { + } } ], "fetchedStart": true, @@ -356,6 +263,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + # null test json = """ "search-photos-models","photos": { @@ -370,13 +278,22 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + # no ownerNsid test json = """ - "search-photos-models","photos": + "search-photos-lite-models","photos": { "_data": [ { - "_flickrModelRegistry": "photo-models", + "_flickrModelRegistry": "photo-lite-models", "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", "sizes": { "o": { "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", @@ -385,39 +302,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", "key": "o" } - }, - "canComment": false, - "rotation": 0, - "owner": { - "_flickrModelRegistry": "person-models", - "pathAlias": "klink692", - "username": "Owner", - "buddyicon": { - "retina": null, - "large": null, - "medium": null, - "small": null, - "default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00" - }, - "isPro": true - }, - "engagement": { - "_flickrModelRegistry": "photo-engagement-models", - "ownerNsid": "59729010@N00", - "faveCount": 21, - "commentCount": 14, - "viewCount": 10160, - "id": "14001294434" - }, - "description": "Description", - "isHD": false, - "secret": "410f653777", - "canAddMeta": false, - "license": 0, - "oWidth": 1803, - "oHeight": 2669, - "safetyLevel": 0, - "id": "14001294434" + } } ], "fetchedStart": true, @@ -430,6 +315,7 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + # garbage test json = """ {"toto":[ {"id":200,"name":"Artist Name", diff --git a/searx/tests/engines/test_google.py b/searx/tests/engines/test_google.py index 2a90fc5ec..b706e511d 100644 --- a/searx/tests/engines/test_google.py +++ b/searx/tests/engines/test_google.py @@ -8,6 +8,12 @@ from searx.testing import SearxTestCase class TestGoogleEngine(SearxTestCase): + def mock_response(self, text): + response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1') + response.search_params = mock.Mock() + response.search_params.get = mock.Mock(return_value='www.google.com') + return response + def test_request(self): query = 'test_query' dicto = defaultdict(dict) @@ -16,14 +22,17 @@ class TestGoogleEngine(SearxTestCase): params = google.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) - self.assertIn('google.com', params['url']) + self.assertIn('google.fr', params['url']) self.assertNotIn('PREF', params['cookies']) + self.assertIn('NID', params['cookies']) self.assertIn('fr', params['headers']['Accept-Language']) dicto['language'] = 'all' params = google.request(query, dicto) + self.assertIn('google.com', params['url']) self.assertIn('en', params['headers']['Accept-Language']) self.assertIn('PREF', params['cookies']) + self.assertIn('NID', params['cookies']) def test_response(self): self.assertRaises(AttributeError, google.response, None) @@ -31,7 +40,7 @@ class TestGoogleEngine(SearxTestCase): self.assertRaises(AttributeError, google.response, '') self.assertRaises(AttributeError, google.response, '[]') - response = mock.Mock(text='<html></html>') + response = self.mock_response('<html></html>') self.assertEqual(google.response(response), []) html = """ @@ -124,7 +133,7 @@ class TestGoogleEngine(SearxTestCase): </a> </p> """ - response = mock.Mock(text=html) + response = self.mock_response(html) results = google.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 2) @@ -137,11 +146,21 @@ class TestGoogleEngine(SearxTestCase): <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> </li> """ - response = mock.Mock(text=html) + response = self.mock_response(html) results = google.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 0) + response = mock.Mock(text='<html></html>', url='https://sorry.google.com') + response.search_params = mock.Mock() + response.search_params.get = mock.Mock(return_value='www.google.com') + self.assertRaises(RuntimeWarning, google.response, response) + + response = mock.Mock(text='<html></html>', url='https://www.google.com/sorry/IndexRedirect') + response.search_params = mock.Mock() + response.search_params.get = mock.Mock(return_value='www.google.com') + self.assertRaises(RuntimeWarning, google.response, response) + def test_parse_images(self): html = """ <li> @@ -154,7 +173,7 @@ class TestGoogleEngine(SearxTestCase): </li> """ dom = lxml.html.fromstring(html) - results = google.parse_images(dom) + results = google.parse_images(dom, 'www.google.com') self.assertEqual(type(results), list) self.assertEqual(len(results), 1) self.assertEqual(results[0]['url'], 'http://this.is.the.url/') diff --git a/searx/tests/engines/test_qwant.py b/searx/tests/engines/test_qwant.py new file mode 100644 index 000000000..7d79d13d8 --- /dev/null +++ b/searx/tests/engines/test_qwant.py @@ -0,0 +1,317 @@ +from collections import defaultdict +import mock +from searx.engines import qwant +from searx.testing import SearxTestCase + + +class TestQwantEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + dicto['language'] = 'fr_FR' + qwant.categories = [''] + params = qwant.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('web', params['url']) + self.assertIn('qwant.com', params['url']) + self.assertIn('fr_fr', params['url']) + + dicto['language'] = 'all' + qwant.categories = ['news'] + params = qwant.request(query, dicto) + self.assertFalse('fr' in params['url']) + self.assertIn('news', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, qwant.response, None) + self.assertRaises(AttributeError, qwant.response, []) + self.assertRaises(AttributeError, qwant.response, '') + self.assertRaises(AttributeError, qwant.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(qwant.response(response), []) + + response = mock.Mock(text='{"data": {}}') + self.assertEqual(qwant.response(response), []) + + json = """ + { + "status": "success", + "data": { + "query": { + "locale": "en_us", + "query": "Test", + "offset": 10 + }, + "result": { + "items": [ + { + "title": "Title", + "score": 9999, + "url": "http://www.url.xyz", + "source": "...", + "desc": "Description", + "date": "", + "_id": "db0aadd62c2a8565567ffc382f5c61fa", + "favicon": "https://s.qwant.com/fav.ico" + } + ], + "filters": [] + }, + "cache": { + "key": "e66aa864c00147a0e3a16ff7a5efafde", + "created": 1433092754, + "expiration": 259200, + "status": "miss", + "age": 0 + } + } + } + """ + response = mock.Mock(text=json) + qwant.categories = ['general'] + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'http://www.url.xyz') + self.assertEqual(results[0]['content'], 'Description') + + json = """ + { + "status": "success", + "data": { + "query": { + "locale": "en_us", + "query": "Test", + "offset": 10 + }, + "result": { + "items": [ + { + "title": "Title", + "score": 9999, + "url": "http://www.url.xyz", + "source": "...", + "media": "http://image.jpg", + "desc": "", + "thumbnail": "http://thumbnail.jpg", + "date": "", + "_id": "db0aadd62c2a8565567ffc382f5c61fa", + "favicon": "https://s.qwant.com/fav.ico" + } + ], + "filters": [] + }, + "cache": { + "key": "e66aa864c00147a0e3a16ff7a5efafde", + "created": 1433092754, + "expiration": 259200, + "status": "miss", + "age": 0 + } + } + } + """ + response = mock.Mock(text=json) + qwant.categories = ['images'] + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'http://www.url.xyz') + self.assertEqual(results[0]['content'], '') + self.assertEqual(results[0]['thumbnail_src'], 'http://thumbnail.jpg') + self.assertEqual(results[0]['img_src'], 'http://image.jpg') + + json = """ + { + "status": "success", + "data": { + "query": { + "locale": "en_us", + "query": "Test", + "offset": 10 + }, + "result": { + "items": [ + { + "title": "Title", + "score": 9999, + "url": "http://www.url.xyz", + "source": "...", + "desc": "Description", + "date": 1433260920, + "_id": "db0aadd62c2a8565567ffc382f5c61fa", + "favicon": "https://s.qwant.com/fav.ico" + } + ], + "filters": [] + }, + "cache": { + "key": "e66aa864c00147a0e3a16ff7a5efafde", + "created": 1433092754, + "expiration": 259200, + "status": "miss", + "age": 0 + } + } + } + """ + response = mock.Mock(text=json) + qwant.categories = ['news'] + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'http://www.url.xyz') + self.assertEqual(results[0]['content'], 'Description') + self.assertIn('publishedDate', results[0]) + + json = """ + { + "status": "success", + "data": { + "query": { + "locale": "en_us", + "query": "Test", + "offset": 10 + }, + "result": { + "items": [ + { + "title": "Title", + "score": 9999, + "url": "http://www.url.xyz", + "source": "...", + "desc": "Description", + "date": 1433260920, + "_id": "db0aadd62c2a8565567ffc382f5c61fa", + "favicon": "https://s.qwant.com/fav.ico" + } + ], + "filters": [] + }, + "cache": { + "key": "e66aa864c00147a0e3a16ff7a5efafde", + "created": 1433092754, + "expiration": 259200, + "status": "miss", + "age": 0 + } + } + } + """ + response = mock.Mock(text=json) + qwant.categories = ['social media'] + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'http://www.url.xyz') + self.assertEqual(results[0]['content'], 'Description') + self.assertIn('publishedDate', results[0]) + + json = """ + { + "status": "success", + "data": { + "query": { + "locale": "en_us", + "query": "Test", + "offset": 10 + }, + "result": { + "items": [ + { + "title": "Title", + "score": 9999, + "url": "http://www.url.xyz", + "source": "...", + "desc": "Description", + "date": 1433260920, + "_id": "db0aadd62c2a8565567ffc382f5c61fa", + "favicon": "https://s.qwant.com/fav.ico" + } + ], + "filters": [] + }, + "cache": { + "key": "e66aa864c00147a0e3a16ff7a5efafde", + "created": 1433092754, + "expiration": 259200, + "status": "miss", + "age": 0 + } + } + } + """ + response = mock.Mock(text=json) + qwant.categories = [''] + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + { + "status": "success", + "data": { + "query": { + "locale": "en_us", + "query": "Test", + "offset": 10 + }, + "result": { + "filters": [] + }, + "cache": { + "key": "e66aa864c00147a0e3a16ff7a5efafde", + "created": 1433092754, + "expiration": 259200, + "status": "miss", + "age": 0 + } + } + } + """ + response = mock.Mock(text=json) + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + { + "status": "success", + "data": { + "query": { + "locale": "en_us", + "query": "Test", + "offset": 10 + }, + "cache": { + "key": "e66aa864c00147a0e3a16ff7a5efafde", + "created": 1433092754, + "expiration": 259200, + "status": "miss", + "age": 0 + } + } + } + """ + response = mock.Mock(text=json) + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + { + "status": "success" + } + """ + response = mock.Mock(text=json) + results = qwant.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_swisscows.py b/searx/tests/engines/test_swisscows.py new file mode 100644 index 000000000..3b4ce7b0f --- /dev/null +++ b/searx/tests/engines/test_swisscows.py @@ -0,0 +1,128 @@ +from collections import defaultdict +import mock +from searx.engines import swisscows +from searx.testing import SearxTestCase + + +class TestSwisscowsEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'de_DE' + params = swisscows.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('swisscows.ch' in params['url']) + self.assertTrue('uiLanguage=de' in params['url']) + self.assertTrue('region=de-DE' in params['url']) + + dicto['language'] = 'all' + params = swisscows.request(query, dicto) + self.assertTrue('uiLanguage=browser' in params['url']) + self.assertTrue('region=browser' in params['url']) + + dicto['category'] = 'images' + params = swisscows.request(query, dicto) + self.assertIn('image', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, swisscows.response, None) + self.assertRaises(AttributeError, swisscows.response, []) + self.assertRaises(AttributeError, swisscows.response, '') + self.assertRaises(AttributeError, swisscows.response, '[]') + + response = mock.Mock(content='<html></html>') + self.assertEqual(swisscows.response(response), []) + + response = mock.Mock(content='<html></html>') + self.assertEqual(swisscows.response(response), []) + + html = u""" + <script> + App.Dispatcher.dispatch("initialize", { + html5history: true, + initialData: {"Request": + {"Page":1, + "ItemsCount":1, + "Query":"This should ", + "NormalizedQuery":"This should ", + "Region":"de-AT", + "UILanguage":"de"}, + "Results":{"items":[ + {"Title":"\uE000This should\uE001 be the title", + "Description":"\uE000This should\uE001 be the content.", + "Url":"http://this.should.be.the.link/", + "DisplayUrl":"www.\uE000this.should.be.the\uE001.link", + "Id":"782ef287-e439-451c-b380-6ebc14ba033d"}, + {"Title":"Datei:This should1.svg", + "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png", + "SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should1.svg", + "DisplayUrl":"de.wikipedia.org/wiki/Datei:This should1.svg", + "Width":950, + "Height":534, + "FileSize":92100, + "ContentType":"image/jpeg", + "Thumbnail":{ + "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png", + "ContentType":"image/jpeg", + "Width":300, + "Height":168, + "FileSize":9134}, + "Id":"6a97a542-8f65-425f-b7f6-1178c3aba7be" + } + ],"TotalCount":55300, + "Query":"This should " + }, + "Images":[{"Title":"Datei:This should.svg", + "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png", + "SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should.svg", + "DisplayUrl":"de.wikipedia.org/wiki/Datei:This should.svg", + "Width":1280, + "Height":677, + "FileSize":50053, + "ContentType":"image/png", + "Thumbnail":{"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png", + "ContentType":"image/png", + "Width":300, + "Height":158, + "FileSize":8023}, + "Id":"ae230fd8-a06a-47d6-99d5-e74766d8143a"}]}, + environment: "production" + }).then(function (options) { + $('#Search_Form').on('submit', function (e) { + if (!Modernizr.history) return; + e.preventDefault(); + + var $form = $(this), + $query = $('#Query'), + query = $.trim($query.val()), + path = App.Router.makePath($form.attr('action'), null, $form.serializeObject()) + + if (query.length) { + options.html5history ? + ReactRouter.HistoryLocation.push(path) : + ReactRouter.RefreshLocation.push(path); + } + else $('#Query').trigger('blur'); + }); + + }); + </script> + """ + response = mock.Mock(content=html) + results = swisscows.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 3) + self.assertEqual(results[0]['title'], 'This should be the title') + self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') + self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[1]['title'], 'Datei:This should1.svg') + self.assertEqual(results[1]['url'], 'http://de.wikipedia.org/wiki/Datei:This should1.svg') + self.assertEqual(results[1]['img_src'], 'http://ts2.mm.This/should1.png') + self.assertEqual(results[1]['template'], 'images.html') + self.assertEqual(results[2]['title'], 'Datei:This should.svg') + self.assertEqual(results[2]['url'], 'http://de.wikipedia.org/wiki/Datei:This should.svg') + self.assertEqual(results[2]['img_src'], 'http://ts2.mm.This/should.png') + self.assertEqual(results[2]['template'], 'images.html') diff --git a/searx/tests/engines/test_vimeo.py b/searx/tests/engines/test_vimeo.py index dad7239b4..50b1cb563 100644 --- a/searx/tests/engines/test_vimeo.py +++ b/searx/tests/engines/test_vimeo.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from collections import defaultdict import mock from searx.engines import vimeo @@ -25,26 +26,42 @@ class TestVimeoEngine(SearxTestCase): self.assertEqual(vimeo.response(response), []) html = """ - <div id="browse_content" class="" data-search-id="696d5f8366914ec4ffec33cf7652de384976d4f4"> - <ol class="js-browse_list clearfix browse browse_videos browse_videos_thumbnails kane" + <div id="browse_content" class="results_grid" data-search-id="696d5f8366914ec4ffec33cf7652de384976d4f4"> + <ul class="js-browse_list clearfix browse browse_videos browse_videos_thumbnails kane" data-stream="c2VhcmNoOjo6ZGVzYzp7InF1ZXJ5IjoidGVzdCJ9"> - <li id="clip_100785455" data-start-page="/search/page:1/sort:relevant/" data-position="1"> - <a href="/videoid" title="Futurama 3d (test shot)"> - <img src="http://image.url.webp" - srcset="http://i.vimeocdn.com/video/482375085_590x332.webp 2x" alt="" - class="thumbnail thumbnail_lg_wide"> - <div class="data"> - <p class="title"> - This is the title - </p> - <p class="meta"> - <time datetime="2014-07-15T04:16:27-04:00" - title="mardi 15 juillet 2014 04:16">Il y a 6 mois</time> - </p> - </div> - </a> + <li data-position="7" data-result-id="clip_79600943"> + <div class="clip_thumbnail"> + <a href="/videoid" class="js-result_url"> + <div class="thumbnail_wrapper"> + <img src="http://image.url.webp" class="js-clip_thumbnail_image"> + <div class="overlay overlay_clip_meta"> + <div class="meta_data_footer"> + <span class="clip_upload_date"> + <time datetime="2013-11-17T08:49:09-05:00" + title="dimanche 17 novembre 2013 08:49">Il y a 1 an</time> + </span> + <span class="clip_likes"> + <img src="https://f.vimeocdn.com/images_v6/svg/heart-icon.svg">2 215 + </span> + <span class="clip_comments"> + <img src="https://f.vimeocdn.com/images_v6/svg/comment-icon.svg">75 + </span> + <span class="overlay meta_data_footer clip_duration">01:12</span> + </div> + </div> + </div> + <span class="title">This is the title</span> + </a> + </div> + <div class="clip_thumbnail_attribution"> + <a href="/fedorshmidt"> + <img src="https://i.vimeocdn.com/portrait/6628061_100x100.jpg" class="avatar"> + <span class="display_name">Fedor Shmidt</span> + </a> + <span class="plays">2,1M lectures</span> + </div> </li> - </ol> + </ul> </div> """ response = mock.Mock(text=html) diff --git a/searx/tests/engines/test_www1x.py b/searx/tests/engines/test_www1x.py index ab4f282c1..9df8de6bf 100644 --- a/searx/tests/engines/test_www1x.py +++ b/searx/tests/engines/test_www1x.py @@ -51,7 +51,7 @@ class TestWww1xEngine(SearxTestCase): results = www1x.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['url'], 'http://1x.com/photo/123456') - self.assertEqual(results[0]['thumbnail_src'], 'http://1x.com/images/user/testimage-123456.jpg') + self.assertEqual(results[0]['url'], 'https://1x.com/photo/123456') + self.assertEqual(results[0]['thumbnail_src'], 'https://1x.com/images/user/testimage-123456.jpg') self.assertEqual(results[0]['content'], '') self.assertEqual(results[0]['template'], 'images.html') diff --git a/searx/tests/engines/test_yahoo_news.py b/searx/tests/engines/test_yahoo_news.py index 94d819d61..4d7fc0a10 100644 --- a/searx/tests/engines/test_yahoo_news.py +++ b/searx/tests/engines/test_yahoo_news.py @@ -29,6 +29,13 @@ class TestYahooNewsEngine(SearxTestCase): self.assertIn('en', params['cookies']['sB']) self.assertIn('en', params['url']) + def test_sanitize_url(self): + url = "test.url" + self.assertEqual(url, yahoo_news.sanitize_url(url)) + + url = "www.yahoo.com/;_ylt=test" + self.assertEqual("www.yahoo.com/", yahoo_news.sanitize_url(url)) + def test_response(self): self.assertRaises(AttributeError, yahoo_news.response, None) self.assertRaises(AttributeError, yahoo_news.response, []) @@ -57,7 +64,17 @@ class TestYahooNewsEngine(SearxTestCase): This is the content </div> </li> - </div> + <li class="first"> + <div class="compTitle"> + <h3> + <a class="yschttl spt" target="_blank"> + </a> + </h3> + </div> + <div class="compText"> + </div> + </li> + </ol> """ response = mock.Mock(text=html) results = yahoo_news.response(response) diff --git a/searx/tests/engines/test_youtube_api.py b/searx/tests/engines/test_youtube_api.py new file mode 100644 index 000000000..0d4d478c3 --- /dev/null +++ b/searx/tests/engines/test_youtube_api.py @@ -0,0 +1,111 @@ +from collections import defaultdict +import mock +from searx.engines import youtube_api +from searx.testing import SearxTestCase + + +class TestYoutubeAPIEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + dicto['language'] = 'fr_FR' + params = youtube_api.request(query, dicto) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertIn('googleapis.com', params['url']) + self.assertIn('youtube', params['url']) + self.assertIn('fr', params['url']) + + dicto['language'] = 'all' + params = youtube_api.request(query, dicto) + self.assertFalse('fr' in params['url']) + + def test_response(self): + self.assertRaises(AttributeError, youtube_api.response, None) + self.assertRaises(AttributeError, youtube_api.response, []) + self.assertRaises(AttributeError, youtube_api.response, '') + self.assertRaises(AttributeError, youtube_api.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(youtube_api.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(youtube_api.response(response), []) + + json = """ + { + "kind": "youtube#searchListResponse", + "etag": "xmg9xJZuZD438sF4hb-VcBBREXc/YJQDcTBCDcaBvl-sRZJoXdvy1ME", + "nextPageToken": "CAUQAA", + "pageInfo": { + "totalResults": 1000000, + "resultsPerPage": 20 + }, + "items": [ + { + "kind": "youtube#searchResult", + "etag": "xmg9xJZuZD438sF4hb-VcBBREXc/IbLO64BMhbHIgWLwLw7MDYe7Hs4", + "id": { + "kind": "youtube#video", + "videoId": "DIVZCPfAOeM" + }, + "snippet": { + "publishedAt": "2015-05-29T22:41:04.000Z", + "channelId": "UCNodmx1ERIjKqvcJLtdzH5Q", + "title": "Title", + "description": "Description", + "thumbnails": { + "default": { + "url": "https://i.ytimg.com/vi/DIVZCPfAOeM/default.jpg" + }, + "medium": { + "url": "https://i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg" + }, + "high": { + "url": "https://i.ytimg.com/vi/DIVZCPfAOeM/hqdefault.jpg" + } + }, + "channelTitle": "MinecraftUniverse", + "liveBroadcastContent": "none" + } + } + ] + } + """ + response = mock.Mock(text=json) + results = youtube_api.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM') + self.assertEqual(results[0]['content'], 'Description') + self.assertEqual(results[0]['thumbnail'], 'https://i.ytimg.com/vi/DIVZCPfAOeM/hqdefault.jpg') + self.assertTrue('DIVZCPfAOeM' in results[0]['embedded']) + + json = """ + { + "kind": "youtube#searchListResponse", + "etag": "xmg9xJZuZD438sF4hb-VcBBREXc/YJQDcTBCDcaBvl-sRZJoXdvy1ME", + "nextPageToken": "CAUQAA", + "pageInfo": { + "totalResults": 1000000, + "resultsPerPage": 20 + } + } + """ + response = mock.Mock(text=json) + results = youtube_api.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) + + json = """ + {"toto":{"entry":[] + } + } + """ + response = mock.Mock(text=json) + results = youtube_api.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_youtube_noapi.py b/searx/tests/engines/test_youtube_noapi.py new file mode 100644 index 000000000..9fa8fd20e --- /dev/null +++ b/searx/tests/engines/test_youtube_noapi.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import youtube_noapi +from searx.testing import SearxTestCase + + +class TestYoutubeNoAPIEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = youtube_noapi.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('youtube.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, youtube_noapi.response, None) + self.assertRaises(AttributeError, youtube_noapi.response, []) + self.assertRaises(AttributeError, youtube_noapi.response, '') + self.assertRaises(AttributeError, youtube_noapi.response, '[]') + + response = mock.Mock(text='<html></html>') + self.assertEqual(youtube_noapi.response(response), []) + + html = """ + <ol id="item-section-063864" class="item-section"> + <li> + <div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile" + data-context-item-id="DIVZCPfAOeM" + data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB"> + <div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto"> + <a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link" + data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA"> + <div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg" + width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a> + <span class="thumb-menu dark-overflow-action-menu video-actions"> + </span> + </div> + <div class="yt-lockup-content"> + <h3 class="yt-lockup-title"> + <a href="/watch?v=DIVZCPfAOeM" + class="yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink spf-link" + data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA" + title="Top Speed Test Kawasaki Ninja H2 (Thailand) By. MEHAY SUPERBIKE" + aria-describedby="description-id-259079" rel="spf-prefetch" dir="ltr"> + Title + </a> + <span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span> + </h3> + <div class="yt-lockup-byline">de + <a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard" + data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA" + data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta"> + <ul class="yt-lockup-meta-info"> + <li>il y a 20 heures</li> + <li>8 424 vues</li> + </ul> + </div> + <div class="yt-lockup-description yt-ui-ellipsis yt-ui-ellipsis-2" dir="ltr"> + Description + </div> + <div class="yt-lockup-badges"> + <ul class="yt-badge-list "> + <li class="yt-badge-item" > + <span class="yt-badge">Nouveauté</span> + </li> + <li class="yt-badge-item" ><span class="yt-badge " >HD</span></li> + </ul> + </div> + <div class="yt-lockup-action-menu yt-uix-menu-container"> + <div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded" + data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu"> + </div> + </div> + </div> + </div> + </div> + </li> + </ol> + """ + response = mock.Mock(text=html) + results = youtube_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM') + self.assertEqual(results[0]['content'], 'Description') + self.assertEqual(results[0]['thumbnail'], 'https://i.ytimg.com/vi/DIVZCPfAOeM/hqdefault.jpg') + self.assertTrue('DIVZCPfAOeM' in results[0]['embedded']) + + html = """ + <ol id="item-section-063864" class="item-section"> + <li> + <div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile" + data-context-item-id="DIVZCPfAOeM" + data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB"> + <div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto"> + <a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link" + data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA"> + <div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg" + width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a> + <span class="thumb-menu dark-overflow-action-menu video-actions"> + </span> + </div> + <div class="yt-lockup-content"> + <h3 class="yt-lockup-title"> + <span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span> + </h3> + <div class="yt-lockup-byline">de + <a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard" + data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA" + data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta"> + <ul class="yt-lockup-meta-info"> + <li>il y a 20 heures</li> + <li>8 424 vues</li> + </ul> + </div> + <div class="yt-lockup-badges"> + <ul class="yt-badge-list "> + <li class="yt-badge-item" > + <span class="yt-badge">Nouveauté</span> + </li> + <li class="yt-badge-item" ><span class="yt-badge " >HD</span></li> + </ul> + </div> + <div class="yt-lockup-action-menu yt-uix-menu-container"> + <div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded" + data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu"> + </div> + </div> + </div> + </div> + </div> + </li> + </ol> + """ + response = mock.Mock(text=html) + results = youtube_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + + html = """ + <ol id="item-section-063864" class="item-section"> + <li> + </li> + </ol> + """ + response = mock.Mock(text=html) + results = youtube_noapi.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py index 5770458f3..dc062e95f 100644 --- a/searx/tests/test_engines.py +++ b/searx/tests/test_engines.py @@ -25,6 +25,7 @@ from searx.tests.engines.test_mixcloud import * # noqa from searx.tests.engines.test_openstreetmap import * # noqa from searx.tests.engines.test_photon import * # noqa from searx.tests.engines.test_piratebay import * # noqa +from searx.tests.engines.test_qwant import * # noqa from searx.tests.engines.test_searchcode_code import * # noqa from searx.tests.engines.test_searchcode_doc import * # noqa from searx.tests.engines.test_soundcloud import * # noqa @@ -32,6 +33,7 @@ from searx.tests.engines.test_spotify import * # noqa from searx.tests.engines.test_stackoverflow import * # noqa from searx.tests.engines.test_startpage import * # noqa from searx.tests.engines.test_subtitleseeker import * # noqa +from searx.tests.engines.test_swisscows import * # noqa from searx.tests.engines.test_twitter import * # noqa from searx.tests.engines.test_vimeo import * # noqa from searx.tests.engines.test_www1x import * # noqa @@ -39,4 +41,6 @@ from searx.tests.engines.test_www500px import * # noqa from searx.tests.engines.test_yacy import * # noqa from searx.tests.engines.test_yahoo import * # noqa from searx.tests.engines.test_youtube import * # noqa +from searx.tests.engines.test_youtube_api import * # noqa +from searx.tests.engines.test_youtube_noapi import * # noqa from searx.tests.engines.test_yahoo_news import * # noqa diff --git a/searx/tests/test_plugins.py b/searx/tests/test_plugins.py index 8dcad1142..c5171127c 100644 --- a/searx/tests/test_plugins.py +++ b/searx/tests/test_plugins.py @@ -38,10 +38,11 @@ class SelfIPTest(SearxTestCase): def test_PluginStore_init(self): store = plugins.PluginStore() - store.register(plugins.self_ip) + store.register(plugins.self_info) self.assertTrue(len(store.plugins) == 1) + # IP test request = Mock(user_plugins=store.plugins, remote_addr='127.0.0.1') request.headers.getlist.return_value = [] @@ -49,3 +50,19 @@ class SelfIPTest(SearxTestCase): query='ip')} store.call('post_search', request, ctx) self.assertTrue('127.0.0.1' in ctx['search'].answers) + + # User agent test + request = Mock(user_plugins=store.plugins, + user_agent='Mock') + request.headers.getlist.return_value = [] + ctx = {'search': Mock(answers=set(), + query='user-agent')} + store.call('post_search', request, ctx) + self.assertTrue('Mock' in ctx['search'].answers) + ctx = {'search': Mock(answers=set(), + query='user agent')} + store.call('post_search', request, ctx) + self.assertTrue('Mock' in ctx['search'].answers) + ctx = {'search': Mock(answers=set(), + query='What is my User-Agent?')} + store.call('post_search', request, ctx) diff --git a/searx/tests/test_search.py b/searx/tests/test_search.py new file mode 100644 index 000000000..89d0b620d --- /dev/null +++ b/searx/tests/test_search.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +from searx.search import score_results +from searx.testing import SearxTestCase + + +def fake_result(url='https://aa.bb/cc?dd=ee#ff', + title='aaa', + content='bbb', + engine='wikipedia'): + return {'url': url, + 'title': title, + 'content': content, + 'engine': engine} + + +class ScoreResultsTestCase(SearxTestCase): + + def test_empty(self): + self.assertEqual(score_results(dict()), []) + + def test_urlparse(self): + results = score_results(dict(a=[fake_result(url='https://aa.bb/cc?dd=ee#ff')])) + parsed_url = results[0]['parsed_url'] + self.assertEqual(parsed_url.query, 'dd=ee') diff --git a/searx/translations/pt/LC_MESSAGES/messages.mo b/searx/translations/pt/LC_MESSAGES/messages.mo Binary files differnew file mode 100644 index 000000000..4a9302b60 --- /dev/null +++ b/searx/translations/pt/LC_MESSAGES/messages.mo diff --git a/searx/translations/pt/LC_MESSAGES/messages.po b/searx/translations/pt/LC_MESSAGES/messages.po new file mode 100644 index 000000000..7c58ebcb7 --- /dev/null +++ b/searx/translations/pt/LC_MESSAGES/messages.po @@ -0,0 +1,574 @@ +# Portuguese translations for PROJECT. +# Copyright (C) 2014 ORGANIZATION +# This file is distributed under the same license as the PROJECT project. +# FIRST AUTHOR <EMAIL@ADDRESS>, 2014. +# +msgid "" +msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" +"PO-Revision-Date: 2014-01-30 15:22+0100\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: en <LL@li.org>\n" +"Plural-Forms: nplurals=2; plural=(n != 1)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 1.3\n" + +#: searx/webapp.py:100 +msgid "files" +msgstr "arquivos" + +#: searx/webapp.py:101 +msgid "general" +msgstr "geral" + +#: searx/webapp.py:102 +msgid "music" +msgstr "música" + +#: searx/webapp.py:103 +msgid "social media" +msgstr "mídias sociais" + +#: searx/webapp.py:104 +msgid "images" +msgstr "imagens" + +#: searx/webapp.py:105 +msgid "videos" +msgstr "vídeos" + +#: searx/webapp.py:106 +msgid "it" +msgstr "informática" + +#: searx/webapp.py:107 +msgid "news" +msgstr "notícias" + +#: searx/webapp.py:108 +msgid "map" +msgstr "mapa" + +#: searx/webapp.py:361 +msgid "{minutes} minute(s) ago" +msgstr "Há {minutes} minutos" + +#: searx/webapp.py:363 +msgid "{hours} hour(s), {minutes} minute(s) ago" +msgstr "Há {hours} e {minutes} minutos" + +#: searx/engines/__init__.py:182 +msgid "Page loads (sec)" +msgstr "Tempo de carregamento (seg)" + +#: searx/engines/__init__.py:186 +msgid "Number of results" +msgstr "Número de resultados" + +#: searx/engines/__init__.py:190 +msgid "Scores" +msgstr "Pontuações" + +#: searx/engines/__init__.py:194 +msgid "Scores per result" +msgstr "Pontuações por resultado" + +#: searx/engines/__init__.py:198 +msgid "Errors" +msgstr "Erros" + +#: searx/templates/courgette/index.html:9 +#: searx/templates/courgette/index.html:13 +#: searx/templates/courgette/results.html:5 +#: searx/templates/default/index.html:8 searx/templates/default/index.html:12 +#: searx/templates/oscar/navbar.html:7 searx/templates/oscar/navbar.html:35 +#: searx/templates/oscar/preferences.html:3 +msgid "preferences" +msgstr "preferências" + +#: searx/templates/courgette/index.html:11 +#: searx/templates/default/index.html:10 searx/templates/oscar/about.html:3 +#: searx/templates/oscar/navbar.html:8 searx/templates/oscar/navbar.html:34 +msgid "about" +msgstr "sobre" + +#: searx/templates/courgette/preferences.html:5 +#: searx/templates/default/preferences.html:5 +#: searx/templates/oscar/preferences.html:12 +msgid "Preferences" +msgstr "Preferências" + +#: searx/templates/courgette/preferences.html:9 +#: searx/templates/default/preferences.html:9 +#: searx/templates/oscar/preferences.html:34 +#: searx/templates/oscar/preferences.html:36 +msgid "Default categories" +msgstr "Categorias padrão" + +#: searx/templates/courgette/preferences.html:13 +#: searx/templates/default/preferences.html:14 +#: searx/templates/oscar/preferences.html:42 +msgid "Search language" +msgstr "Língua de pesquisa" + +#: searx/templates/courgette/preferences.html:16 +#: searx/templates/default/preferences.html:17 +#: searx/templates/oscar/preferences.html:46 +msgid "Automatic" +msgstr "Automático" + +#: searx/templates/courgette/preferences.html:24 +#: searx/templates/default/preferences.html:25 +#: searx/templates/oscar/preferences.html:53 +msgid "Interface language" +msgstr "Linguagem da interface" + +#: searx/templates/courgette/preferences.html:34 +#: searx/templates/default/preferences.html:35 +#: searx/templates/oscar/preferences.html:63 +msgid "Autocomplete" +msgstr "Autocompletar" + +#: searx/templates/courgette/preferences.html:45 +#: searx/templates/default/preferences.html:46 +#: searx/templates/oscar/preferences.html:74 +msgid "Image proxy" +msgstr "Proxy de imagens" + +#: searx/templates/courgette/preferences.html:48 +#: searx/templates/default/preferences.html:49 +#: searx/templates/oscar/preferences.html:78 +msgid "Enabled" +msgstr "Ativado" + +#: searx/templates/courgette/preferences.html:49 +#: searx/templates/default/preferences.html:50 +#: searx/templates/oscar/preferences.html:79 +msgid "Disabled" +msgstr "Desativado" + +#: searx/templates/courgette/preferences.html:54 +#: searx/templates/default/preferences.html:55 +#: searx/templates/oscar/preferences.html:83 +msgid "Method" +msgstr "Método" + +#: searx/templates/courgette/preferences.html:63 +#: searx/templates/default/preferences.html:64 +#: searx/templates/oscar/preferences.html:92 +msgid "SafeSearch" +msgstr "Pesquisa segura" + +#: searx/templates/courgette/preferences.html:66 +#: searx/templates/default/preferences.html:67 +#: searx/templates/oscar/preferences.html:96 +msgid "Strict" +msgstr "Estrito" + +#: searx/templates/courgette/preferences.html:67 +#: searx/templates/default/preferences.html:68 +#: searx/templates/oscar/preferences.html:97 +msgid "Moderate" +msgstr "Moderado" + +#: searx/templates/courgette/preferences.html:68 +#: searx/templates/default/preferences.html:69 +#: searx/templates/oscar/preferences.html:98 +msgid "None" +msgstr "Nenhum" + +#: searx/templates/courgette/preferences.html:73 +#: searx/templates/default/preferences.html:74 +#: searx/templates/oscar/preferences.html:102 +msgid "Themes" +msgstr "Temas" + +#: searx/templates/courgette/preferences.html:83 +msgid "Color" +msgstr "Cor" + +#: searx/templates/courgette/preferences.html:86 +msgid "Blue (default)" +msgstr "Azul (padrão)" + +#: searx/templates/courgette/preferences.html:87 +msgid "Violet" +msgstr "Violeta" + +#: searx/templates/courgette/preferences.html:88 +msgid "Green" +msgstr "Verde" + +#: searx/templates/courgette/preferences.html:89 +msgid "Cyan" +msgstr "Ciano" + +#: searx/templates/courgette/preferences.html:90 +msgid "Orange" +msgstr "Laranja" + +#: searx/templates/courgette/preferences.html:91 +msgid "Red" +msgstr "Vermelho" + +#: searx/templates/courgette/preferences.html:96 +#: searx/templates/default/preferences.html:84 +msgid "Currently used search engines" +msgstr "Motores de busca sendo usados atualmente" + +#: searx/templates/courgette/preferences.html:100 +#: searx/templates/default/preferences.html:88 +msgid "Engine name" +msgstr "Nome do motor" + +#: searx/templates/courgette/preferences.html:101 +#: searx/templates/default/preferences.html:89 +msgid "Category" +msgstr "Categoria" + +#: searx/templates/courgette/preferences.html:102 +#: searx/templates/courgette/preferences.html:113 +#: searx/templates/default/preferences.html:90 +#: searx/templates/default/preferences.html:101 +#: searx/templates/oscar/preferences.html:145 +msgid "Allow" +msgstr "Permitir" + +#: searx/templates/courgette/preferences.html:102 +#: searx/templates/courgette/preferences.html:114 +#: searx/templates/default/preferences.html:90 +#: searx/templates/default/preferences.html:102 +#: searx/templates/oscar/preferences.html:144 +msgid "Block" +msgstr "Bloquear" + +#: searx/templates/courgette/preferences.html:122 +#: searx/templates/default/preferences.html:110 +#: searx/templates/oscar/preferences.html:161 +msgid "" +"These settings are stored in your cookies, this allows us not to store " +"this data about you." +msgstr "Essas configurações são armazenadas em seus cookies, isto nos permite não armazenar dados sobre você." + +#: searx/templates/courgette/preferences.html:124 +#: searx/templates/default/preferences.html:112 +#: searx/templates/oscar/preferences.html:163 +msgid "" +"These cookies serve your sole convenience, we don't use these cookies to " +"track you." +msgstr "Esses cookies servem unicamente para sua conveniência, nós não usamos eles para te rastrear." + +#: searx/templates/courgette/preferences.html:127 +#: searx/templates/default/preferences.html:115 +#: searx/templates/oscar/preferences.html:166 +msgid "save" +msgstr "salvar" + +#: searx/templates/courgette/preferences.html:128 +#: searx/templates/default/preferences.html:116 +#: searx/templates/oscar/preferences.html:167 +msgid "back" +msgstr "voltar" + +#: searx/templates/courgette/results.html:12 +#: searx/templates/default/results.html:13 +#: searx/templates/oscar/results.html:110 +msgid "Search URL" +msgstr "Pesquisar URL" + +#: searx/templates/courgette/results.html:16 +#: searx/templates/default/results.html:17 +#: searx/templates/oscar/results.html:115 +msgid "Download results" +msgstr "Baixar resultados" + +#: searx/templates/courgette/results.html:34 +#: searx/templates/default/results.html:35 +msgid "Answers" +msgstr "Respostas" + +#: searx/templates/courgette/results.html:42 +#: searx/templates/default/results.html:43 +#: searx/templates/oscar/results.html:90 +msgid "Suggestions" +msgstr "Sugestões" + +#: searx/templates/courgette/results.html:70 +#: searx/templates/default/results.html:81 +#: searx/templates/oscar/results.html:51 searx/templates/oscar/results.html:63 +msgid "previous page" +msgstr "página anterior" + +#: searx/templates/courgette/results.html:81 +#: searx/templates/default/results.html:92 +#: searx/templates/oscar/results.html:44 searx/templates/oscar/results.html:71 +msgid "next page" +msgstr "próxima página" + +#: searx/templates/courgette/search.html:3 +#: searx/templates/default/search.html:3 searx/templates/oscar/search.html:4 +#: searx/templates/oscar/search_full.html:9 +msgid "Search for..." +msgstr "Pesquisar por..." + +#: searx/templates/courgette/stats.html:4 searx/templates/default/stats.html:4 +#: searx/templates/oscar/stats.html:5 +msgid "Engine stats" +msgstr "Estatísticas do motor de busca" + +#: searx/templates/courgette/result_templates/images.html:4 +#: searx/templates/default/result_templates/images.html:4 +msgid "original context" +msgstr "contexto original" + +#: searx/templates/courgette/result_templates/torrent.html:7 +#: searx/templates/default/result_templates/torrent.html:11 +#: searx/templates/oscar/result_templates/torrent.html:6 +msgid "Seeder" +msgstr "Seeder" + +#: searx/templates/courgette/result_templates/torrent.html:7 +#: searx/templates/default/result_templates/torrent.html:11 +#: searx/templates/oscar/result_templates/torrent.html:6 +msgid "Leecher" +msgstr "Leecher" + +#: searx/templates/courgette/result_templates/torrent.html:9 +#: searx/templates/default/result_templates/torrent.html:9 +#: searx/templates/oscar/macros.html:21 +msgid "magnet link" +msgstr "link magnético" + +#: searx/templates/courgette/result_templates/torrent.html:10 +#: searx/templates/default/result_templates/torrent.html:10 +#: searx/templates/oscar/macros.html:22 +msgid "torrent file" +msgstr "arquivo torrent" + +#: searx/templates/default/categories.html:8 +msgid "Click on the magnifier to perform search" +msgstr "Clique na lupa para realizar a busca" + +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "em cache" + +#: searx/templates/oscar/base.html:74 +msgid "Powered by" +msgstr "Criado por" + +#: searx/templates/oscar/base.html:74 +msgid "a privacy-respecting, hackable metasearch engine" +msgstr "Um metapesquisador hackeável que respeita a privacidade" + +#: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 +msgid "home" +msgstr "início" + +#: searx/templates/oscar/navbar.html:14 searx/templates/oscar/navbar.html:24 +msgid "Toggle navigation" +msgstr "Mudar navegação" + +#: searx/templates/oscar/preferences.html:17 +#: searx/templates/oscar/preferences.html:23 +msgid "General" +msgstr "Geral" + +#: searx/templates/oscar/preferences.html:18 +#: searx/templates/oscar/preferences.html:124 +msgid "Engines" +msgstr "Motores de busca" + +#: searx/templates/oscar/preferences.html:43 +msgid "What language do you prefer for search?" +msgstr "Que linguagem você prefere para a busca?" + +#: searx/templates/oscar/preferences.html:54 +msgid "Change the language of the layout" +msgstr "Mudar a linguagem da interface" + +#: searx/templates/oscar/preferences.html:64 +msgid "Find stuff as you type" +msgstr "Achar coisas enquanto você digita" + +#: searx/templates/oscar/preferences.html:75 +msgid "Proxying image results through searx" +msgstr "Filtrar resultados de imagens no searx" + +#: searx/templates/oscar/preferences.html:84 +msgid "" +"Change how forms are submited, <a " +"href=\"http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\"" +" rel=\"external\">learn more about request methods</a>" +msgstr "Muda o modo como formulários são enviados <a href=\"https://pt.wikipedia.org/wiki/Hypertext_Transfer_Protocol#M.C3.A9todosn\" rel=\"external\">mais informações sobre os modos de pedido</a>" + +#: searx/templates/oscar/preferences.html:93 +msgid "Filter content" +msgstr "Filtrar conteúdo" + +#: searx/templates/oscar/preferences.html:103 +msgid "Change searx layout" +msgstr "Mudar a interface do searx" + +#: searx/templates/oscar/results.html:7 +msgid "Search results" +msgstr "Pesquisar" + +#: searx/templates/oscar/results.html:105 +msgid "Links" +msgstr "Links" + +#: searx/templates/oscar/search.html:6 +#: searx/templates/oscar/search_full.html:11 +msgid "Start search" +msgstr "Começar pesquisa" + +#: searx/templates/oscar/search_full.html:15 +msgid "Show search filters" +msgstr "Mostrar filtros de pesquisa" + +#: searx/templates/oscar/search_full.html:15 +msgid "Hide search filters" +msgstr "Esconder filtros de pesquisa" + +#: searx/templates/oscar/stats.html:2 +msgid "stats" +msgstr "Estatísticas" + +#: searx/templates/oscar/messages/first_time.html:4 +#: searx/templates/oscar/messages/no_results.html:5 +#: searx/templates/oscar/messages/save_settings_successfull.html:5 +#: searx/templates/oscar/messages/unknow_error.html:5 +msgid "Close" +msgstr "Fechar" + +#: searx/templates/oscar/messages/first_time.html:6 +#: searx/templates/oscar/messages/no_data_available.html:3 +msgid "Heads up!" +msgstr "Atenção" + +#: searx/templates/oscar/messages/first_time.html:7 +msgid "It look like you are using searx first time." +msgstr "Parece que você está usando o seax pela primeira vez." + +#: searx/templates/oscar/messages/js_disabled.html:2 +msgid "Warning!" +msgstr "Atenção!" + +#: searx/templates/oscar/messages/js_disabled.html:3 +msgid "Please enable JavaScript to use full functionality of this site." +msgstr "Ative o Javascript para poder usar toda a funcionalidade deste site, por favor." + +#: searx/templates/oscar/messages/no_data_available.html:4 +msgid "There is currently no data available. " +msgstr "Não há dados disponíveis atualmente. " + +#: searx/templates/oscar/messages/no_results.html:7 +msgid "Sorry!" +msgstr "Desculpe!" + +#: searx/templates/oscar/messages/no_results.html:8 +msgid "" +"we didn't find any results. Please use another query or search in more " +"categories." +msgstr "Nós não achamos nenhum resultado. Reformule sua busca ou procure em outras categorias, por favor." + +#: searx/templates/oscar/messages/save_settings_successfull.html:7 +msgid "Well done!" +msgstr "Bem feito!" + +#: searx/templates/oscar/messages/save_settings_successfull.html:8 +msgid "Settings saved successfully." +msgstr "Opções salvadas com sucesso." + +#: searx/templates/oscar/messages/unknow_error.html:7 +msgid "Oh snap!" +msgstr "Droga!" + +#: searx/templates/oscar/messages/unknow_error.html:8 +msgid "Something went wrong." +msgstr "algo de errado aconteceu." + +#: searx/templates/oscar/result_templates/default.html:7 +msgid "show media" +msgstr "mostrar mídia" + +#: searx/templates/oscar/result_templates/default.html:7 +msgid "hide media" +msgstr "esconder mídia" + +#: searx/templates/oscar/result_templates/images.html:23 +msgid "Get image" +msgstr "oter imagem" + +#: searx/templates/oscar/result_templates/images.html:24 +msgid "View source" +msgstr "ver fonte" + +#: searx/templates/oscar/result_templates/map.html:7 +msgid "show map" +msgstr "mostrar mapa" + +#: searx/templates/oscar/result_templates/map.html:7 +msgid "hide map" +msgstr "esconder mapa" + +#: searx/templates/oscar/result_templates/map.html:11 +msgid "show details" +msgstr "mostrar detalhes" + +#: searx/templates/oscar/result_templates/map.html:11 +msgid "hide details" +msgstr "esconder detalhes" + +#: searx/templates/oscar/result_templates/torrent.html:7 +msgid "Filesize" +msgstr "Tamanho de arquivo" + +#: searx/templates/oscar/result_templates/torrent.html:9 +msgid "Bytes" +msgstr "Bytes" + +#: searx/templates/oscar/result_templates/torrent.html:10 +msgid "kiB" +msgstr "kiB" + +#: searx/templates/oscar/result_templates/torrent.html:11 +msgid "MiB" +msgstr "miB" + +#: searx/templates/oscar/result_templates/torrent.html:12 +msgid "GiB" +msgstr "GiB" + +#: searx/templates/oscar/result_templates/torrent.html:13 +msgid "TiB" +msgstr "TiB" + +#: searx/templates/oscar/result_templates/torrent.html:15 +msgid "Number of Files" +msgstr "Número de Arquivos" + +#: searx/templates/oscar/result_templates/videos.html:7 +msgid "show video" +msgstr "mostrar vídeo" + +#: searx/templates/oscar/result_templates/videos.html:7 +msgid "hide video" +msgstr "esconder vídeo" + +#~ msgid "Localization" +#~ msgstr "Localização" + +#~ msgid "Yes" +#~ msgstr "Sim" + +#~ msgid "No" +#~ msgstr "Não" diff --git a/searx/utils.py b/searx/utils.py index 129971e31..c9784159c 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -228,6 +228,14 @@ def prettify_url(url): return url +# get element in list or default value +def list_get(a_list, index, default=None): + if len(a_list) > index: + return a_list[index] + else: + return default + + def get_blocked_engines(engines, cookies): if 'blocked_engines' not in cookies: return [(engine_name, category) for engine_name in engines diff --git a/searx/webapp.py b/searx/webapp.py index 3ef5a72c8..fb7157b47 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -279,6 +279,12 @@ def render(template_name, override_theme=None, **kwargs): if x != 'general' and x in nonblocked_categories) + if 'all_categories' not in kwargs: + kwargs['all_categories'] = ['general'] + kwargs['all_categories'].extend(x for x in + sorted(categories.keys()) + if x != 'general') + if 'selected_categories' not in kwargs: kwargs['selected_categories'] = [] for arg in request.args: @@ -286,11 +292,13 @@ def render(template_name, override_theme=None, **kwargs): c = arg.split('_', 1)[1] if c in categories: kwargs['selected_categories'].append(c) + if not kwargs['selected_categories']: cookie_categories = request.cookies.get('categories', '').split(',') for ccateg in cookie_categories: if ccateg in categories: kwargs['selected_categories'].append(ccateg) + if not kwargs['selected_categories']: kwargs['selected_categories'] = ['general'] @@ -623,6 +631,24 @@ def preferences(): resp.set_cookie('theme', theme, max_age=cookie_max_age) return resp + + # stats for preferences page + stats = {} + + for c in categories: + for e in categories[c]: + stats[e.name] = {'time': None, + 'warn_timeout': False, + 'warn_time': False} + if e.timeout > settings['server']['request_timeout']: + stats[e.name]['warn_timeout'] = True + + for engine_stat in get_engines_stats()[0][1]: + stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3) + if engine_stat.get('avg') > settings['server']['request_timeout']: + stats[engine_stat.get('name')]['warn_time'] = True + # end of stats + return render('preferences.html', locales=settings['locales'], current_locale=get_locale(), @@ -630,6 +656,7 @@ def preferences(): image_proxy=image_proxy, language_codes=language_codes, engines_by_category=categories, + stats=stats, blocked_engines=blocked_engines, autocomplete_backends=autocomplete_backends, shortcuts={y: x for x, y in engine_shortcuts.items()}, @@ -670,7 +697,7 @@ def image_proxy(): return '', 400 if not resp.headers.get('content-type', '').startswith('image/'): - logger.debug('image-proxy: wrong content-type: {0}'.format(resp.get('content-type'))) + logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type'))) return '', 400 img = '' @@ -754,10 +781,45 @@ def run(): ) -application = app +class ReverseProxyPathFix(object): + '''Wrap the application in this middleware and configure the + front-end server to add these headers, to let you quietly bind + this to a URL other than / and to an HTTP scheme that is + different than what is used locally. + + http://flask.pocoo.org/snippets/35/ + + In nginx: + location /myprefix { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /myprefix; + } + + :param app: the WSGI application + ''' + def __init__(self, app): + self.app = app -app.wsgi_app = ProxyFix(application.wsgi_app) + def __call__(self, environ, start_response): + script_name = environ.get('HTTP_X_SCRIPT_NAME', '') + if script_name: + environ['SCRIPT_NAME'] = script_name + path_info = environ['PATH_INFO'] + if path_info.startswith(script_name): + environ['PATH_INFO'] = path_info[len(script_name):] + scheme = environ.get('HTTP_X_SCHEME', '') + if scheme: + environ['wsgi.url_scheme'] = scheme + return self.app(environ, start_response) + + +application = app +# patch app to handle non root url-s behind proxy & wsgi +app.wsgi_app = ReverseProxyPathFix(ProxyFix(application.wsgi_app)) if __name__ == "__main__": run() diff --git a/versions.cfg b/versions.cfg index c4cca4ad9..945217415 100644 --- a/versions.cfg +++ b/versions.cfg @@ -13,82 +13,90 @@ collective.recipe.omelette = 0.16 coverage = 3.7.1 decorator = 3.4.2 docutils = 0.12 -flake8 = 2.4.0 +flake8 = 2.4.1 itsdangerous = 0.24 -mccabe = 0.3 +mccabe = 0.3.1 mock = 1.0.1 pep8 = 1.5.7 plone.testing = 4.0.13 pyflakes = 0.8.1 -pytz = 2015.2 +pytz = 2015.4 pyyaml = 3.11 -requests = 2.6.2 +requests = 2.7.0 robotframework-debuglibrary = 0.3 robotframework-httplibrary = 0.4.2 -robotframework-selenium2library = 1.6.0 +robotframework-selenium2library = 1.7.1 robotsuite = 1.6.1 -selenium = 2.45.0 +selenium = 2.46.0 speaklater = 1.3 unittest2 = 1.0.1 waitress = 0.8.9 zc.recipe.testrunner = 2.0.0 pyopenssl = 0.15.1 -ndg-httpsclient = 0.3.3 -pyasn1 = 0.1.7 -pyasn1-modules = 0.0.5 +ndg-httpsclient = 0.4.0 +pyasn1 = 0.1.8 +pyasn1-modules = 0.0.6 certifi = 2015.04.28 # -cffi = 0.9.2 -cryptography = 0.8.2 +cffi = 1.1.2 +cryptography = 0.9.1 + +# Required by: +# robotsuite==1.6.1 +# searx==0.7.0 +lxml = 3.4.4 + +# Required by: +# searx==0.7.0 +python-dateutil = 2.4.2 + +# Required by: +# searx==0.7.0 +# zope.exceptions==4.0.7 +# zope.interface==4.1.2 +# zope.testrunner==4.4.9 +setuptools = 18.0.1 # Required by: # WebTest==2.0.18 beautifulsoup4 = 4.3.2 # Required by: -# cryptography==0.8.2 +# cryptography==0.9.1 enum34 = 1.0.4 # Required by: +# cryptography==0.9.1 +idna = 2.0 + +# Required by: +# cryptography==0.9.1 +ipaddress = 1.0.7 + +# Required by: # robotframework-httplibrary==0.4.2 -jsonpatch = 1.9 +jsonpatch = 1.11 # Required by: # robotframework-httplibrary==0.4.2 -jsonpointer = 1.7 +jsonpointer = 1.9 # Required by: # traceback2==1.4.0 linecache2 = 1.0.0 # Required by: -# robotsuite==1.6.1 -# searx==0.7.0 -lxml = 3.4.4 - -# Required by: -# cffi==0.9.2 +# cffi==1.1.2 pycparser = 2.12 # Required by: -# searx==0.7.0 -python-dateutil = 2.4.2 - -# Required by: # robotframework-httplibrary==0.4.2 robotframework = 2.8.7 # Required by: -# searx==0.7.0 -# zope.exceptions==4.0.7 -# zope.interface==4.1.2 -# zope.testrunner==4.4.8 -setuptools = 15.2 - -# Required by: # robotsuite==1.6.1 -# zope.testrunner==4.4.8 +# zope.testrunner==4.4.9 six = 1.9.0 # Required by: @@ -100,17 +108,17 @@ traceback2 = 1.4.0 zc.recipe.egg = 2.0.1 # Required by: -# zope.testrunner==4.4.8 +# zope.testrunner==4.4.9 zope.exceptions = 4.0.7 # Required by: -# zope.testrunner==4.4.8 +# zope.testrunner==4.4.9 zope.interface = 4.1.2 # Required by: # plone.testing==4.0.13 -zope.testing = 4.1.3 +zope.testing = 4.2.0 # Required by: # zc.recipe.testrunner==2.0.0 -zope.testrunner = 4.4.8 +zope.testrunner = 4.4.9 |
