diff options
94 files changed, 925 insertions, 330 deletions
diff --git a/searx/__init__.py b/searx/__init__.py index 110f46af8..2d545a809 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -36,11 +36,6 @@ if 'SEARX_SETTINGS_PATH' in environ: else: settings_path = join(searx_dir, 'settings.yml') -if 'SEARX_HTTPS_REWRITE_PATH' in environ: - https_rewrite_path = environ['SEARX_HTTPS_REWRITE_PATH'] -else: - https_rewrite_path = join(searx_dir, 'https_rules') - # load settings with open(settings_path) as settings_yaml: settings = load(settings_yaml) @@ -52,10 +47,4 @@ else: logger = logging.getLogger('searx') -# load https rules only if https rewrite is enabled -if settings.get('server', {}).get('https_rewrite'): - # loade https rules - from searx.https_rewrite import load_https_rules - load_https_rules(https_rewrite_path) - logger.info('Initialisation done') diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 9d31aa36f..83e204890 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -19,11 +19,19 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from lxml import etree from json import loads from urllib import urlencode +from searx import settings from searx.languages import language_codes from searx.engines import ( categories, engines, engine_shortcuts ) -from searx.poolrequests import get +from searx.poolrequests import get as http_get + + +def get(*args, **kwargs): + if not 'timeout' in kwargs: + kwargs['timeout'] = settings['server']['request_timeout'] + + return http_get(*args, **kwargs) def searx_bang(full_query): diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index 944250628..d0f05ec2d 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -29,6 +29,10 @@ def request(query, params): params['url'] = search_url.format(search_term=quote(query), pageno=params['pageno']-1) + # FIX: SSLError: hostname 'btdigg.org' + # doesn't match either of 'ssl2000.cloudflare.com', 'cloudflare.com', '*.cloudflare.com' + params['verify'] = False + return params diff --git a/searx/engines/deezer.py b/searx/engines/deezer.py index 433ceffa1..7fbd3c200 100644 --- a/searx/engines/deezer.py +++ b/searx/engines/deezer.py @@ -16,11 +16,11 @@ categories = ['music'] paging = True # search-url -url = 'http://api.deezer.com/' +url = 'https://api.deezer.com/' search_url = url + 'search?{query}&index={offset}' embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true" ' +\ - 'data-src="http://www.deezer.com/plugins/player?type=tracks&id={audioid}" ' +\ + 'data-src="https://www.deezer.com/plugins/player?type=tracks&id={audioid}" ' +\ 'width="540" height="80"></iframe>' @@ -45,6 +45,10 @@ def response(resp): if result['type'] == 'track': title = result['title'] url = result['link'] + + if url.startswith('http://'): + url = 'https' + url[4:] + content = result['artist']['name'] +\ " • " +\ result['album']['title'] +\ diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py index fa5c61128..adee6c1fd 100644 --- a/searx/engines/piratebay.py +++ b/searx/engines/piratebay.py @@ -1,4 +1,4 @@ -## Piratebay (Videos, Music, Files) +# Piratebay (Videos, Music, Files) # # @website https://thepiratebay.se # @provide-api no (nothing found) @@ -42,6 +42,10 @@ def request(query, params): search_type=search_type, pageno=params['pageno'] - 1) + # FIX: SSLError: hostname 'kthepiratebay.se' + # doesn't match either of 'ssl2000.cloudflare.com', 'cloudflare.com', '*.cloudflare.com' + params['verify'] = False + return params @@ -78,7 +82,11 @@ def response(resp): leech = 0 magnetlink = result.xpath(magnet_xpath)[0] - torrentfile = result.xpath(torrent_xpath)[0] + torrentfile_links = result.xpath(torrent_xpath) + if torrentfile_links: + torrentfile_link = torrentfile_links[0].attrib.get('href') + else: + torrentfile_link = None # append result results.append({'url': href, @@ -87,7 +95,7 @@ def response(resp): 'seed': seed, 'leech': leech, 'magnetlink': magnetlink.attrib.get('href'), - 'torrentfile': torrentfile.attrib.get('href'), + 'torrentfile': torrentfile_link, 'template': 'torrent.html'}) # return results sorted by seeder diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py new file mode 100644 index 000000000..61f3721ec --- /dev/null +++ b/searx/engines/spotify.py @@ -0,0 +1,60 @@ +## Spotify (Music) +# +# @website https://spotify.com +# @provide-api yes (https://developer.spotify.com/web-api/search-item/) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, content, embedded + +from json import loads +from urllib import urlencode + +# engine dependent config +categories = ['music'] +paging = True + +# search-url +url = 'https://api.spotify.com/' +search_url = url + 'v1/search?{query}&type=track&offset={offset}' + +embedded_url = '<iframe data-src="https://embed.spotify.com/?uri=spotify:track:{audioid}"\ + width="300" height="80" frameborder="0" allowtransparency="true"></iframe>' + + +# do search-request +def request(query, params): + offset = (params['pageno'] - 1) * 20 + + params['url'] = search_url.format(query=urlencode({'q': query}), + offset=offset) + + return params + + +# get response from search-request +def response(resp): + results = [] + + search_res = loads(resp.text) + + # parse results + for result in search_res.get('tracks', {}).get('items', {}): + if result['type'] == 'track': + title = result['name'] + url = result['external_urls']['spotify'] + content = result['artists'][0]['name'] +\ + " • " +\ + result['album']['name'] +\ + " • " + result['name'] + embedded = embedded_url.format(audioid=result['id']) + + # append result + results.append({'url': url, + 'title': title, + 'embedded': embedded, + 'content': content}) + + # return results + return results diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 161f7513b..11663a415 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -24,11 +24,11 @@ base_url = 'https://search.yahoo.com/' search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}' # specific xpath variables -results_xpath = '//div[@class="res"]' +results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]" url_xpath = './/h3/a/@href' title_xpath = './/h3/a' -content_xpath = './/div[@class="abstr"]' -suggestion_xpath = '//div[@id="satat"]//a' +content_xpath = './/div[@class="compText aAbs"]' +suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a" # remove yahoo-specific tracking-url @@ -91,11 +91,12 @@ def response(resp): 'content': content}) # if no suggestion found, return results - if not dom.xpath(suggestion_xpath): + suggestions = dom.xpath(suggestion_xpath) + if not suggestions: return results # parse suggestion - for suggestion in dom.xpath(suggestion_xpath): + for suggestion in suggestions: # append suggestion results.append({'suggestion': extract_text(suggestion)}) diff --git a/searx/plugins/__init__.py b/searx/plugins/__init__.py new file mode 100644 index 000000000..5ac3f447c --- /dev/null +++ b/searx/plugins/__init__.py @@ -0,0 +1,75 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2015 by Adam Tauber, <asciimoo@gmail.com> +''' +from sys import exit +from searx import logger + +logger = logger.getChild('plugins') + +from searx.plugins import (https_rewrite, + self_ip, + search_on_category_select) + +required_attrs = (('name', str), + ('description', str), + ('default_on', bool)) + +optional_attrs = (('js_dependencies', tuple), + ('css_dependencies', tuple)) + + +class Plugin(): + default_on = False + name = 'Default plugin' + description = 'Default plugin description' + + +class PluginStore(): + + def __init__(self): + self.plugins = [] + + def __iter__(self): + for plugin in self.plugins: + yield plugin + + def register(self, *plugins): + for plugin in plugins: + for plugin_attr, plugin_attr_type in required_attrs: + if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type): + logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin)) + exit(3) + for plugin_attr, plugin_attr_type in optional_attrs: + if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type): + setattr(plugin, plugin_attr, plugin_attr_type()) + plugin.id = plugin.name.replace(' ', '_') + self.plugins.append(plugin) + + def call(self, plugin_type, request, *args, **kwargs): + ret = True + for plugin in request.user_plugins: + if hasattr(plugin, plugin_type): + ret = getattr(plugin, plugin_type)(request, *args, **kwargs) + if not ret: + break + + return ret + + +plugins = PluginStore() +plugins.register(https_rewrite) +plugins.register(self_ip) +plugins.register(search_on_category_select) diff --git a/searx/https_rewrite.py b/searx/plugins/https_rewrite.py index 71aec1c9b..a24f15a28 100644 --- a/searx/https_rewrite.py +++ b/searx/plugins/https_rewrite.py @@ -18,11 +18,22 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. import re from urlparse import urlparse from lxml import etree -from os import listdir +from os import listdir, environ from os.path import isfile, isdir, join -from searx import logger +from searx.plugins import logger +from flask.ext.babel import gettext +from searx import searx_dir +name = "HTTPS rewrite" +description = gettext('Rewrite HTTP links to HTTPS if possible') +default_on = True + +if 'SEARX_HTTPS_REWRITE_PATH' in environ: + rules_path = environ['SEARX_rules_path'] +else: + rules_path = join(searx_dir, 'plugins/https_rules') + logger = logger.getChild("https_rewrite") # https://gitweb.torproject.org/\ @@ -33,7 +44,7 @@ https_rules = [] # load single ruleset from a xml file -def load_single_https_ruleset(filepath): +def load_single_https_ruleset(rules_path): ruleset = () # init parser @@ -41,7 +52,7 @@ def load_single_https_ruleset(filepath): # load and parse xml-file try: - tree = etree.parse(filepath, parser) + tree = etree.parse(rules_path, parser) except: # TODO, error message return () @@ -207,3 +218,13 @@ def https_url_rewrite(result): # target has matched, do not search over the other rules break return result + + +def on_result(request, ctx): + result = ctx['result'] + if result['parsed_url'].scheme == 'http': + https_url_rewrite(result) + return True + + +load_https_rules(rules_path) diff --git a/searx/https_rules/00README b/searx/plugins/https_rules/00README index fcd8a7724..fcd8a7724 100644 --- a/searx/https_rules/00README +++ b/searx/plugins/https_rules/00README diff --git a/searx/https_rules/Bing.xml b/searx/plugins/https_rules/Bing.xml index 8b403f108..8b403f108 100644 --- a/searx/https_rules/Bing.xml +++ b/searx/plugins/https_rules/Bing.xml diff --git a/searx/https_rules/Dailymotion.xml b/searx/plugins/https_rules/Dailymotion.xml index 743100cb7..743100cb7 100644 --- a/searx/https_rules/Dailymotion.xml +++ b/searx/plugins/https_rules/Dailymotion.xml diff --git a/searx/https_rules/Deviantart.xml b/searx/plugins/https_rules/Deviantart.xml index 7830fc20f..7830fc20f 100644 --- a/searx/https_rules/Deviantart.xml +++ b/searx/plugins/https_rules/Deviantart.xml diff --git a/searx/https_rules/DuckDuckGo.xml b/searx/plugins/https_rules/DuckDuckGo.xml index 173a9ad9f..173a9ad9f 100644 --- a/searx/https_rules/DuckDuckGo.xml +++ b/searx/plugins/https_rules/DuckDuckGo.xml diff --git a/searx/https_rules/Flickr.xml b/searx/plugins/https_rules/Flickr.xml index 85c6e8065..85c6e8065 100644 --- a/searx/https_rules/Flickr.xml +++ b/searx/plugins/https_rules/Flickr.xml diff --git a/searx/https_rules/Github-Pages.xml b/searx/plugins/https_rules/Github-Pages.xml index d3be58a4c..d3be58a4c 100644 --- a/searx/https_rules/Github-Pages.xml +++ b/searx/plugins/https_rules/Github-Pages.xml diff --git a/searx/https_rules/Github.xml b/searx/plugins/https_rules/Github.xml index a9a3a1e53..a9a3a1e53 100644 --- a/searx/https_rules/Github.xml +++ b/searx/plugins/https_rules/Github.xml diff --git a/searx/https_rules/Google-mismatches.xml b/searx/plugins/https_rules/Google-mismatches.xml index de9d3eb18..de9d3eb18 100644 --- a/searx/https_rules/Google-mismatches.xml +++ b/searx/plugins/https_rules/Google-mismatches.xml diff --git a/searx/https_rules/Google.org.xml b/searx/plugins/https_rules/Google.org.xml index d6cc47881..d6cc47881 100644 --- a/searx/https_rules/Google.org.xml +++ b/searx/plugins/https_rules/Google.org.xml diff --git a/searx/https_rules/GoogleAPIs.xml b/searx/plugins/https_rules/GoogleAPIs.xml index 85a5a8081..85a5a8081 100644 --- a/searx/https_rules/GoogleAPIs.xml +++ b/searx/plugins/https_rules/GoogleAPIs.xml diff --git a/searx/https_rules/GoogleCanada.xml b/searx/plugins/https_rules/GoogleCanada.xml index d5eefe816..d5eefe816 100644 --- a/searx/https_rules/GoogleCanada.xml +++ b/searx/plugins/https_rules/GoogleCanada.xml diff --git a/searx/https_rules/GoogleImages.xml b/searx/plugins/https_rules/GoogleImages.xml index 0112001e0..0112001e0 100644 --- a/searx/https_rules/GoogleImages.xml +++ b/searx/plugins/https_rules/GoogleImages.xml diff --git a/searx/https_rules/GoogleMainSearch.xml b/searx/plugins/https_rules/GoogleMainSearch.xml index df504d90c..df504d90c 100644 --- a/searx/https_rules/GoogleMainSearch.xml +++ b/searx/plugins/https_rules/GoogleMainSearch.xml diff --git a/searx/https_rules/GoogleMaps.xml b/searx/plugins/https_rules/GoogleMaps.xml index 0f82c5267..0f82c5267 100644 --- a/searx/https_rules/GoogleMaps.xml +++ b/searx/plugins/https_rules/GoogleMaps.xml diff --git a/searx/https_rules/GoogleMelange.xml b/searx/plugins/https_rules/GoogleMelange.xml index ec23cd45f..ec23cd45f 100644 --- a/searx/https_rules/GoogleMelange.xml +++ b/searx/plugins/https_rules/GoogleMelange.xml diff --git a/searx/https_rules/GoogleSearch.xml b/searx/plugins/https_rules/GoogleSearch.xml index 66b7ffdb0..66b7ffdb0 100644 --- a/searx/https_rules/GoogleSearch.xml +++ b/searx/plugins/https_rules/GoogleSearch.xml diff --git a/searx/https_rules/GoogleServices.xml b/searx/plugins/https_rules/GoogleServices.xml index 704646b53..704646b53 100644 --- a/searx/https_rules/GoogleServices.xml +++ b/searx/plugins/https_rules/GoogleServices.xml diff --git a/searx/https_rules/GoogleShopping.xml b/searx/plugins/https_rules/GoogleShopping.xml index 6ba69a91d..6ba69a91d 100644 --- a/searx/https_rules/GoogleShopping.xml +++ b/searx/plugins/https_rules/GoogleShopping.xml diff --git a/searx/https_rules/GoogleSorry.xml b/searx/plugins/https_rules/GoogleSorry.xml index 72a19210d..72a19210d 100644 --- a/searx/https_rules/GoogleSorry.xml +++ b/searx/plugins/https_rules/GoogleSorry.xml diff --git a/searx/https_rules/GoogleTranslate.xml b/searx/plugins/https_rules/GoogleTranslate.xml index a004025ae..a004025ae 100644 --- a/searx/https_rules/GoogleTranslate.xml +++ b/searx/plugins/https_rules/GoogleTranslate.xml diff --git a/searx/https_rules/GoogleVideos.xml b/searx/plugins/https_rules/GoogleVideos.xml index a5e88fcf0..a5e88fcf0 100644 --- a/searx/https_rules/GoogleVideos.xml +++ b/searx/plugins/https_rules/GoogleVideos.xml diff --git a/searx/https_rules/GoogleWatchBlog.xml b/searx/plugins/https_rules/GoogleWatchBlog.xml index afec70c97..afec70c97 100644 --- a/searx/https_rules/GoogleWatchBlog.xml +++ b/searx/plugins/https_rules/GoogleWatchBlog.xml diff --git a/searx/https_rules/Google_App_Engine.xml b/searx/plugins/https_rules/Google_App_Engine.xml index 851e051d1..851e051d1 100644 --- a/searx/https_rules/Google_App_Engine.xml +++ b/searx/plugins/https_rules/Google_App_Engine.xml diff --git a/searx/https_rules/Googleplex.com.xml b/searx/plugins/https_rules/Googleplex.com.xml index 7ddbb5ba9..7ddbb5ba9 100644 --- a/searx/https_rules/Googleplex.com.xml +++ b/searx/plugins/https_rules/Googleplex.com.xml diff --git a/searx/https_rules/OpenStreetMap.xml b/searx/plugins/https_rules/OpenStreetMap.xml index 58a661823..58a661823 100644 --- a/searx/https_rules/OpenStreetMap.xml +++ b/searx/plugins/https_rules/OpenStreetMap.xml diff --git a/searx/https_rules/Rawgithub.com.xml b/searx/plugins/https_rules/Rawgithub.com.xml index 3868f332a..3868f332a 100644 --- a/searx/https_rules/Rawgithub.com.xml +++ b/searx/plugins/https_rules/Rawgithub.com.xml diff --git a/searx/https_rules/Soundcloud.xml b/searx/plugins/https_rules/Soundcloud.xml index 6958e8cbc..6958e8cbc 100644 --- a/searx/https_rules/Soundcloud.xml +++ b/searx/plugins/https_rules/Soundcloud.xml diff --git a/searx/https_rules/ThePirateBay.xml b/searx/plugins/https_rules/ThePirateBay.xml index 010387b6b..010387b6b 100644 --- a/searx/https_rules/ThePirateBay.xml +++ b/searx/plugins/https_rules/ThePirateBay.xml diff --git a/searx/https_rules/Torproject.xml b/searx/plugins/https_rules/Torproject.xml index 69269af7e..69269af7e 100644 --- a/searx/https_rules/Torproject.xml +++ b/searx/plugins/https_rules/Torproject.xml diff --git a/searx/https_rules/Twitter.xml b/searx/plugins/https_rules/Twitter.xml index 3285f44e0..3285f44e0 100644 --- a/searx/https_rules/Twitter.xml +++ b/searx/plugins/https_rules/Twitter.xml diff --git a/searx/https_rules/Vimeo.xml b/searx/plugins/https_rules/Vimeo.xml index f2a3e5764..f2a3e5764 100644 --- a/searx/https_rules/Vimeo.xml +++ b/searx/plugins/https_rules/Vimeo.xml diff --git a/searx/https_rules/WikiLeaks.xml b/searx/plugins/https_rules/WikiLeaks.xml index 977709d2d..977709d2d 100644 --- a/searx/https_rules/WikiLeaks.xml +++ b/searx/plugins/https_rules/WikiLeaks.xml diff --git a/searx/https_rules/Wikimedia.xml b/searx/plugins/https_rules/Wikimedia.xml index 9f25831a2..9f25831a2 100644 --- a/searx/https_rules/Wikimedia.xml +++ b/searx/plugins/https_rules/Wikimedia.xml diff --git a/searx/https_rules/Yahoo.xml b/searx/plugins/https_rules/Yahoo.xml index 33548c4ab..33548c4ab 100644 --- a/searx/https_rules/Yahoo.xml +++ b/searx/plugins/https_rules/Yahoo.xml diff --git a/searx/https_rules/YouTube.xml b/searx/plugins/https_rules/YouTube.xml index bddc2a5f3..bddc2a5f3 100644 --- a/searx/https_rules/YouTube.xml +++ b/searx/plugins/https_rules/YouTube.xml diff --git a/searx/plugins/search_on_category_select.py b/searx/plugins/search_on_category_select.py new file mode 100644 index 000000000..7d124cc45 --- /dev/null +++ b/searx/plugins/search_on_category_select.py @@ -0,0 +1,22 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2015 by Adam Tauber, <asciimoo@gmail.com> +''' +from flask.ext.babel import gettext +name = 'Search on category select' +description = gettext('Perform search immediately if a category selected') +default_on = False + +js_dependencies = ('js/search_on_category_select.js',) diff --git a/searx/plugins/self_ip.py b/searx/plugins/self_ip.py new file mode 100644 index 000000000..5184ea4cf --- /dev/null +++ b/searx/plugins/self_ip.py @@ -0,0 +1,35 @@ +''' +searx is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +searx is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with searx. If not, see < http://www.gnu.org/licenses/ >. + +(C) 2015 by Adam Tauber, <asciimoo@gmail.com> +''' +from flask.ext.babel import gettext +name = "Self IP" +description = gettext('Display your source IP address if the query expression is "ip"') +default_on = True + + +# attach callback to the post search hook +# request: flask request object +# ctx: the whole local context of the pre search hook +def post_search(request, ctx): + if ctx['search'].query == 'ip': + x_forwarded_for = request.headers.getlist("X-Forwarded-For") + if x_forwarded_for: + ip = x_forwarded_for[0] + else: + ip = request.remote_addr + ctx['search'].answers.clear() + ctx['search'].answers.add(ip) + return True diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 65853c2e9..b74d43a02 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -1,20 +1,63 @@ import requests +from itertools import cycle +from searx import settings -the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100) -the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100) +class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter): + + def __init__(self, pool_connections=requests.adapters.DEFAULT_POOLSIZE, + pool_maxsize=requests.adapters.DEFAULT_POOLSIZE, + max_retries=requests.adapters.DEFAULT_RETRIES, + pool_block=requests.adapters.DEFAULT_POOLBLOCK, + **conn_params): + if max_retries == requests.adapters.DEFAULT_RETRIES: + self.max_retries = requests.adapters.Retry(0, read=False) + else: + self.max_retries = requests.adapters.Retry.from_int(max_retries) + self.config = {} + self.proxy_manager = {} + + super(requests.adapters.HTTPAdapter, self).__init__() + + self._pool_connections = pool_connections + self._pool_maxsize = pool_maxsize + self._pool_block = pool_block + self._conn_params = conn_params + + self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block, **conn_params) + + def __setstate__(self, state): + # Can't handle by adding 'proxy_manager' to self.__attrs__ because + # because self.poolmanager uses a lambda function, which isn't pickleable. + self.proxy_manager = {} + self.config = {} + + for attr, value in state.items(): + setattr(self, attr, value) + + self.init_poolmanager(self._pool_connections, self._pool_maxsize, + block=self._pool_block, **self._conn_params) + + +if settings.get('source_ips'): + http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + for source_ip in settings['source_ips']) + https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0)) + for source_ip in settings['source_ips']) +else: + http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) + https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), )) class SessionSinglePool(requests.Session): def __init__(self): - global the_https_adapter, the_http_adapter super(SessionSinglePool, self).__init__() # reuse the same adapters self.adapters.clear() - self.mount('https://', the_https_adapter) - self.mount('http://', the_http_adapter) + self.mount('https://', next(https_adapters)) + self.mount('http://', next(http_adapters)) def close(self): """Call super, but clear adapters since there are managed globaly""" diff --git a/searx/search.py b/searx/search.py index 476b92197..862b17e33 100644 --- a/searx/search.py +++ b/searx/search.py @@ -90,6 +90,7 @@ def make_callback(engine_name, results_queue, callback, params): # check if redirect comparing to the True value, # because resp can be a Mock object, and any attribut name returns something. if response.is_redirect is True: + logger.debug('{0} redirect on: {1}'.format(engine_name, response)) return response.search_params = params @@ -328,8 +329,8 @@ class Search(object): self.blocked_engines = get_blocked_engines(engines, request.cookies) self.results = [] - self.suggestions = [] - self.answers = [] + self.suggestions = set() + self.answers = set() self.infoboxes = [] self.request_data = {} @@ -428,9 +429,6 @@ class Search(object): requests = [] results_queue = Queue() results = {} - suggestions = set() - answers = set() - infoboxes = [] # increase number of searches number_of_searches += 1 @@ -510,7 +508,7 @@ class Search(object): selected_engine['name'])) if not requests: - return results, suggestions, answers, infoboxes + return self # send all search-request threaded_requests(requests) @@ -518,19 +516,19 @@ class Search(object): engine_name, engine_results = results_queue.get_nowait() # TODO type checks - [suggestions.add(x['suggestion']) + [self.suggestions.add(x['suggestion']) for x in list(engine_results) if 'suggestion' in x and engine_results.remove(x) is None] - [answers.add(x['answer']) + [self.answers.add(x['answer']) for x in list(engine_results) if 'answer' in x and engine_results.remove(x) is None] - infoboxes.extend(x for x in list(engine_results) - if 'infobox' in x - and engine_results.remove(x) is None) + self.infoboxes.extend(x for x in list(engine_results) + if 'infobox' in x + and engine_results.remove(x) is None) results[engine_name] = engine_results @@ -540,16 +538,16 @@ class Search(object): engines[engine_name].stats['result_count'] += len(engine_results) # score results and remove duplications - results = score_results(results) + self.results = score_results(results) # merge infoboxes according to their ids - infoboxes = merge_infoboxes(infoboxes) + self.infoboxes = merge_infoboxes(self.infoboxes) # update engine stats, using calculated score - for result in results: + for result in self.results: for res_engine in result['engines']: engines[result['engine']]\ .stats['score_count'] += result['score'] # return results, suggestions, answers and infoboxes - return results, suggestions, answers, infoboxes + return self diff --git a/searx/settings.yml b/searx/settings.yml index 8e2833ef0..f37c56b26 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -6,11 +6,16 @@ server: base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/" themes_path : "" # Custom ui themes path - leave it blank if you didn't change default_theme : oscar # ui theme - https_rewrite : True # Force rewrite result urls. See searx/https_rewrite.py useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator image_proxy : False # Proxying image results through searx default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section +# uncomment below section only if you have more than one network interface +# which can be the source of outgoing search requests +#source_ips: +# - 1.1.1.1 +# - 1.1.1.2 + engines: - name : wikipedia engine : mediawiki @@ -33,11 +38,6 @@ engines: locale : en-US shortcut : bin - - name : blekko images - engine : blekko_images - locale : en-US - shortcut : bli - - name : btdigg engine : btdigg shortcut : bt @@ -78,12 +78,6 @@ engines: # shortcut : fa # api_key : 'apikey' # required! -# down - website is under criminal investigation by the UK -# - name : filecrop -# engine : filecrop -# categories : files -# shortcut : fc - - name : 500px engine : www500px shortcut : px @@ -103,14 +97,10 @@ engines: # Or you can use the html non-stable engine, activated by default engine : flickr_noapi - - name : general-file - engine : generalfile - shortcut : gf - disabled : True - - name : gigablast engine : gigablast shortcut : gb + disabled: True - name : github engine : github @@ -195,6 +185,10 @@ engines: shortcut : scc disabled : True + - name : spotify + engine : spotify + shortcut : stf + - name : subtitleseeker engine : subtitleseeker shortcut : ss @@ -253,6 +247,12 @@ engines: locale : en-US shortcut : vm +#The blekko technology and team have joined IBM Watson! -> https://blekko.com/ +# - name : blekko images +# engine : blekko_images +# locale : en-US +# shortcut : bli + # - name : yacy # engine : yacy # shortcut : ya diff --git a/searx/static/js/search_on_category_select.js b/searx/static/js/search_on_category_select.js new file mode 100644 index 000000000..6156ca4e8 --- /dev/null +++ b/searx/static/js/search_on_category_select.js @@ -0,0 +1,14 @@ +$(document).ready(function() { + if($('#q')) { + $('#categories label').click(function(e) { + $('#categories input[type="checkbox"]').each(function(i, checkbox) { + $(checkbox).prop('checked', false); + }); + $('#categories label').removeClass('btn-primary').removeClass('active').addClass('btn-default'); + $(this).removeClass('btn-default').addClass('btn-primary').addClass('active'); + $($(this).children()[0]).prop('checked', 'checked'); + $('#search_form').submit(); + return false; + }); + } +}); diff --git a/searx/static/themes/courgette/js/searx.js b/searx/static/themes/courgette/js/searx.js index 47dc722da..92a25e349 100644 --- a/searx/static/themes/courgette/js/searx.js +++ b/searx/static/themes/courgette/js/searx.js @@ -1,6 +1,6 @@ if(searx.autocompleter) { window.addEvent('domready', function() { - new Autocompleter.Request.JSON('q', '/autocompleter', { + new Autocompleter.Request.JSON('q', './autocompleter', { postVar:'q', postData:{ 'format': 'json' diff --git a/searx/static/themes/default/js/searx.js b/searx/static/themes/default/js/searx.js index 9be969bb3..d6d5b74bb 100644 --- a/searx/static/themes/default/js/searx.js +++ b/searx/static/themes/default/js/searx.js @@ -1,6 +1,6 @@ if(searx.autocompleter) { window.addEvent('domready', function() { - new Autocompleter.Request.JSON('q', '/autocompleter', { + new Autocompleter.Request.JSON('q', './autocompleter', { postVar:'q', postData:{ 'format': 'json' diff --git a/searx/static/themes/oscar/js/searx.min.js b/searx/static/themes/oscar/js/searx.min.js index 2015c4d6a..8a1055da3 100644 --- a/searx/static/themes/oscar/js/searx.min.js +++ b/searx/static/themes/oscar/js/searx.min.js @@ -1,2 +1,2 @@ /*! oscar/searx.min.js | 09-01-2015 | https://github.com/asciimoo/searx */ -requirejs.config({baseUrl:"./static/themes/oscar/js",paths:{app:"../app"}}),searx.autocompleter&&(searx.searchResults=new Bloodhound({datumTokenizer:Bloodhound.tokenizers.obj.whitespace("value"),queryTokenizer:Bloodhound.tokenizers.whitespace,remote:"/autocompleter?q=%QUERY"}),searx.searchResults.initialize()),$(document).ready(function(){searx.autocompleter&&$("#q").typeahead(null,{name:"search-results",displayKey:function(a){return a},source:searx.searchResults.ttAdapter()})}),$(document).ready(function(){$("#q.autofocus").focus(),$(".select-all-on-click").click(function(){$(this).select()}),$(".btn-collapse").click(function(){var a=$(this).data("btn-text-collapsed"),b=$(this).data("btn-text-not-collapsed");""!==a&&""!==b&&(new_html=$(this).hasClass("collapsed")?$(this).html().replace(a,b):$(this).html().replace(b,a),$(this).html(new_html))}),$(".btn-toggle .btn").click(function(){var a="btn-"+$(this).data("btn-class"),b=$(this).data("btn-label-default"),c=$(this).data("btn-label-toggled");""!==c&&(new_html=$(this).hasClass("btn-default")?$(this).html().replace(b,c):$(this).html().replace(c,b),$(this).html(new_html)),$(this).toggleClass(a),$(this).toggleClass("btn-default")}),$(".media-loader").click(function(){var a=$(this).data("target"),b=$(a+" > iframe"),c=b.attr("src");(void 0===c||c===!1)&&b.attr("src",b.data("src"))}),$(".btn-sm").dblclick(function(){var a="btn-"+$(this).data("btn-class");$(this).hasClass("btn-default")?($(".btn-sm > input").attr("checked","checked"),$(".btn-sm > input").prop("checked",!0),$(".btn-sm").addClass(a),$(".btn-sm").addClass("active"),$(".btn-sm").removeClass("btn-default")):($(".btn-sm > input").attr("checked",""),$(".btn-sm > input").removeAttr("checked"),$(".btn-sm > input").checked=!1,$(".btn-sm").removeClass(a),$(".btn-sm").removeClass("active"),$(".btn-sm").addClass("btn-default"))})}),$(document).ready(function(){$(".searx_overpass_request").on("click",function(a){var b="https://overpass-api.de/api/interpreter?data=",c=b+"[out:json][timeout:25];(",d=");out meta;",e=$(this).data("osm-id"),f=$(this).data("osm-type"),g=$(this).data("result-table"),h="#"+$(this).data("result-table-loadicon"),i=["addr:city","addr:country","addr:housenumber","addr:postcode","addr:street"];if(e&&f&&g){g="#"+g;var j=null;switch(f){case"node":j=c+"node("+e+");"+d;break;case"way":j=c+"way("+e+");"+d;break;case"relation":j=c+"relation("+e+");"+d}if(j){$.ajax(j).done(function(a){if(a&&a.elements&&a.elements[0]){var b=a.elements[0],c=$(g).html();for(var d in b.tags)if(null===b.tags.name||-1==i.indexOf(d)){switch(c+="<tr><td>"+d+"</td><td>",d){case"phone":case"fax":c+='<a href="tel:'+b.tags[d].replace(/ /g,"")+'">'+b.tags[d]+"</a>";break;case"email":c+='<a href="mailto:'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"website":case"url":c+='<a href="'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikidata":c+='<a href="https://www.wikidata.org/wiki/'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikipedia":if(-1!=b.tags[d].indexOf(":")){c+='<a href="https://'+b.tags[d].substring(0,b.tags[d].indexOf(":"))+".wikipedia.org/wiki/"+b.tags[d].substring(b.tags[d].indexOf(":")+1)+'">'+b.tags[d]+"</a>";break}default:c+=b.tags[d]}c+="</td></tr>"}$(g).html(c),$(g).removeClass("hidden"),$(h).addClass("hidden")}}).fail(function(){$(h).html($(h).html()+'<p class="text-muted">could not load data!</p>')})}}$(this).off(a)}),$(".searx_init_map").on("click",function(a){var b=$(this).data("leaflet-target"),c=$(this).data("map-lon"),d=$(this).data("map-lat"),e=$(this).data("map-zoom"),f=$(this).data("map-boundingbox"),g=$(this).data("map-geojson");require(["leaflet-0.7.3.min"],function(){f&&(southWest=L.latLng(f[0],f[2]),northEast=L.latLng(f[1],f[3]),map_bounds=L.latLngBounds(southWest,northEast)),L.Icon.Default.imagePath="./static/themes/oscar/img/map";{var a=L.map(b),h="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",i='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors',j=new L.TileLayer(h,{minZoom:1,maxZoom:19,attribution:i}),k="http://otile{s}.mqcdn.com/tiles/1.0.0/map/{z}/{x}/{y}.jpg",l='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="http://developer.mapquest.com/content/osm/mq_logo.png">',m=new L.TileLayer(k,{minZoom:1,maxZoom:18,subdomains:"1234",attribution:l}),n="http://otile{s}.mqcdn.com/tiles/1.0.0/sat/{z}/{x}/{y}.jpg",o='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="https://developer.mapquest.com/content/osm/mq_logo.png"> | Portions Courtesy NASA/JPL-Caltech and U.S. Depart. of Agriculture, Farm Service Agency';new L.TileLayer(n,{minZoom:1,maxZoom:11,subdomains:"1234",attribution:o})}map_bounds?setTimeout(function(){a.fitBounds(map_bounds,{maxZoom:17})},0):c&&d&&(e?a.setView(new L.LatLng(d,c),e):a.setView(new L.LatLng(d,c),8)),a.addLayer(m);var p={"OSM Mapnik":j,MapQuest:m};L.control.layers(p).addTo(a),g&&L.geoJson(g).addTo(a)}),$(this).off(a)})});
\ No newline at end of file +requirejs.config({baseUrl:"./static/themes/oscar/js",paths:{app:"../app"}}),searx.autocompleter&&(searx.searchResults=new Bloodhound({datumTokenizer:Bloodhound.tokenizers.obj.whitespace("value"),queryTokenizer:Bloodhound.tokenizers.whitespace,remote:"./autocompleter?q=%QUERY"}),searx.searchResults.initialize()),$(document).ready(function(){searx.autocompleter&&$("#q").typeahead(null,{name:"search-results",displayKey:function(a){return a},source:searx.searchResults.ttAdapter()})}),$(document).ready(function(){$("#q.autofocus").focus(),$(".select-all-on-click").click(function(){$(this).select()}),$(".btn-collapse").click(function(){var a=$(this).data("btn-text-collapsed"),b=$(this).data("btn-text-not-collapsed");""!==a&&""!==b&&(new_html=$(this).hasClass("collapsed")?$(this).html().replace(a,b):$(this).html().replace(b,a),$(this).html(new_html))}),$(".btn-toggle .btn").click(function(){var a="btn-"+$(this).data("btn-class"),b=$(this).data("btn-label-default"),c=$(this).data("btn-label-toggled");""!==c&&(new_html=$(this).hasClass("btn-default")?$(this).html().replace(b,c):$(this).html().replace(c,b),$(this).html(new_html)),$(this).toggleClass(a),$(this).toggleClass("btn-default")}),$(".media-loader").click(function(){var a=$(this).data("target"),b=$(a+" > iframe"),c=b.attr("src");(void 0===c||c===!1)&&b.attr("src",b.data("src"))}),$(".btn-sm").dblclick(function(){var a="btn-"+$(this).data("btn-class");$(this).hasClass("btn-default")?($(".btn-sm > input").attr("checked","checked"),$(".btn-sm > input").prop("checked",!0),$(".btn-sm").addClass(a),$(".btn-sm").addClass("active"),$(".btn-sm").removeClass("btn-default")):($(".btn-sm > input").attr("checked",""),$(".btn-sm > input").removeAttr("checked"),$(".btn-sm > input").checked=!1,$(".btn-sm").removeClass(a),$(".btn-sm").removeClass("active"),$(".btn-sm").addClass("btn-default"))})}),$(document).ready(function(){$(".searx_overpass_request").on("click",function(a){var b="https://overpass-api.de/api/interpreter?data=",c=b+"[out:json][timeout:25];(",d=");out meta;",e=$(this).data("osm-id"),f=$(this).data("osm-type"),g=$(this).data("result-table"),h="#"+$(this).data("result-table-loadicon"),i=["addr:city","addr:country","addr:housenumber","addr:postcode","addr:street"];if(e&&f&&g){g="#"+g;var j=null;switch(f){case"node":j=c+"node("+e+");"+d;break;case"way":j=c+"way("+e+");"+d;break;case"relation":j=c+"relation("+e+");"+d}if(j){$.ajax(j).done(function(a){if(a&&a.elements&&a.elements[0]){var b=a.elements[0],c=$(g).html();for(var d in b.tags)if(null===b.tags.name||-1==i.indexOf(d)){switch(c+="<tr><td>"+d+"</td><td>",d){case"phone":case"fax":c+='<a href="tel:'+b.tags[d].replace(/ /g,"")+'">'+b.tags[d]+"</a>";break;case"email":c+='<a href="mailto:'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"website":case"url":c+='<a href="'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikidata":c+='<a href="https://www.wikidata.org/wiki/'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikipedia":if(-1!=b.tags[d].indexOf(":")){c+='<a href="https://'+b.tags[d].substring(0,b.tags[d].indexOf(":"))+".wikipedia.org/wiki/"+b.tags[d].substring(b.tags[d].indexOf(":")+1)+'">'+b.tags[d]+"</a>";break}default:c+=b.tags[d]}c+="</td></tr>"}$(g).html(c),$(g).removeClass("hidden"),$(h).addClass("hidden")}}).fail(function(){$(h).html($(h).html()+'<p class="text-muted">could not load data!</p>')})}}$(this).off(a)}),$(".searx_init_map").on("click",function(a){var b=$(this).data("leaflet-target"),c=$(this).data("map-lon"),d=$(this).data("map-lat"),e=$(this).data("map-zoom"),f=$(this).data("map-boundingbox"),g=$(this).data("map-geojson");require(["leaflet-0.7.3.min"],function(){f&&(southWest=L.latLng(f[0],f[2]),northEast=L.latLng(f[1],f[3]),map_bounds=L.latLngBounds(southWest,northEast)),L.Icon.Default.imagePath="./static/themes/oscar/img/map";{var a=L.map(b),h="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",i='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors',j=new L.TileLayer(h,{minZoom:1,maxZoom:19,attribution:i}),k="http://otile{s}.mqcdn.com/tiles/1.0.0/map/{z}/{x}/{y}.jpg",l='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="http://developer.mapquest.com/content/osm/mq_logo.png">',m=new L.TileLayer(k,{minZoom:1,maxZoom:18,subdomains:"1234",attribution:l}),n="http://otile{s}.mqcdn.com/tiles/1.0.0/sat/{z}/{x}/{y}.jpg",o='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="https://developer.mapquest.com/content/osm/mq_logo.png"> | Portions Courtesy NASA/JPL-Caltech and U.S. Depart. of Agriculture, Farm Service Agency';new L.TileLayer(n,{minZoom:1,maxZoom:11,subdomains:"1234",attribution:o})}map_bounds?setTimeout(function(){a.fitBounds(map_bounds,{maxZoom:17})},0):c&&d&&(e?a.setView(new L.LatLng(d,c),e):a.setView(new L.LatLng(d,c),8)),a.addLayer(m);var p={"OSM Mapnik":j,MapQuest:m};L.control.layers(p).addTo(a),g&&L.geoJson(g).addTo(a)}),$(this).off(a)})});
\ No newline at end of file diff --git a/searx/static/themes/oscar/js/searx_src/autocompleter.js b/searx/static/themes/oscar/js/searx_src/autocompleter.js index 561bff35a..70c66d2fc 100644 --- a/searx/static/themes/oscar/js/searx_src/autocompleter.js +++ b/searx/static/themes/oscar/js/searx_src/autocompleter.js @@ -19,7 +19,7 @@ if(searx.autocompleter) { searx.searchResults = new Bloodhound({
datumTokenizer: Bloodhound.tokenizers.obj.whitespace('value'),
queryTokenizer: Bloodhound.tokenizers.whitespace,
- remote: '/autocompleter?q=%QUERY'
+ remote: './autocompleter?q=%QUERY'
});
searx.searchResults.initialize();
}
diff --git a/searx/templates/courgette/preferences.html b/searx/templates/courgette/preferences.html index 2abfafb13..2afb74d11 100644 --- a/searx/templates/courgette/preferences.html +++ b/searx/templates/courgette/preferences.html @@ -101,8 +101,8 @@ <th>{{ _('Category') }}</th> <th>{{ _('Allow') }} / {{ _('Block') }}</th> </tr> - {% for (categ,search_engines) in categs %} - {% for search_engine in search_engines %} + {% for categ in categories %} + {% for search_engine in engines_by_category[categ] %} {% if not search_engine.private %} <tr> @@ -125,7 +125,8 @@ </p> <input type="submit" value="{{ _('save') }}" /> + <div class="right preferences_back"><a href="{{ url_for('clear_cookies') }}">{{ _('Reset defaults') }}</a></div> <div class="right preferences_back"><a href="{{ url_for('index') }}">{{ _('back') }}</a></div> - </form> + </form> </div> {% endblock %} diff --git a/searx/templates/default/preferences.html b/searx/templates/default/preferences.html index e03c18e3f..0afe9f7d0 100644 --- a/searx/templates/default/preferences.html +++ b/searx/templates/default/preferences.html @@ -89,8 +89,8 @@ <th>{{ _('Category') }}</th> <th>{{ _('Allow') }} / {{ _('Block') }}</th> </tr> - {% for (categ,search_engines) in categs %} - {% for search_engine in search_engines %} + {% for categ in categories %} + {% for search_engine in engines_by_category[categ] %} {% if not search_engine.private %} <tr> @@ -113,7 +113,8 @@ </p> <input type="submit" value="{{ _('save') }}" /> + <div class="{% if rtl %}left{% else %}right{% endif %} preferences_back"><a href="{{ url_for('clear_cookies') }}">{{ _('Reset defaults') }}</a></div> <div class="{% if rtl %}left{% else %}right{% endif %} preferences_back"><a href="{{ url_for('index') }}">{{ _('back') }}</a></div> - </form> + </form> </div> {% endblock %} diff --git a/searx/templates/oscar/base.html b/searx/templates/oscar/base.html index df5c53965..88eedc994 100644 --- a/searx/templates/oscar/base.html +++ b/searx/templates/oscar/base.html @@ -9,17 +9,20 @@ <meta name="viewport" content="width=device-width, initial-scale=1 , maximum-scale=1.0, user-scalable=1" /> {% block meta %}{% endblock %} <title>{% block title %}{% endblock %}searx</title> - + <link rel="stylesheet" href="{{ url_for('static', filename='css/bootstrap.min.css') }}" type="text/css" /> - <link rel="stylesheet" href="{{ url_for('static', filename='css/oscar.min.css') }}" type="text/css" /> + <link rel="stylesheet" href="{{ url_for('static', filename='css/oscar.min.css') }}" type="text/css" /> <link rel="stylesheet" href="{{ url_for('static', filename='css/leaflet.min.css') }}" type="text/css" /> + {% for css in styles %} + <link rel="stylesheet" href="{{ url_for('static', filename=css) }}" type="text/css" /> + {% endfor %} <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries --> <!--[if lt IE 9]> <script src="{{ url_for('static', filename='js/html5shiv.min.js') }}"></script> <script src="{{ url_for('static', filename='js/respond.min.js') }}"></script> <![endif]--> - + <link rel="shortcut icon" href="{{ url_for('static', filename='img/favicon.png') }}" /> {% block styles %} @@ -28,7 +31,7 @@ {% endblock %} <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/> - + <script type="text/javascript"> searx = {}; searx.method = "{{ method or 'POST' }}"; @@ -46,7 +49,7 @@ <body> <div class="container"> {% include 'oscar/navbar.html' %} - + {% block site_alert_error %} {% endblock %} {% block site_alert_warning %} @@ -62,7 +65,7 @@ {% endblock %} {% block site_alert_success %} {% endblock %} - + {% block content %} {% endblock %} @@ -79,5 +82,8 @@ {% if autocomplete %}<script src="{{ url_for('static', filename='js/typeahead.bundle.min.js') }}"></script>{% endif %} <script src="{{ url_for('static', filename='js/require-2.1.15.min.js') }}"></script> <script src="{{ url_for('static', filename='js/searx.min.js') }}"></script> + {% for script in scripts %} + <script src="{{ url_for('static', filename=script) }}"></script> + {% endfor %} </body> </html> diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html index 1ba1617a9..feb9ba942 100644 --- a/searx/templates/oscar/macros.html +++ b/searx/templates/oscar/macros.html @@ -59,3 +59,11 @@ </div>
{% endif %}
{%- endmacro %}
+
+{% macro checkbox_toggle(id, blocked) -%}
+ <div class="checkbox">
+ <input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} />
+ <label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label>
+ <label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label>
+ </div>
+{%- endmacro %}
diff --git a/searx/templates/oscar/preferences.html b/searx/templates/oscar/preferences.html index 126bdbd7b..9beb21009 100644 --- a/searx/templates/oscar/preferences.html +++ b/searx/templates/oscar/preferences.html @@ -1,4 +1,4 @@ -{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %} +{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %} {% extends "oscar/base.html" %} {% block title %}{{ _('preferences') }} - {% endblock %} {% block site_alert_warning_nojs %} @@ -16,6 +16,7 @@ <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;"> <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li> <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li> + <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li> </ul> <!-- Tab panes --> @@ -115,7 +116,7 @@ <!-- Nav tabs --> <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;"> - {% for (categ,search_engines) in categs %} + {% for categ in categories %} <li{% if loop.first %} class="active"{% endif %}><a href="#tab_engine_{{ categ|replace(' ', '_') }}" role="tab" data-toggle="tab">{{ _(categ) }}</a></li> {% endfor %} </ul> @@ -126,24 +127,20 @@ <!-- Tab panes --> <div class="tab-content"> - {% for (categ,search_engines) in categs %} + {% for categ in categories %} <noscript><label>{{ _(categ) }}</label> </noscript> <div class="tab-pane{% if loop.first %} active{% endif %} active_if_nojs" id="tab_engine_{{ categ|replace(' ', '_') }}"> <div class="container-fluid"> <fieldset> - {% for search_engine in search_engines %} + {% for search_engine in engines_by_category[categ] %} {% if not search_engine.private %} <div class="row"> {% if not rtl %} <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div> {% endif %} <div class="col-xs-6 col-sm-4 col-md-4"> - <div class="checkbox"> - <input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} /> - <label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label> - <label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label> - </div> + {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }} </div> {% if rtl %} <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})‎</div> @@ -157,6 +154,28 @@ {% endfor %} </div> </div> + <div class="tab-pane active_if_nojs" id="tab_plugins"> + <noscript> + <h3>{{ _('Plugins') }}</h3> + </noscript> + <fieldset> + <div class="container-fluid"> + {% for plugin in plugins %} + <div class="panel panel-default"> + <div class="panel-heading"> + <h3 class="panel-title">{{ plugin.name }}</h3> + </div> + <div class="panel-body"> + <div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div> + <div class="col-xs-6 col-sm-4 col-md-6"> + {{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }} + </div> + </div> + </div> + {% endfor %} + </div> + </fieldset> + </div> </div> <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }} <br /> @@ -165,6 +184,7 @@ <input type="submit" class="btn btn-primary" value="{{ _('save') }}" /> <a href="{{ url_for('index') }}"><div class="btn btn-default">{{ _('back') }}</div></a> + <a href="{{ url_for('clear_cookies') }}"><div class="btn btn-default">{{ _('Reset defaults') }}</div></a> </form> </div> {% endblock %} diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html index a75825611..155194546 100644 --- a/searx/templates/oscar/results.html +++ b/searx/templates/oscar/results.html @@ -25,8 +25,8 @@ {% endif %}
</div>
{% endfor %}
-
- {% if not results %}
+
+ {% if not results and not answers %}
{% include 'oscar/messages/no_results.html' %}
{% endif %}
@@ -82,7 +82,7 @@ {% for infobox in infoboxes %}
{% include 'oscar/infobox.html' %}
{% endfor %}
- {% endif %}
+ {% endif %}
{% if suggestions %}
<div class="panel panel-default">
@@ -111,7 +111,7 @@ <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly>
</div>
</form>
-
+
<label>{{ _('Download results') }}</label>
<div class="clearfix"></div>
{% for output_type in ('csv', 'json', 'rss') %}
@@ -122,7 +122,7 @@ <input type="hidden" name="pageno" value="{{ pageno }}">
<button type="submit" class="btn btn-default">{{ output_type }}</button>
</form>
- {% endfor %}
+ {% endfor %}
<div class="clearfix"></div>
</div>
</div>
diff --git a/searx/tests/engines/test_blekko_images.py b/searx/tests/engines/test_blekko_images.py index 793fadbad..beb0853e3 100644 --- a/searx/tests/engines/test_blekko_images.py +++ b/searx/tests/engines/test_blekko_images.py @@ -12,9 +12,14 @@ class TestBlekkoImagesEngine(SearxTestCase): dicto['pageno'] = 0 dicto['safesearch'] = 1 params = blekko_images.request(query, dicto) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('blekko.com' in params['url']) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('blekko.com', params['url']) + self.assertIn('page', params['url']) + + dicto['pageno'] = 1 + params = blekko_images.request(query, dicto) + self.assertNotIn('page', params['url']) def test_response(self): self.assertRaises(AttributeError, blekko_images.response, None) diff --git a/searx/tests/engines/test_deezer.py b/searx/tests/engines/test_deezer.py index c8c2c90f2..ad09d2a2c 100644 --- a/searx/tests/engines/test_deezer.py +++ b/searx/tests/engines/test_deezer.py @@ -30,9 +30,9 @@ class TestDeezerEngine(SearxTestCase): json = """ {"data":[ {"id":100, "title":"Title of track", - "link":"http:\/\/www.deezer.com\/track\/1094042","duration":232, + "link":"https:\/\/www.deezer.com\/track\/1094042","duration":232, "artist":{"id":200,"name":"Artist Name", - "link":"http:\/\/www.deezer.com\/artist\/1217","type":"artist"}, + "link":"https:\/\/www.deezer.com\/artist\/1217","type":"artist"}, "album":{"id":118106,"title":"Album Title","type":"album"},"type":"track"} ]} """ @@ -41,14 +41,14 @@ class TestDeezerEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'Title of track') - self.assertEqual(results[0]['url'], 'http://www.deezer.com/track/1094042') + self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042') self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') self.assertTrue('100' in results[0]['embedded']) json = """ {"data":[ {"id":200,"name":"Artist Name", - "link":"http:\/\/www.deezer.com\/artist\/1217","type":"artist"} + "link":"https:\/\/www.deezer.com\/artist\/1217","type":"artist"} ]} """ response = mock.Mock(text=json) diff --git a/searx/tests/engines/test_google_images.py b/searx/tests/engines/test_google_images.py index 6870ff52f..32d133334 100644 --- a/searx/tests/engines/test_google_images.py +++ b/searx/tests/engines/test_google_images.py @@ -11,9 +11,14 @@ class TestGoogleImagesEngine(SearxTestCase): dicto = defaultdict(dict) dicto['pageno'] = 1 params = google_images.request(query, dicto) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('googleapis.com' in params['url']) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('googleapis.com', params['url']) + self.assertIn('safe=on', params['url']) + + dicto['safesearch'] = 0 + params = google_images.request(query, dicto) + self.assertIn('safe=off', params['url']) def test_response(self): self.assertRaises(AttributeError, google_images.response, None) diff --git a/searx/tests/engines/test_piratebay.py b/searx/tests/engines/test_piratebay.py index 17bc3a526..5699380be 100644 --- a/searx/tests/engines/test_piratebay.py +++ b/searx/tests/engines/test_piratebay.py @@ -65,12 +65,39 @@ class TestPiratebayEngine(SearxTestCase): <td align="right">13</td> <td align="right">334</td> </tr> + <tr> + <td class="vertTh"> + <center> + <a href="#" title="More from this category">Anime</a><br/> + (<a href="#" title="More from this category">Anime</a>) + </center> + </td> + <td> + <div class="detName"> + <a href="/this.is.the.link" class="detLink" title="Title"> + This is the title + </a> + </div> + <a href="magnet:?xt=urn:btih:MAGNETLINK" title="Download this torrent using magnet"> + <img src="/static/img/icon-magnet.gif" alt="Magnet link"/> + </a> + <a href="/user/HorribleSubs"> + <img src="/static/img/vip.gif" alt="VIP" title="VIP" style="width:11px;" border='0'/> + </a> + <img src="/static/img/11x11p.png"/> + <font class="detDesc"> + This is the content <span>and should be</span> OK + </font> + </td> + <td align="right">13</td> + <td align="right">334</td> + </tr> </table> """ response = mock.Mock(text=html) results = piratebay.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This is the title') self.assertEqual(results[0]['url'], 'https://thepiratebay.se/this.is.the.link') self.assertEqual(results[0]['content'], 'This is the content and should be OK') @@ -79,6 +106,8 @@ class TestPiratebayEngine(SearxTestCase): self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETLINK') self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/TORRENTFILE.torrent') + self.assertEqual(results[1]['torrentfile'], None) + html = """ <table id="searchResult"> <tr> diff --git a/searx/tests/engines/test_spotify.py b/searx/tests/engines/test_spotify.py new file mode 100644 index 000000000..fd274abbd --- /dev/null +++ b/searx/tests/engines/test_spotify.py @@ -0,0 +1,124 @@ +from collections import defaultdict +import mock +from searx.engines import spotify +from searx.testing import SearxTestCase + + +class TestSpotifyEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 0 + params = spotify.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('spotify.com', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, spotify.response, None) + self.assertRaises(AttributeError, spotify.response, []) + self.assertRaises(AttributeError, spotify.response, '') + self.assertRaises(AttributeError, spotify.response, '[]') + + response = mock.Mock(text='{}') + self.assertEqual(spotify.response(response), []) + + response = mock.Mock(text='{"data": []}') + self.assertEqual(spotify.response(response), []) + + json = """ + { + "tracks": { + "href": "https://api.spotify.com/v1/search?query=nosfell&offset=0&limit=20&type=track", + "items": [ + { + "album": { + "album_type": "album", + "external_urls": { + "spotify": "https://open.spotify.com/album/5c9ap1PBkSGLxT3J73toxA" + }, + "href": "https://api.spotify.com/v1/albums/5c9ap1PBkSGLxT3J73toxA", + "id": "5c9ap1PBkSGLxT3J73toxA", + "name": "Album Title", + "type": "album", + "uri": "spotify:album:5c9ap1PBkSGLxT3J73toxA" + }, + "artists": [ + { + "external_urls": { + "spotify": "https://open.spotify.com/artist/0bMc6b75FfZEpQHG1jifKu" + }, + "href": "https://api.spotify.com/v1/artists/0bMc6b75FfZEpQHG1jifKu", + "id": "0bMc6b75FfZEpQHG1jifKu", + "name": "Artist Name", + "type": "artist", + "uri": "spotify:artist:0bMc6b75FfZEpQHG1jifKu" + } + ], + "disc_number": 1, + "duration_ms": 202386, + "explicit": false, + "external_ids": { + "isrc": "FRV640600067" + }, + "external_urls": { + "spotify": "https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa" + }, + "href": "https://api.spotify.com/v1/tracks/2GzvFiedqW8hgqUpWcASZa", + "id": "1000", + "is_playable": true, + "name": "Title of track", + "popularity": 6, + "preview_url": "https://p.scdn.co/mp3-preview/7b8ecda580965a066b768c2647f877e43f7b1a0a", + "track_number": 3, + "type": "track", + "uri": "spotify:track:2GzvFiedqW8hgqUpWcASZa" + } + ], + "limit": 20, + "next": "https://api.spotify.com/v1/search?query=nosfell&offset=20&limit=20&type=track", + "offset": 0, + "previous": null, + "total": 107 + } + } + """ + response = mock.Mock(text=json) + results = spotify.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title of track') + self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa') + self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') + self.assertIn('1000', results[0]['embedded']) + + json = """ + { + "tracks": { + "href": "https://api.spotify.com/v1/search?query=nosfell&offset=0&limit=20&type=track", + "items": [ + { + "href": "https://api.spotify.com/v1/tracks/2GzvFiedqW8hgqUpWcASZa", + "id": "1000", + "is_playable": true, + "name": "Title of track", + "popularity": 6, + "preview_url": "https://p.scdn.co/mp3-preview/7b8ecda580965a066b768c2647f877e43f7b1a0a", + "track_number": 3, + "type": "album", + "uri": "spotify:track:2GzvFiedqW8hgqUpWcASZa" + } + ], + "limit": 20, + "next": "https://api.spotify.com/v1/search?query=nosfell&offset=20&limit=20&type=track", + "offset": 0, + "previous": null, + "total": 107 + } + } + """ + response = mock.Mock(text=json) + results = spotify.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 0) diff --git a/searx/tests/engines/test_yahoo.py b/searx/tests/engines/test_yahoo.py index e5c78701d..11ef9db22 100644 --- a/searx/tests/engines/test_yahoo.py +++ b/searx/tests/engines/test_yahoo.py @@ -55,37 +55,44 @@ class TestYahooEngine(SearxTestCase): self.assertEqual(yahoo.response(response), []) html = """ - <div class="res"> - <div> - <h3> - <a id="link-1" class="yschttl spt" href="http://r.search.yahoo.com/_ylt=A0LEVzClb9JUSKcAEGRXNyoA; - _ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb2xvA2JmMQR2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10 - /RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=dtcJsfP4mEeBOjnVfUQ-"target="_blank" data-bk="5063.1"> - <b>This</b> is the title - </a> +<ol class="reg mb-15 searchCenterMiddle"> + <li class="first"> + <div class="dd algo fst Sr"> + <div class="compTitle"> + <h3 class="title"><a class=" td-u" href="http://r.search.yahoo.com/_ylt=A0LEb9JUSKcAEGRXNyoA; + _ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb2xvA2Jm2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10 + /RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=dtcJsfP4mEeBOjnVfUQ-" + target="_blank" data-bid="54e712e13671c"> + <b><b>This is the title</b></b></a> </h3> </div> - <span class="url" dir="ltr">www.<b>test</b>.com</span> - <div class="abstr"> - <b>This</b> is the content + <div class="compText aAbs"> + <p class="lh-18"><b><b>This is the </b>content</b> + </p> </div> </div> - <div id="satat" data-bns="Yahoo" data-bk="124.1"> - <h2>Also Try</h2> - <table> - <tbody> - <tr> - <td> - <a id="srpnat0" class="" href="https://search.yahoo.com/search=rs-bottom" > - <span> - <b></b>This is <b>the suggestion</b> - </span> - </a> - </td> - </tr> - </tbody> - </table> + </li> + <li> + <div class="dd algo lst Sr"> + <div class="compTitle"> + </div> + <div class="compText aAbs"> + <p class="lh-18">This is the second content</p> + </div> </div> + </li> +</ol> +<div class="dd assist fst lst AlsoTry" data-bid="54e712e138d04"> + <div class="compTitle mb-4 h-17"> + <h3 class="title">Also Try</h3> </div> + <table class="compTable m-0 ac-1st td-u fz-ms"> + <tbody> + <tr> + <td class="w-50p pr-28"><a href="https://search.yahoo.com/"><B>This is the </B>suggestion<B></B></a> + </td> + </tr> + </table> +</div> """ response = mock.Mock(text=html) results = yahoo.response(response) @@ -97,44 +104,24 @@ class TestYahooEngine(SearxTestCase): self.assertEqual(results[1]['suggestion'], 'This is the suggestion') html = """ - <div class="res"> - <div> - <h3> - <a id="link-1" class="yschttl spt" href="http://r.search.yahoo.com/_ylt=A0LEVzClb9JUSKcAEGRXNyoA; - _ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb2xvA2JmMQR2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10 - /RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=dtcJsfP4mEeBOjnVfUQ-"target="_blank" data-bk="5063.1"> - <b>This</b> is the title - </a> - </h3> - </div> - <span class="url" dir="ltr">www.<b>test</b>.com</span> - <div class="abstr"> - <b>This</b> is the content - </div> - </div> - <div class="res"> - <div> - <h3> - <a id="link-1" class="yschttl spt"> - <b>This</b> is the title - </a> - </h3> - </div> - <span class="url" dir="ltr">www.<b>test</b>.com</span> - <div class="abstr"> - <b>This</b> is the content - </div> - </div> - <div class="res"> - <div> - <h3> +<ol class="reg mb-15 searchCenterMiddle"> + <li class="first"> + <div class="dd algo fst Sr"> + <div class="compTitle"> + <h3 class="title"><a class=" td-u" href="http://r.search.yahoo.com/_ylt=A0LEb9JUSKcAEGRXNyoA; + _ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb2xvA2Jm2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10 + /RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=dtcJsfP4mEeBOjnVfUQ-" + target="_blank" data-bid="54e712e13671c"> + <b><b>This is the title</b></b></a> </h3> </div> - <span class="url" dir="ltr">www.<b>test</b>.com</span> - <div class="abstr"> - <b>This</b> is the content + <div class="compText aAbs"> + <p class="lh-18"><b><b>This is the </b>content</b> + </p> </div> </div> + </li> +</ol> """ response = mock.Mock(text=html) results = yahoo.response(response) diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py index 9b1c12cb1..5770458f3 100644 --- a/searx/tests/test_engines.py +++ b/searx/tests/test_engines.py @@ -28,6 +28,7 @@ from searx.tests.engines.test_piratebay import * # noqa from searx.tests.engines.test_searchcode_code import * # noqa from searx.tests.engines.test_searchcode_doc import * # noqa from searx.tests.engines.test_soundcloud import * # noqa +from searx.tests.engines.test_spotify import * # noqa from searx.tests.engines.test_stackoverflow import * # noqa from searx.tests.engines.test_startpage import * # noqa from searx.tests.engines.test_subtitleseeker import * # noqa diff --git a/searx/tests/test_plugins.py b/searx/tests/test_plugins.py new file mode 100644 index 000000000..8dcad1142 --- /dev/null +++ b/searx/tests/test_plugins.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +from searx.testing import SearxTestCase +from searx import plugins +from mock import Mock + + +class PluginStoreTest(SearxTestCase): + + def test_PluginStore_init(self): + store = plugins.PluginStore() + self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0) + + def test_PluginStore_register(self): + store = plugins.PluginStore() + testplugin = plugins.Plugin() + store.register(testplugin) + + self.assertTrue(len(store.plugins) == 1) + + def test_PluginStore_call(self): + store = plugins.PluginStore() + testplugin = plugins.Plugin() + store.register(testplugin) + setattr(testplugin, 'asdf', Mock()) + request = Mock(user_plugins=[]) + store.call('asdf', request, Mock()) + + self.assertFalse(testplugin.asdf.called) + + request.user_plugins.append(testplugin) + store.call('asdf', request, Mock()) + + self.assertTrue(testplugin.asdf.called) + + +class SelfIPTest(SearxTestCase): + + def test_PluginStore_init(self): + store = plugins.PluginStore() + store.register(plugins.self_ip) + + self.assertTrue(len(store.plugins) == 1) + + request = Mock(user_plugins=store.plugins, + remote_addr='127.0.0.1') + request.headers.getlist.return_value = [] + ctx = {'search': Mock(answers=set(), + query='ip')} + store.call('post_search', request, ctx) + self.assertTrue('127.0.0.1' in ctx['search'].answers) diff --git a/searx/tests/test_webapp.py b/searx/tests/test_webapp.py index 8bbe5d056..32eff5fa5 100644 --- a/searx/tests/test_webapp.py +++ b/searx/tests/test_webapp.py @@ -2,7 +2,6 @@ import json from urlparse import ParseResult -from mock import patch from searx import webapp from searx.testing import SearxTestCase @@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase): }, ] + def search_mock(search_self, *args): + search_self.results = self.test_results + + webapp.Search.search = search_mock + self.maxDiff = None # to see full diffs def test_index_empty(self): @@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase): self.assertEqual(result.status_code, 200) self.assertIn('<div class="title"><h1>searx</h1></div>', result.data) - @patch('searx.search.Search.search') - def test_index_html(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_html(self): result = self.app.post('/', data={'q': 'test'}) self.assertIn( '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa @@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase): result.data ) - @patch('searx.search.Search.search') - def test_index_json(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_json(self): result = self.app.post('/', data={'q': 'test', 'format': 'json'}) result_dict = json.loads(result.data) @@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase): self.assertEqual( result_dict['results'][0]['url'], 'http://first.test.xyz') - @patch('searx.search.Search.search') - def test_index_csv(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_csv(self): result = self.app.post('/', data={'q': 'test', 'format': 'csv'}) self.assertEqual( @@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase): result.data ) - @patch('searx.search.Search.search') - def test_index_rss(self, search): - search.return_value = ( - self.test_results, - set(), - set(), - set() - ) + def test_index_rss(self): result = self.app.post('/', data={'q': 'test', 'format': 'rss'}) self.assertIn( diff --git a/searx/translations/de/LC_MESSAGES/messages.mo b/searx/translations/de/LC_MESSAGES/messages.mo Binary files differindex 265f46788..d4202b004 100644 --- a/searx/translations/de/LC_MESSAGES/messages.mo +++ b/searx/translations/de/LC_MESSAGES/messages.mo diff --git a/searx/translations/de/LC_MESSAGES/messages.po b/searx/translations/de/LC_MESSAGES/messages.po index 056bf8167..c6a78a57d 100644 --- a/searx/translations/de/LC_MESSAGES/messages.po +++ b/searx/translations/de/LC_MESSAGES/messages.po @@ -10,7 +10,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 17:41+0000\n" "Last-Translator: pointhi\n" "Language-Team: German " @@ -361,6 +361,13 @@ msgstr "torrent Datei" msgid "Click on the magnifier to perform search" msgstr "klicke auf die Lupe, um die Suche zu starten" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "Im Cache" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "Powered by" @@ -369,10 +376,6 @@ msgstr "Powered by" msgid "a privacy-respecting, hackable metasearch engine" msgstr "eine privatsphären-respektierende, hackbare Metasuchmaschine" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "Im Cache" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "Startseite" diff --git a/searx/translations/en/LC_MESSAGES/messages.mo b/searx/translations/en/LC_MESSAGES/messages.mo Binary files differindex f1e013b35..1871f6b77 100644 --- a/searx/translations/en/LC_MESSAGES/messages.mo +++ b/searx/translations/en/LC_MESSAGES/messages.mo diff --git a/searx/translations/en/LC_MESSAGES/messages.po b/searx/translations/en/LC_MESSAGES/messages.po index eba5c86b2..c143038da 100644 --- a/searx/translations/en/LC_MESSAGES/messages.po +++ b/searx/translations/en/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2014-01-30 15:22+0100\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Language-Team: en <LL@li.org>\n" @@ -353,6 +353,13 @@ msgstr "" msgid "Click on the magnifier to perform search" msgstr "" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "" @@ -361,10 +368,6 @@ msgstr "" msgid "a privacy-respecting, hackable metasearch engine" msgstr "" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "" diff --git a/searx/translations/es/LC_MESSAGES/messages.mo b/searx/translations/es/LC_MESSAGES/messages.mo Binary files differindex 6089ea6d9..76e6a900e 100644 --- a/searx/translations/es/LC_MESSAGES/messages.mo +++ b/searx/translations/es/LC_MESSAGES/messages.mo diff --git a/searx/translations/es/LC_MESSAGES/messages.po b/searx/translations/es/LC_MESSAGES/messages.po index 866840e06..890c2bd4a 100644 --- a/searx/translations/es/LC_MESSAGES/messages.po +++ b/searx/translations/es/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-11 12:51+0000\n" "Last-Translator: Alejandro León Aznar\n" "Language-Team: Spanish " @@ -359,6 +359,13 @@ msgstr "archivo torrent" msgid "Click on the magnifier to perform search" msgstr "Haz clic en la lupa para realizar la búsqueda" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "en caché" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "Creado por" @@ -367,10 +374,6 @@ msgstr "Creado por" msgid "a privacy-respecting, hackable metasearch engine" msgstr "un metabuscador hackeable que respeta la privacidad" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "en caché" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "inicio" diff --git a/searx/translations/fr/LC_MESSAGES/messages.mo b/searx/translations/fr/LC_MESSAGES/messages.mo Binary files differindex 3944fd77e..28141d905 100644 --- a/searx/translations/fr/LC_MESSAGES/messages.mo +++ b/searx/translations/fr/LC_MESSAGES/messages.mo diff --git a/searx/translations/fr/LC_MESSAGES/messages.po b/searx/translations/fr/LC_MESSAGES/messages.po index e17ecbad0..15adbb302 100644 --- a/searx/translations/fr/LC_MESSAGES/messages.po +++ b/searx/translations/fr/LC_MESSAGES/messages.po @@ -10,7 +10,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 17:01+0000\n" "Last-Translator: Cqoicebordel <david.barouh@wanadoo.fr>\n" "Language-Team: French " @@ -361,6 +361,13 @@ msgstr "fichier torrent" msgid "Click on the magnifier to perform search" msgstr "Cliquez sur la loupe pour effectuer une recherche" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "en cache" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "Powered by" @@ -369,10 +376,6 @@ msgstr "Powered by" msgid "a privacy-respecting, hackable metasearch engine" msgstr "un meta-moteur de recherche hackable et respectueux de la vie privée" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "en cache" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "accueil" diff --git a/searx/translations/he/LC_MESSAGES/messages.mo b/searx/translations/he/LC_MESSAGES/messages.mo Binary files differindex e0f26c913..ee34e6b5b 100644 --- a/searx/translations/he/LC_MESSAGES/messages.mo +++ b/searx/translations/he/LC_MESSAGES/messages.mo diff --git a/searx/translations/he/LC_MESSAGES/messages.po b/searx/translations/he/LC_MESSAGES/messages.po index 83723f9d4..7924d3444 100644 --- a/searx/translations/he/LC_MESSAGES/messages.po +++ b/searx/translations/he/LC_MESSAGES/messages.po @@ -1,27 +1,26 @@ -# Translations template for PROJECT. +# Hebrew translations for . # Copyright (C) 2015 ORGANIZATION -# This file is distributed under the same license as the PROJECT project. -# +# This file is distributed under the same license as the project. +# # Translators: # GenghisKhan <genghiskhan@gmx.ca>, 2015 -# GenghisKhan <genghiskhan@gmx.ca>, 2015 # pointhi, 2014 # rike, 2014 # stf <stefan.marsiske@gmail.com>, 2014 msgid "" msgstr "" -"Project-Id-Version: searx\n" +"Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-11 13:31+0000\n" "Last-Translator: pointhi\n" -"Language-Team: Hebrew (http://www.transifex.com/projects/p/searx/language/he/)\n" +"Language-Team: Hebrew " +"(http://www.transifex.com/projects/p/searx/language/he/)\n" +"Plural-Forms: nplurals=2; plural=(n != 1)\n" "MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" +"Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 1.3\n" -"Language: he\n" -"Plural-Forms: nplurals=2; plural=(n != 1);\n" #: searx/webapp.py:100 msgid "files" @@ -256,9 +255,11 @@ msgstr "חסום" #: searx/templates/default/preferences.html:110 #: searx/templates/oscar/preferences.html:161 msgid "" -"These settings are stored in your cookies, this allows us not to store this " -"data about you." -msgstr "הגדרות אלו הינן מאוחסנות בתוך העוגיות שלך, אלו מאפשרות לנו לא לאחסן את מידע זה אודותייך." +"These settings are stored in your cookies, this allows us not to store " +"this data about you." +msgstr "" +"הגדרות אלו הינן מאוחסנות בתוך העוגיות שלך, אלו מאפשרות לנו לא לאחסן את " +"מידע זה אודותייך." #: searx/templates/courgette/preferences.html:124 #: searx/templates/default/preferences.html:112 @@ -266,7 +267,9 @@ msgstr "הגדרות אלו הינן מאוחסנות בתוך העוגיות ש msgid "" "These cookies serve your sole convenience, we don't use these cookies to " "track you." -msgstr "עוגיות אלו משרתות את נוחיותך הבלעדית, אנחנו לא משתמשים בהן כדי לעקוב אחריך." +msgstr "" +"עוגיות אלו משרתות את נוחיותך הבלעדית, אנחנו לא משתמשים בהן כדי לעקוב " +"אחריך." #: searx/templates/courgette/preferences.html:127 #: searx/templates/default/preferences.html:115 @@ -359,6 +362,13 @@ msgstr "קובץ torrent" msgid "Click on the magnifier to perform search" msgstr "לחצו על הזכוכית מגדלת כדי לבצע חיפוש" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "מוטמן" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "" @@ -367,10 +377,6 @@ msgstr "" msgid "a privacy-respecting, hackable metasearch engine" msgstr "" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "מוטמן" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "בית" @@ -410,7 +416,10 @@ msgid "" "Change how forms are submited, <a " "href=\"http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\"" " rel=\"external\">learn more about request methods</a>" -msgstr "שינוי האופן בו טפסים נשלחים, <a href=\"http://he.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\" rel=\"external\">למדו עוד אודות שיטות בקשה (request methods)</a>" +msgstr "" +"שינוי האופן בו טפסים נשלחים, <a " +"href=\"http://he.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\"" +" rel=\"external\">למדו עוד אודות שיטות בקשה (request methods)</a>" #: searx/templates/oscar/preferences.html:93 msgid "Filter content" @@ -566,3 +575,4 @@ msgstr "הצג וידאו" #: searx/templates/oscar/result_templates/videos.html:7 msgid "hide video" msgstr "הסתר וידאו" + diff --git a/searx/translations/hu/LC_MESSAGES/messages.mo b/searx/translations/hu/LC_MESSAGES/messages.mo Binary files differindex 45c7aa508..b868510a2 100644 --- a/searx/translations/hu/LC_MESSAGES/messages.mo +++ b/searx/translations/hu/LC_MESSAGES/messages.mo diff --git a/searx/translations/hu/LC_MESSAGES/messages.po b/searx/translations/hu/LC_MESSAGES/messages.po index 38ed56893..99d94797f 100644 --- a/searx/translations/hu/LC_MESSAGES/messages.po +++ b/searx/translations/hu/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 17:21+0000\n" "Last-Translator: Adam Tauber <asciimoo@gmail.com>\n" "Language-Team: Hungarian " @@ -357,6 +357,13 @@ msgstr "torrent fájl" msgid "Click on the magnifier to perform search" msgstr "A nagyítóra kattintva indítható a keresés" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "tárolt" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "Az oldalt kiszolgálja: " @@ -365,10 +372,6 @@ msgstr "Az oldalt kiszolgálja: " msgid "a privacy-respecting, hackable metasearch engine" msgstr "egy privátszféra tisztelő, könnyen módosítható metakereső" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "tárolt" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "főoldal" diff --git a/searx/translations/it/LC_MESSAGES/messages.mo b/searx/translations/it/LC_MESSAGES/messages.mo Binary files differindex dac396304..015752577 100644 --- a/searx/translations/it/LC_MESSAGES/messages.mo +++ b/searx/translations/it/LC_MESSAGES/messages.mo diff --git a/searx/translations/it/LC_MESSAGES/messages.po b/searx/translations/it/LC_MESSAGES/messages.po index ba594d4ae..e7f0185bf 100644 --- a/searx/translations/it/LC_MESSAGES/messages.po +++ b/searx/translations/it/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 16:50+0000\n" "Last-Translator: Adam Tauber <asciimoo@gmail.com>\n" "Language-Team: Italian " @@ -359,6 +359,13 @@ msgstr "" msgid "Click on the magnifier to perform search" msgstr "" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "" @@ -367,10 +374,6 @@ msgstr "" msgid "a privacy-respecting, hackable metasearch engine" msgstr "" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "" diff --git a/searx/translations/ja/LC_MESSAGES/messages.mo b/searx/translations/ja/LC_MESSAGES/messages.mo Binary files differindex 6868cb7df..39ad86342 100644 --- a/searx/translations/ja/LC_MESSAGES/messages.mo +++ b/searx/translations/ja/LC_MESSAGES/messages.mo diff --git a/searx/translations/ja/LC_MESSAGES/messages.po b/searx/translations/ja/LC_MESSAGES/messages.po index d3350cdd3..9fdabec7d 100644 --- a/searx/translations/ja/LC_MESSAGES/messages.po +++ b/searx/translations/ja/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 17:31+0000\n" "Last-Translator: pointhi\n" "Language-Team: Japanese " @@ -355,6 +355,13 @@ msgstr "" msgid "Click on the magnifier to perform search" msgstr "" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "" @@ -363,10 +370,6 @@ msgstr "" msgid "a privacy-respecting, hackable metasearch engine" msgstr "" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "スタートページ" diff --git a/searx/translations/nl/LC_MESSAGES/messages.mo b/searx/translations/nl/LC_MESSAGES/messages.mo Binary files differindex e03facda4..7decf2d99 100644 --- a/searx/translations/nl/LC_MESSAGES/messages.mo +++ b/searx/translations/nl/LC_MESSAGES/messages.mo diff --git a/searx/translations/nl/LC_MESSAGES/messages.po b/searx/translations/nl/LC_MESSAGES/messages.po index 681c925df..af6c8c48f 100644 --- a/searx/translations/nl/LC_MESSAGES/messages.po +++ b/searx/translations/nl/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 16:50+0000\n" "Last-Translator: André Koot <meneer@tken.net>\n" "Language-Team: Dutch " @@ -359,6 +359,13 @@ msgstr "" msgid "Click on the magnifier to perform search" msgstr "Klik op het vergrootglas om te zoeken" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "gecached" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "Powered by" @@ -367,10 +374,6 @@ msgstr "Powered by" msgid "a privacy-respecting, hackable metasearch engine" msgstr "een privacy eerbiedigende, aanpasbare metazoekmachine" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "gecached" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "thuis" diff --git a/searx/translations/ru/LC_MESSAGES/messages.mo b/searx/translations/ru/LC_MESSAGES/messages.mo Binary files differindex b03eee4ff..fab6a43f5 100644 --- a/searx/translations/ru/LC_MESSAGES/messages.mo +++ b/searx/translations/ru/LC_MESSAGES/messages.mo diff --git a/searx/translations/ru/LC_MESSAGES/messages.po b/searx/translations/ru/LC_MESSAGES/messages.po index 4b75e3625..224c9ac0a 100644 --- a/searx/translations/ru/LC_MESSAGES/messages.po +++ b/searx/translations/ru/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 17:01+0000\n" "Last-Translator: dimqua <dimqua@riseup.net>\n" "Language-Team: Russian (Russia) " @@ -46,7 +46,7 @@ msgstr "видео" #: searx/webapp.py:106 msgid "it" -msgstr "it" +msgstr "IT" #: searx/webapp.py:107 msgid "news" @@ -54,7 +54,7 @@ msgstr "новости" #: searx/webapp.py:108 msgid "map" -msgstr "карта" +msgstr "карты" #: searx/webapp.py:361 msgid "{minutes} minute(s) ago" @@ -140,19 +140,19 @@ msgstr "Авто-подсказки" #: searx/templates/default/preferences.html:46 #: searx/templates/oscar/preferences.html:74 msgid "Image proxy" -msgstr "" +msgstr "Прокси для картинок" #: searx/templates/courgette/preferences.html:48 #: searx/templates/default/preferences.html:49 #: searx/templates/oscar/preferences.html:78 msgid "Enabled" -msgstr "" +msgstr "Включен" #: searx/templates/courgette/preferences.html:49 #: searx/templates/default/preferences.html:50 #: searx/templates/oscar/preferences.html:79 msgid "Disabled" -msgstr "" +msgstr "Выключен" #: searx/templates/courgette/preferences.html:54 #: searx/templates/default/preferences.html:55 @@ -164,25 +164,25 @@ msgstr "Метод" #: searx/templates/default/preferences.html:64 #: searx/templates/oscar/preferences.html:92 msgid "SafeSearch" -msgstr "" +msgstr "Безопасный поиск" #: searx/templates/courgette/preferences.html:66 #: searx/templates/default/preferences.html:67 #: searx/templates/oscar/preferences.html:96 msgid "Strict" -msgstr "" +msgstr "Строгий" #: searx/templates/courgette/preferences.html:67 #: searx/templates/default/preferences.html:68 #: searx/templates/oscar/preferences.html:97 msgid "Moderate" -msgstr "" +msgstr "Умеренный" #: searx/templates/courgette/preferences.html:68 #: searx/templates/default/preferences.html:69 #: searx/templates/oscar/preferences.html:98 msgid "None" -msgstr "" +msgstr "Выключен" #: searx/templates/courgette/preferences.html:73 #: searx/templates/default/preferences.html:74 @@ -192,31 +192,31 @@ msgstr "Темы" #: searx/templates/courgette/preferences.html:83 msgid "Color" -msgstr "" +msgstr "Цвет" #: searx/templates/courgette/preferences.html:86 msgid "Blue (default)" -msgstr "" +msgstr "Синий (по-умолчанию)" #: searx/templates/courgette/preferences.html:87 msgid "Violet" -msgstr "" +msgstr "Фиолетовый" #: searx/templates/courgette/preferences.html:88 msgid "Green" -msgstr "" +msgstr "Зелёный" #: searx/templates/courgette/preferences.html:89 msgid "Cyan" -msgstr "" +msgstr "Бирюзовый" #: searx/templates/courgette/preferences.html:90 msgid "Orange" -msgstr "" +msgstr "Оранжевый" #: searx/templates/courgette/preferences.html:91 msgid "Red" -msgstr "" +msgstr "Красный" #: searx/templates/courgette/preferences.html:96 #: searx/templates/default/preferences.html:84 @@ -330,7 +330,7 @@ msgstr "Статистика движков" #: searx/templates/courgette/result_templates/images.html:4 #: searx/templates/default/result_templates/images.html:4 msgid "original context" -msgstr "" +msgstr "в контексте" #: searx/templates/courgette/result_templates/torrent.html:7 #: searx/templates/default/result_templates/torrent.html:11 @@ -348,18 +348,25 @@ msgstr "Личер" #: searx/templates/default/result_templates/torrent.html:9 #: searx/templates/oscar/macros.html:21 msgid "magnet link" -msgstr "" +msgstr "магнет-ссылка" #: searx/templates/courgette/result_templates/torrent.html:10 #: searx/templates/default/result_templates/torrent.html:10 #: searx/templates/oscar/macros.html:22 msgid "torrent file" -msgstr "" +msgstr "торрент-файл" #: searx/templates/default/categories.html:8 msgid "Click on the magnifier to perform search" msgstr "Нажмите на лупу, чтобы выполнить поиск" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "в архиве" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "Используется" @@ -368,10 +375,6 @@ msgstr "Используется" msgid "a privacy-respecting, hackable metasearch engine" msgstr "свободный движок метапоиска, уважающий вашу приватность" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "в архиве" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "главная" @@ -404,7 +407,7 @@ msgstr "Поисковые предложения при наборе" #: searx/templates/oscar/preferences.html:75 msgid "Proxying image results through searx" -msgstr "" +msgstr "Проксировать найденные изображения с помощью searx" #: searx/templates/oscar/preferences.html:84 msgid "" @@ -412,13 +415,13 @@ msgid "" "href=\"http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\"" " rel=\"external\">learn more about request methods</a>" msgstr "" -"Изменяет способ отправки запросов, <a " +"Изменяет способ отправки запросов (<a " "href=\"http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods\"" -" rel=\"external\">более подробно о методах запросов</a>" +" rel=\"external\">подробней о методах запросов</a>)" #: searx/templates/oscar/preferences.html:93 msgid "Filter content" -msgstr "" +msgstr "Использовать ли фильтр контента" #: searx/templates/oscar/preferences.html:103 msgid "Change searx layout" @@ -539,31 +542,31 @@ msgstr "скрыть подробности" #: searx/templates/oscar/result_templates/torrent.html:7 msgid "Filesize" -msgstr "" +msgstr "Размер файла" #: searx/templates/oscar/result_templates/torrent.html:9 msgid "Bytes" -msgstr "" +msgstr "Байт" #: searx/templates/oscar/result_templates/torrent.html:10 msgid "kiB" -msgstr "" +msgstr "Кбайт" #: searx/templates/oscar/result_templates/torrent.html:11 msgid "MiB" -msgstr "" +msgstr "Мбайт" #: searx/templates/oscar/result_templates/torrent.html:12 msgid "GiB" -msgstr "" +msgstr "Гбайт" #: searx/templates/oscar/result_templates/torrent.html:13 msgid "TiB" -msgstr "" +msgstr "Тбайт" #: searx/templates/oscar/result_templates/torrent.html:15 msgid "Number of Files" -msgstr "" +msgstr "Число файлов" #: searx/templates/oscar/result_templates/videos.html:7 msgid "show video" @@ -574,5 +577,4 @@ msgid "hide video" msgstr "скрыть видео" #~ msgid "Filter explicite content" -#~ msgstr "" - +#~ msgstr "Скрыть непристойный контент" diff --git a/searx/translations/tr/LC_MESSAGES/messages.mo b/searx/translations/tr/LC_MESSAGES/messages.mo Binary files differindex ce447a3f7..1dc52c5b9 100644 --- a/searx/translations/tr/LC_MESSAGES/messages.mo +++ b/searx/translations/tr/LC_MESSAGES/messages.mo diff --git a/searx/translations/tr/LC_MESSAGES/messages.po b/searx/translations/tr/LC_MESSAGES/messages.po index 951cdb34c..e65ee0b43 100644 --- a/searx/translations/tr/LC_MESSAGES/messages.po +++ b/searx/translations/tr/LC_MESSAGES/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: searx\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-02-11 14:09+0100\n" +"POT-Creation-Date: 2015-02-13 18:27+0100\n" "PO-Revision-Date: 2015-02-10 16:50+0000\n" "Last-Translator: Caner Başaran <basaran.caner@gmail.com>\n" "Language-Team: Turkish " @@ -357,6 +357,13 @@ msgstr "" msgid "Click on the magnifier to perform search" msgstr "Arama yapmak için büyütece tıklayın" +#: searx/templates/default/result_templates/code.html:3 +#: searx/templates/default/result_templates/default.html:3 +#: searx/templates/default/result_templates/map.html:9 +#: searx/templates/oscar/macros.html:20 +msgid "cached" +msgstr "önbellek" + #: searx/templates/oscar/base.html:74 msgid "Powered by" msgstr "Gücümün kaynağı" @@ -367,10 +374,6 @@ msgstr "" "kişisel gizliliğe saygılı ve merak edenlerin kurcalayabildiği bir meta " "arama motoru" -#: searx/templates/oscar/macros.html:20 -msgid "cached" -msgstr "önbellek" - #: searx/templates/oscar/navbar.html:9 searx/templates/oscar/navbar.html:33 msgid "home" msgstr "anasayfa" diff --git a/searx/webapp.py b/searx/webapp.py index d2aa91910..0d06fbe0c 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -26,6 +26,19 @@ import json import cStringIO import os import hashlib +import requests + +from searx import logger +logger = logger.getChild('webapp') + +try: + from pygments import highlight + from pygments.lexers import get_lexer_by_name + from pygments.formatters import HtmlFormatter +except: + logger.critical("cannot import dependency: pygments") + from sys import exit + exit(1) from datetime import datetime, timedelta from urllib import urlencode @@ -37,7 +50,6 @@ from flask import ( ) from flask.ext.babel import Babel, gettext, format_date from searx import settings, searx_dir -from searx.poolrequests import get as http_get from searx.engines import ( categories, engines, get_engines_stats, engine_shortcuts ) @@ -48,22 +60,11 @@ from searx.utils import ( ) from searx.version import VERSION_STRING from searx.languages import language_codes -from searx.https_rewrite import https_url_rewrite from searx.search import Search from searx.query import Query from searx.autocomplete import searx_bang, backends as autocomplete_backends -from searx import logger -try: - from pygments import highlight - from pygments.lexers import get_lexer_by_name - from pygments.formatters import HtmlFormatter -except: - logger.critical("cannot import dependency: pygments") - from sys import exit - exit(1) - +from searx.plugins import plugins -logger = logger.getChild('webapp') static_path, templates_path, themes =\ get_themes(settings['themes_path'] @@ -82,6 +83,8 @@ app = Flask( template_folder=templates_path ) +app.jinja_env.trim_blocks = True +app.jinja_env.lstrip_blocks = True app.secret_key = settings['server']['secret_key'] babel = Babel(app) @@ -304,10 +307,37 @@ def render(template_name, override_theme=None, **kwargs): kwargs['cookies'] = request.cookies + kwargs['scripts'] = set() + for plugin in request.user_plugins: + for script in plugin.js_dependencies: + kwargs['scripts'].add(script) + + kwargs['styles'] = set() + for plugin in request.user_plugins: + for css in plugin.css_dependencies: + kwargs['styles'].add(css) + return render_template( '{}/{}'.format(kwargs['theme'], template_name), **kwargs) +@app.before_request +def pre_request(): + # merge GET, POST vars + request.form = dict(request.form.items()) + for k, v in request.args.items(): + if k not in request.form: + request.form[k] = v + + request.user_plugins = [] + allowed_plugins = request.cookies.get('allowed_plugins', '').split(',') + disabled_plugins = request.cookies.get('disabled_plugins', '').split(',') + for plugin in plugins: + if ((plugin.default_on and plugin.id not in disabled_plugins) + or plugin.id in allowed_plugins): + request.user_plugins.append(plugin) + + @app.route('/search', methods=['GET', 'POST']) @app.route('/', methods=['GET', 'POST']) def index(): @@ -328,20 +358,17 @@ def index(): 'index.html', ) - search.results, search.suggestions,\ - search.answers, search.infoboxes = search.search(request) + if plugins.call('pre_search', request, locals()): + search.search(request) + + plugins.call('post_search', request, locals()) for result in search.results: + plugins.call('on_result', request, locals()) if not search.paging and engines[result['engine']].paging: search.paging = True - # check if HTTPS rewrite is required - if settings['server']['https_rewrite']\ - and result['parsed_url'].scheme == 'http': - - result = https_url_rewrite(result) - if search.request_data.get('format', 'html') == 'html': if 'content' in result: result['content'] = highlight_content(result['content'], @@ -349,11 +376,10 @@ def index(): result['title'] = highlight_content(result['title'], search.query.encode('utf-8')) else: - if 'content' in result: + if result.get('content'): result['content'] = html_to_text(result['content']).strip() # removing html content and whitespace duplications - result['title'] = ' '.join(html_to_text(result['title']) - .strip().split()) + result['title'] = ' '.join(html_to_text(result['title']).strip().split()) result['pretty_url'] = prettify_url(result['url']) @@ -492,11 +518,11 @@ def preferences(): blocked_engines = get_blocked_engines(engines, request.cookies) else: # on save selected_categories = [] + post_disabled_plugins = [] locale = None autocomplete = '' method = 'POST' safesearch = '1' - for pd_name, pd in request.form.items(): if pd_name.startswith('category_'): category = pd_name[9:] @@ -519,14 +545,34 @@ def preferences(): safesearch = pd elif pd_name.startswith('engine_'): if pd_name.find('__') > -1: - engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1) + # TODO fix underscore vs space + engine_name, category = [x.replace('_', ' ') for x in + pd_name.replace('engine_', '', 1).split('__', 1)] if engine_name in engines and category in engines[engine_name].categories: blocked_engines.append((engine_name, category)) elif pd_name == 'theme': theme = pd if pd in themes else default_theme + elif pd_name.startswith('plugin_'): + plugin_id = pd_name.replace('plugin_', '', 1) + if not any(plugin.id == plugin_id for plugin in plugins): + continue + post_disabled_plugins.append(plugin_id) else: resp.set_cookie(pd_name, pd, max_age=cookie_max_age) + disabled_plugins = [] + allowed_plugins = [] + for plugin in plugins: + if plugin.default_on: + if plugin.id in post_disabled_plugins: + disabled_plugins.append(plugin.id) + elif plugin.id not in post_disabled_plugins: + allowed_plugins.append(plugin.id) + + resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age) + + resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age) + resp.set_cookie( 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines), max_age=cookie_max_age @@ -571,11 +617,13 @@ def preferences(): current_language=lang or 'all', image_proxy=image_proxy, language_codes=language_codes, - categs=categories.items(), + engines_by_category=categories, blocked_engines=blocked_engines, autocomplete_backends=autocomplete_backends, shortcuts={y: x for x, y in engine_shortcuts.items()}, themes=themes, + plugins=plugins, + allowed_plugins=[plugin.id for plugin in request.user_plugins], theme=get_current_theme_name()) @@ -594,10 +642,10 @@ def image_proxy(): headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'}) headers['User-Agent'] = gen_useragent() - resp = http_get(url, - stream=True, - timeout=settings['server'].get('request_timeout', 2), - headers=headers) + resp = requests.get(url, + stream=True, + timeout=settings['server'].get('request_timeout', 2), + headers=headers) if resp.status_code == 304: return '', resp.status_code @@ -649,6 +697,10 @@ Disallow: /preferences @app.route('/opensearch.xml', methods=['GET']) def opensearch(): method = 'post' + + if request.cookies.get('method', 'POST') == 'GET': + method = 'get' + # chrome/chromium only supports HTTP GET.... if request.headers.get('User-Agent', '').lower().find('webkit') >= 0: method = 'get' @@ -673,6 +725,14 @@ def favicon(): mimetype='image/vnd.microsoft.icon') +@app.route('/clear_cookies') +def clear_cookies(): + resp = make_response(redirect(url_for('index'))) + for cookie_name in request.cookies: + resp.delete_cookie(cookie_name) + return resp + + def run(): app.run( debug=settings['server']['debug'], diff --git a/versions.cfg b/versions.cfg index 7f1734908..ef8082281 100644 --- a/versions.cfg +++ b/versions.cfg @@ -22,7 +22,7 @@ plone.testing = 4.0.8 pyflakes = 0.7.3 pytz = 2013b pyyaml = 3.10 -requests = 2.2.0 +requests = 2.5.3 robotframework-debuglibrary = 0.3 robotframework-httplibrary = 0.4.2 robotframework-selenium2library = 1.5.0 |