summaryrefslogtreecommitdiff
path: root/searx
diff options
context:
space:
mode:
authorpw3t <romain@berthor.fr>2014-01-23 22:11:36 +0100
committerpw3t <romain@berthor.fr>2014-01-23 22:11:36 +0100
commit132681b3aaf5b330d9d19624038b51fe2ebfd8d5 (patch)
tree393114f41b487eea4b71dd4073903726310a1257 /searx
parentd6b017efb5b51623a02c85690c7335cfc6674092 (diff)
parent59eeeaab87951fd6fa3302ec240db98902a20b2c (diff)
Merge branch 'master' of https://github.com/asciimoo/searx
Diffstat (limited to 'searx')
-rw-r--r--searx/__init__.py6
-rw-r--r--searx/engines/__init__.py122
-rw-r--r--searx/engines/bing.py5
-rw-r--r--searx/engines/currency_convert.py35
-rw-r--r--searx/engines/dailymotion.py13
-rw-r--r--searx/engines/deviantart.py8
-rw-r--r--searx/engines/duckduckgo.py12
-rw-r--r--searx/engines/duckduckgo_definitions.py12
-rw-r--r--searx/engines/filecrop.py25
-rw-r--r--searx/engines/flickr.py10
-rw-r--r--searx/engines/github.py7
-rw-r--r--searx/engines/google_images.py10
-rw-r--r--searx/engines/json_engine.py20
-rw-r--r--searx/engines/mediawiki.py8
-rw-r--r--searx/engines/piratebay.py27
-rw-r--r--searx/engines/soundcloud.py7
-rw-r--r--searx/engines/stackoverflow.py4
-rw-r--r--searx/engines/startpage.py6
-rw-r--r--searx/engines/twitter.py11
-rw-r--r--searx/engines/vimeo.py27
-rw-r--r--searx/engines/xpath.py29
-rw-r--r--searx/engines/yacy.py6
-rw-r--r--searx/engines/youtube.py14
-rw-r--r--searx/settings.yml111
-rw-r--r--searx/settings_robot.py16
-rw-r--r--searx/settings_robot.yml107
-rw-r--r--searx/static/css/style.css5
-rw-r--r--searx/templates/about.html22
-rw-r--r--searx/templates/categories.html2
-rw-r--r--searx/templates/engines.html9
-rw-r--r--searx/templates/index.html4
-rw-r--r--searx/templates/preferences.html22
-rw-r--r--searx/templates/result_templates/default.html6
-rw-r--r--searx/templates/result_templates/videos.html6
-rw-r--r--searx/templates/results.html8
-rw-r--r--searx/templates/stats.html2
-rw-r--r--searx/testing.py19
-rw-r--r--searx/translations/hu/LC_MESSAGES/messages.mobin0 -> 1454 bytes
-rw-r--r--searx/translations/hu/LC_MESSAGES/messages.po115
-rw-r--r--searx/utils.py26
-rw-r--r--searx/webapp.py150
41 files changed, 791 insertions, 263 deletions
diff --git a/searx/__init__.py b/searx/__init__.py
index e313306e3..375a5414a 100644
--- a/searx/__init__.py
+++ b/searx/__init__.py
@@ -1,5 +1,5 @@
from os import environ
-from os.path import realpath, dirname, join
+from os.path import realpath, dirname, join, abspath
try:
from yaml import load
except:
@@ -7,8 +7,7 @@ except:
stderr.write('[E] install pyyaml\n')
exit(2)
-
-searx_dir = realpath(dirname(realpath(__file__))+'/../')
+searx_dir = abspath(dirname(__file__))
engine_dir = dirname(realpath(__file__))
if 'SEARX_SETTINGS_PATH' in environ:
@@ -19,4 +18,3 @@ else:
with open(settings_path) as settings_yaml:
settings = load(settings_yaml)
-
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index 457af4cda..96b074ae9 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -26,6 +26,7 @@ from searx import settings
from searx.utils import gen_useragent
import sys
from datetime import datetime
+from flask.ext.babel import gettext
engine_dir = dirname(realpath(__file__))
@@ -35,6 +36,7 @@ engines = {}
categories = {'general': []}
+
def load_module(filename):
modname = splitext(filename)[0]
if modname in sys.modules:
@@ -50,7 +52,7 @@ if not 'engines' in settings or not settings['engines']:
for engine_data in settings['engines']:
engine_name = engine_data['engine']
- engine = load_module(engine_name+'.py')
+ engine = load_module(engine_name + '.py')
for param_name in engine_data:
if param_name == 'engine':
continue
@@ -58,38 +60,50 @@ for engine_data in settings['engines']:
if engine_data['categories'] == 'none':
engine.categories = []
else:
- engine.categories = map(str.strip, engine_data['categories'].split(','))
+ engine.categories = map(
+ str.strip, engine_data['categories'].split(','))
continue
setattr(engine, param_name, engine_data[param_name])
for engine_attr in dir(engine):
if engine_attr.startswith('_'):
continue
- if getattr(engine, engine_attr) == None:
- print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr)
+ if getattr(engine, engine_attr) is None:
+ print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr) # noqa
sys.exit(1)
engines[engine.name] = engine
- engine.stats = {'result_count': 0, 'search_count': 0, 'page_load_time': 0, 'score_count': 0, 'errors': 0}
+ engine.stats = {
+ 'result_count': 0,
+ 'search_count': 0,
+ 'page_load_time': 0,
+ 'score_count': 0,
+ 'errors': 0
+ }
if hasattr(engine, 'categories'):
for category_name in engine.categories:
categories.setdefault(category_name, []).append(engine)
else:
categories['general'].append(engine)
+
def default_request_params():
- return {'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
+ return {
+ 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
+
def make_callback(engine_name, results, suggestions, callback, params):
# creating a callback wrapper for the search engine results
def process_callback(response, **kwargs):
cb_res = []
response.search_params = params
- engines[engine_name].stats['page_load_time'] += (datetime.now() - params['started']).total_seconds()
+ engines[engine_name].stats['page_load_time'] += \
+ (datetime.now() - params['started']).total_seconds()
try:
search_results = callback(response)
except Exception, e:
engines[engine_name].stats['errors'] += 1
results[engine_name] = cb_res
- print '[E] Error with engine "{0}":\n\t{1}'.format(engine_name, str(e))
+ print '[E] Error with engine "{0}":\n\t{1}'.format(
+ engine_name, str(e))
return
for result in search_results:
result['engine'] = engine_name
@@ -101,23 +115,25 @@ def make_callback(engine_name, results, suggestions, callback, params):
results[engine_name] = cb_res
return process_callback
+
def score_results(results):
- flat_res = filter(None, chain.from_iterable(izip_longest(*results.values())))
+ flat_res = filter(
+ None, chain.from_iterable(izip_longest(*results.values())))
flat_len = len(flat_res)
engines_len = len(results)
results = []
# deduplication + scoring
- for i,res in enumerate(flat_res):
+ for i, res in enumerate(flat_res):
res['parsed_url'] = urlparse(res['url'])
res['engines'] = [res['engine']]
weight = 1.0
if hasattr(engines[res['engine']], 'weight'):
weight = float(engines[res['engine']].weight)
- score = int((flat_len - i)/engines_len)*weight+1
+ score = int((flat_len - i) / engines_len) * weight + 1
duplicated = False
for new_res in results:
- p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path
- p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path
+ p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
+ p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
if res['parsed_url'].netloc == new_res['parsed_url'].netloc and\
p1 == p2 and\
res['parsed_url'].query == new_res['parsed_url'].query and\
@@ -125,7 +141,7 @@ def score_results(results):
duplicated = new_res
break
if duplicated:
- if len(res.get('content', '')) > len(duplicated.get('content', '')):
+ if len(res.get('content', '')) > len(duplicated.get('content', '')): # noqa
duplicated['content'] = res['content']
duplicated['score'] += score
duplicated['engines'].append(res['engine'])
@@ -139,6 +155,7 @@ def score_results(results):
results.append(res)
return sorted(results, key=itemgetter('score'), reverse=True)
+
def search(query, request, selected_engines):
global engines, categories, number_of_searches
requests = []
@@ -160,13 +177,20 @@ def search(query, request, selected_engines):
request_params['started'] = datetime.now()
request_params = engine.request(query, request_params)
- callback = make_callback(selected_engine['name'], results, suggestions, engine.response, request_params)
-
- request_args = dict(headers = request_params['headers']
- ,hooks = dict(response=callback)
- ,cookies = request_params['cookies']
- ,timeout = settings['server']['request_timeout']
- )
+ callback = make_callback(
+ selected_engine['name'],
+ results,
+ suggestions,
+ engine.response,
+ request_params
+ )
+
+ request_args = dict(
+ headers=request_params['headers'],
+ hooks=dict(response=callback),
+ cookies=request_params['cookies'],
+ timeout=settings['server']['request_timeout']
+ )
if request_params['method'] == 'GET':
req = grequests.get
@@ -180,7 +204,7 @@ def search(query, request, selected_engines):
requests.append(req(request_params['url'], **request_args))
grequests.map(requests)
- for engine_name,engine_results in results.items():
+ for engine_name, engine_results in results.items():
engines[engine_name].stats['search_count'] += 1
engines[engine_name].stats['result_count'] += len(engine_results)
@@ -192,6 +216,7 @@ def search(query, request, selected_engines):
return results, suggestions
+
def get_engines_stats():
# TODO refactor
pageloads = []
@@ -200,14 +225,15 @@ def get_engines_stats():
errors = []
scores_per_result = []
- max_pageload = max_results = max_score = max_errors = max_score_per_result = 0
+ max_pageload = max_results = max_score = max_errors = max_score_per_result = 0 # noqa
for engine in engines.values():
if engine.stats['search_count'] == 0:
continue
- results_num = engine.stats['result_count']/float(engine.stats['search_count'])
- load_times = engine.stats['page_load_time']/float(engine.stats['search_count'])
+ results_num = \
+ engine.stats['result_count'] / float(engine.stats['search_count'])
+ load_times = engine.stats['page_load_time'] / float(engine.stats['search_count']) # noqa
if results_num:
- score = engine.stats['score_count'] / float(engine.stats['search_count'])
+ score = engine.stats['score_count'] / float(engine.stats['search_count']) # noqa
score_per_result = score / results_num
else:
score = score_per_result = 0.0
@@ -220,30 +246,48 @@ def get_engines_stats():
results.append({'avg': results_num, 'name': engine.name})
scores.append({'avg': score, 'name': engine.name})
errors.append({'avg': engine.stats['errors'], 'name': engine.name})
- scores_per_result.append({'avg': score_per_result, 'name': engine.name})
+ scores_per_result.append({
+ 'avg': score_per_result,
+ 'name': engine.name
+ })
for engine in pageloads:
- engine['percentage'] = int(engine['avg']/max_pageload*100)
+ engine['percentage'] = int(engine['avg'] / max_pageload * 100)
for engine in results:
- engine['percentage'] = int(engine['avg']/max_results*100)
+ engine['percentage'] = int(engine['avg'] / max_results * 100)
for engine in scores:
- engine['percentage'] = int(engine['avg']/max_score*100)
+ engine['percentage'] = int(engine['avg'] / max_score * 100)
for engine in scores_per_result:
- engine['percentage'] = int(engine['avg']/max_score_per_result*100)
+ engine['percentage'] = int(engine['avg'] / max_score_per_result * 100)
for engine in errors:
if max_errors:
- engine['percentage'] = int(float(engine['avg'])/max_errors*100)
+ engine['percentage'] = int(float(engine['avg']) / max_errors * 100)
else:
engine['percentage'] = 0
-
- return [('Page loads (sec)', sorted(pageloads, key=itemgetter('avg')))
- ,('Number of results', sorted(results, key=itemgetter('avg'), reverse=True))
- ,('Scores', sorted(scores, key=itemgetter('avg'), reverse=True))
- ,('Scores per result', sorted(scores_per_result, key=itemgetter('avg'), reverse=True))
- ,('Errors', sorted(errors, key=itemgetter('avg'), reverse=True))
- ]
+ return [
+ (
+ gettext('Page loads (sec)'),
+ sorted(pageloads, key=itemgetter('avg'))
+ ),
+ (
+ gettext('Number of results'),
+ sorted(results, key=itemgetter('avg'), reverse=True)
+ ),
+ (
+ gettext('Scores'),
+ sorted(scores, key=itemgetter('avg'), reverse=True)
+ ),
+ (
+ gettext('Scores per result'),
+ sorted(scores_per_result, key=itemgetter('avg'), reverse=True)
+ ),
+ (
+ gettext('Errors'),
+ sorted(errors, key=itemgetter('avg'), reverse=True)
+ ),
+ ]
diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index 6b0bf5a3f..c4b945633 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -4,11 +4,12 @@ from cgi import escape
base_url = 'http://www.bing.com/'
search_string = 'search?{query}'
-locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx
+locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx
def request(query, params):
- search_path = search_string.format(query=urlencode({'q': query, 'setmkt': locale}))
+ search_path = search_string.format(
+ query=urlencode({'q': query, 'setmkt': locale}))
#if params['category'] == 'images':
# params['url'] = base_url + 'images/' + search_path
params['url'] = base_url + search_path
diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py
index 358d6b67e..ce6b3b854 100644
--- a/searx/engines/currency_convert.py
+++ b/searx/engines/currency_convert.py
@@ -5,7 +5,8 @@ categories = []
url = 'http://finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'
weight = 100
-parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I)
+parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I) # noqa
+
def request(query, params):
m = parser_re.match(query)
@@ -19,7 +20,7 @@ def request(query, params):
# wrong params
return params
- q = (from_currency+to_currency).upper()
+ q = (from_currency + to_currency).upper()
params['url'] = url.format(query=q)
params['ammount'] = ammount
@@ -33,25 +34,29 @@ def response(resp):
global base_url
results = []
try:
- _,conversion_rate,_ = resp.text.split(',', 2)
+ _, conversion_rate, _ = resp.text.split(',', 2)
conversion_rate = float(conversion_rate)
except:
return results
- title = '{0} {1} in {2} is {3}'.format(resp.search_params['ammount']
- ,resp.search_params['from']
- ,resp.search_params['to']
- ,resp.search_params['ammount']*conversion_rate
- )
+ title = '{0} {1} in {2} is {3}'.format(
+ resp.search_params['ammount'],
+ resp.search_params['from'],
+ resp.search_params['to'],
+ resp.search_params['ammount'] * conversion_rate
+ )
- content = '1 {0} is {1} {2}'.format(resp.search_params['from'], conversion_rate, resp.search_params['to'])
+ content = '1 {0} is {1} {2}'.format(resp.search_params['from'],
+ conversion_rate,
+ resp.search_params['to'])
now_date = datetime.now().strftime('%Y%m%d')
- url = 'http://finance.yahoo.com/currency/converter-results/{0}/{1}-{2}-to-{3}.html'
- url = url.format(now_date
- ,resp.search_params['ammount']
- ,resp.search_params['from'].lower()
- ,resp.search_params['to'].lower()
- )
+ url = 'http://finance.yahoo.com/currency/converter-results/{0}/{1}-{2}-to-{3}.html' # noqa
+ url = url.format(
+ now_date,
+ resp.search_params['ammount'],
+ resp.search_params['from'].lower(),
+ resp.search_params['to'].lower()
+ )
results.append({'title': title, 'content': content, 'url': url})
return results
diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py
index 655485957..510dbbfae 100644
--- a/searx/engines/dailymotion.py
+++ b/searx/engines/dailymotion.py
@@ -1,17 +1,21 @@
from urllib import urlencode
from lxml import html
from json import loads
-from cgi import escape
categories = ['videos']
locale = 'en_US'
# see http://www.dailymotion.com/doc/api/obj-video.html
-search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}'
+search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}' # noqa
+
+# TODO use video result template
+content_tpl = '<a href="{0}" title="{0}" ><img src="{1}" /></a><br />'
+
def request(query, params):
global search_url
- params['url'] = search_url.format(query=urlencode({'search': query, 'localization': locale }))
+ params['url'] = search_url.format(
+ query=urlencode({'search': query, 'localization': locale}))
return params
@@ -24,7 +28,7 @@ def response(resp):
title = res['title']
url = res['url']
if res['thumbnail_360_url']:
- content = '<a href="{0}" title="{0}" ><img src="{1}" /></a><br />'.format(url, res['thumbnail_360_url'])
+ content = content_tpl.format(url, res['thumbnail_360_url'])
else:
content = ''
if res['description']:
@@ -33,6 +37,7 @@ def response(resp):
results.append({'url': url, 'title': title, 'content': content})
return results
+
def text_content_from_html(html_string):
desc_html = html.fragment_fromstring(html_string, create_parent=True)
return desc_html.text_content()
diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py
index 9a4a8abde..94a94bf16 100644
--- a/searx/engines/deviantart.py
+++ b/searx/engines/deviantart.py
@@ -7,6 +7,7 @@ categories = ['images']
base_url = 'https://www.deviantart.com/'
search_url = base_url+'search?'
+
def request(query, params):
global search_url
params['url'] = search_url + urlencode({'q': query})
@@ -22,8 +23,11 @@ def response(resp):
for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):
link = result.xpath('.//a[contains(@class, "thumb")]')[0]
url = urljoin(base_url, link.attrib.get('href'))
- title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')
+ title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]') # noqa
title = ''.join(title_links[0].xpath('.//text()'))
img_src = link.xpath('.//img')[0].attrib['src']
- results.append({'url': url, 'title': title, 'img_src': img_src, 'template': 'images.html'})
+ results.append({'url': url,
+ 'title': title,
+ 'img_src': img_src,
+ 'template': 'images.html'})
return results
diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
index 4bf770972..7cae87d95 100644
--- a/searx/engines/duckduckgo.py
+++ b/searx/engines/duckduckgo.py
@@ -6,8 +6,11 @@ url = 'https://duckduckgo.com/'
search_url = url + 'd.js?{query}&p=1&s=0'
locale = 'us-en'
+
def request(query, params):
- params['url'] = search_url.format(query=urlencode({'q': query, 'l': locale}))
+ q = urlencode({'q': query,
+ 'l': locale})
+ params['url'] = search_url.format(query=q)
return params
@@ -17,8 +20,7 @@ def response(resp):
for r in search_res:
if not r.get('t'):
continue
- results.append({'title': r['t']
- ,'content': html_to_text(r['a'])
- ,'url': r['u']
- })
+ results.append({'title': r['t'],
+ 'content': html_to_text(r['a']),
+ 'url': r['u']})
return results
diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py
index 7b3950b85..3037aae53 100644
--- a/searx/engines/duckduckgo_definitions.py
+++ b/searx/engines/duckduckgo_definitions.py
@@ -3,8 +3,9 @@ from urllib import urlencode
url = 'http://api.duckduckgo.com/?{query}&format=json&pretty=0&no_redirect=1'
+
def request(query, params):
- params['url'] = url.format(query=urlencode({'q': query}))
+ params['url'] = url.format(query=urlencode({'q': query}))
return params
@@ -13,11 +14,10 @@ def response(resp):
results = []
if 'Definition' in search_res:
if search_res.get('AbstractURL'):
- res = {'title' : search_res.get('Heading', '')
- ,'content' : search_res.get('Definition', '')
- ,'url' : search_res.get('AbstractURL', '')
- ,'class' : 'definition_result'
- }
+ res = {'title': search_res.get('Heading', ''),
+ 'content': search_res.get('Definition', ''),
+ 'url': search_res.get('AbstractURL', ''),
+ 'class': 'definition_result'}
results.append(res)
return results
diff --git a/searx/engines/filecrop.py b/searx/engines/filecrop.py
index 52426b84a..81340e601 100644
--- a/searx/engines/filecrop.py
+++ b/searx/engines/filecrop.py
@@ -2,7 +2,8 @@ from urllib import urlencode
from HTMLParser import HTMLParser
url = 'http://www.filecrop.com/'
-search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1'
+search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1' # noqa
+
class FilecropResultParser(HTMLParser):
def __init__(self):
@@ -18,22 +19,28 @@ class FilecropResultParser(HTMLParser):
def handle_starttag(self, tag, attrs):
if tag == 'tr':
- if ('bgcolor', '#edeff5') in attrs or ('bgcolor', '#ffffff') in attrs:
+ if ('bgcolor', '#edeff5') in attrs or\
+ ('bgcolor', '#ffffff') in attrs:
self.__start_processing = True
if not self.__start_processing:
return
if tag == 'label':
- self.result['title'] = [attr[1] for attr in attrs if attr[0] == 'title'][0]
- elif tag == 'a' and ('rel', 'nofollow') in attrs and ('class', 'sourcelink') in attrs:
+ self.result['title'] = [attr[1] for attr in attrs
+ if attr[0] == 'title'][0]
+ elif tag == 'a' and ('rel', 'nofollow') in attrs\
+ and ('class', 'sourcelink') in attrs:
if 'content' in self.result:
- self.result['content'] += [attr[1] for attr in attrs if attr[0] == 'title'][0]
+ self.result['content'] += [attr[1] for attr in attrs
+ if attr[0] == 'title'][0]
else:
- self.result['content'] = [attr[1] for attr in attrs if attr[0] == 'title'][0]
+ self.result['content'] = [attr[1] for attr in attrs
+ if attr[0] == 'title'][0]
self.result['content'] += ' '
elif tag == 'a':
- self.result['url'] = url + [attr[1] for attr in attrs if attr[0] == 'href'][0]
+ self.result['url'] = url + [attr[1] for attr in attrs
+ if attr[0] == 'href'][0]
def handle_endtag(self, tag):
if self.__start_processing is False:
@@ -60,10 +67,12 @@ class FilecropResultParser(HTMLParser):
self.data_counter += 1
+
def request(query, params):
- params['url'] = search_url.format(query=urlencode({'w' :query}))
+ params['url'] = search_url.format(query=urlencode({'w': query}))
return params
+
def response(resp):
parser = FilecropResultParser()
parser.feed(resp.text)
diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py
index a9832856d..d9554b99a 100644
--- a/searx/engines/flickr.py
+++ b/searx/engines/flickr.py
@@ -8,21 +8,27 @@ categories = ['images']
url = 'https://secure.flickr.com/'
search_url = url+'search/?{query}'
+results_xpath = '//div[@id="thumbnails"]//a[@class="rapidnofollow photo-click" and @data-track="photo-click"]' # noqa
+
def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}))
return params
+
def response(resp):
global base_url
results = []
dom = html.fromstring(resp.text)
- for result in dom.xpath('//div[@id="thumbnails"]//a[@class="rapidnofollow photo-click" and @data-track="photo-click"]'):
+ for result in dom.xpath(results_xpath):
href = urljoin(url, result.attrib.get('href'))
img = result.xpath('.//img')[0]
title = img.attrib.get('alt', '')
img_src = img.attrib.get('data-defer-src')
if not img_src:
continue
- results.append({'url': href, 'title': title, 'img_src': img_src, 'template': 'images.html'})
+ results.append({'url': href,
+ 'title': title,
+ 'img_src': img_src,
+ 'template': 'images.html'})
return results
diff --git a/searx/engines/github.py b/searx/engines/github.py
index b4baea6e8..be2cfe7c5 100644
--- a/searx/engines/github.py
+++ b/searx/engines/github.py
@@ -4,12 +4,15 @@ from cgi import escape
categories = ['it']
-search_url = 'https://api.github.com/search/repositories?sort=stars&order=desc&{query}'
+search_url = 'https://api.github.com/search/repositories?sort=stars&order=desc&{query}' # noqa
+
+accept_header = 'application/vnd.github.preview.text-match+json'
+
def request(query, params):
global search_url
params['url'] = search_url.format(query=urlencode({'q': query}))
- params['headers']['Accept'] = 'application/vnd.github.preview.text-match+json'
+ params['headers']['Accept'] = accept_header
return params
diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py
index d828a9c4b..57e749265 100644
--- a/searx/engines/google_images.py
+++ b/searx/engines/google_images.py
@@ -6,12 +6,14 @@ from json import loads
categories = ['images']
url = 'https://ajax.googleapis.com/'
-search_url = url + 'ajax/services/search/images?v=1.0&start=0&rsz=large&safe=off&filter=off&{query}'
+search_url = url + 'ajax/services/search/images?v=1.0&start=0&rsz=large&safe=off&filter=off&{query}' # noqa
+
def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}))
return params
+
def response(resp):
results = []
search_res = loads(resp.text)
@@ -24,5 +26,9 @@ def response(resp):
title = result['title']
if not result['url']:
continue
- results.append({'url': href, 'title': title, 'content': '', 'img_src': result['url'], 'template': 'images.html'})
+ results.append({'url': href,
+ 'title': title,
+ 'content': '',
+ 'img_src': result['url'],
+ 'template': 'images.html'})
return results
diff --git a/searx/engines/json_engine.py b/searx/engines/json_engine.py
index 0386d53f7..e7cc808bb 100644
--- a/searx/engines/json_engine.py
+++ b/searx/engines/json_engine.py
@@ -2,12 +2,13 @@ from urllib import urlencode
from json import loads
from collections import Iterable
-search_url = None
-url_query = None
+search_url = None
+url_query = None
content_query = None
-title_query = None
+title_query = None
#suggestion_xpath = ''
+
def iterate(iterable):
if type(iterable) == dict:
it = iterable.iteritems()
@@ -17,11 +18,15 @@ def iterate(iterable):
for index, value in it:
yield str(index), value
+
def is_iterable(obj):
- if type(obj) == str: return False
- if type(obj) == unicode: return False
+ if type(obj) == str:
+ return False
+ if type(obj) == unicode:
+ return False
return isinstance(obj, Iterable)
+
def parse(query):
q = []
for part in query.split('/'):
@@ -31,6 +36,7 @@ def parse(query):
q.append(part)
return q
+
def do_query(data, q):
ret = []
if not len(q):
@@ -38,7 +44,7 @@ def do_query(data, q):
qkey = q[0]
- for key,value in iterate(data):
+ for key, value in iterate(data):
if len(q) == 1:
if key == qkey:
@@ -54,11 +60,13 @@ def do_query(data, q):
ret.extend(do_query(value, q))
return ret
+
def query(data, query_string):
q = parse(query_string)
return do_query(data, q)
+
def request(query, params):
query = urlencode({'q': query})[2:]
params['url'] = search_url.format(query=query)
diff --git a/searx/engines/mediawiki.py b/searx/engines/mediawiki.py
index 00ad0f106..bc4aab6df 100644
--- a/searx/engines/mediawiki.py
+++ b/searx/engines/mediawiki.py
@@ -3,10 +3,12 @@ from urllib import urlencode, quote
url = 'https://en.wikipedia.org/'
+search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json' # noqa
+
number_of_results = 10
+
def request(query, params):
- search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json'
params['url'] = search_url.format(query=urlencode({'srsearch': query}))
return params
@@ -14,7 +16,5 @@ def request(query, params):
def response(resp):
search_results = loads(resp.text)
res = search_results.get('query', {}).get('search', [])
-
- return [{'url': url + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')),
+ return [{'url': url + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')), # noqa
'title': result['title']} for result in res[:int(number_of_results)]]
-
diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py
index 9cf410106..7319b49c1 100644
--- a/searx/engines/piratebay.py
+++ b/searx/engines/piratebay.py
@@ -7,13 +7,18 @@ categories = ['videos', 'music']
url = 'https://thepiratebay.se/'
search_url = url + 'search/{search_term}/0/99/{search_type}'
-search_types = {'videos': '200'
- ,'music' : '100'
- ,'files' : '0'
- }
+search_types = {'videos': '200',
+ 'music': '100',
+ 'files': '0'}
+
+magnet_xpath = './/a[@title="Download this torrent using magnet"]'
+content_xpath = './/font[@class="detDesc"]//text()'
+
def request(query, params):
- params['url'] = search_url.format(search_term=quote(query), search_type=search_types.get(params['category']))
+ search_type = search_types.get(params['category'])
+ params['url'] = search_url.format(search_term=quote(query),
+ search_type=search_type)
return params
@@ -27,10 +32,14 @@ def response(resp):
link = result.xpath('.//div[@class="detName"]//a')[0]
href = urljoin(url, link.attrib.get('href'))
title = ' '.join(link.xpath('.//text()'))
- content = escape(' '.join(result.xpath('.//font[@class="detDesc"]//text()')))
+ content = escape(' '.join(result.xpath(content_xpath)))
seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]
- magnetlink = result.xpath('.//a[@title="Download this torrent using magnet"]')[0]
- results.append({'url': href, 'title': title, 'content': content,
- 'seed': seed, 'leech': leech, 'magnetlink': magnetlink.attrib['href'],
+ magnetlink = result.xpath(magnet_xpath)[0]
+ results.append({'url': href,
+ 'title': title,
+ 'content': content,
+ 'seed': seed,
+ 'leech': leech,
+ 'magnetlink': magnetlink.attrib['href'],
'template': 'torrent.html'})
return results
diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py
index 50414f153..b1930b2ee 100644
--- a/searx/engines/soundcloud.py
+++ b/searx/engines/soundcloud.py
@@ -5,7 +5,8 @@ categories = ['music']
guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'
url = 'https://api.soundcloud.com/'
-search_url = url + 'search?{query}&facet=model&limit=20&offset=0&linked_partitioning=1&client_id='+guest_client_id
+search_url = url + 'search?{query}&facet=model&limit=20&offset=0&linked_partitioning=1&client_id='+guest_client_id # noqa
+
def request(query, params):
global search_url
@@ -21,5 +22,7 @@ def response(resp):
if result['kind'] in ('track', 'playlist'):
title = result['title']
content = result['description']
- results.append({'url': result['permalink_url'], 'title': title, 'content': content})
+ results.append({'url': result['permalink_url'],
+ 'title': title,
+ 'content': content})
return results
diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py
index 9ee89bc6e..35230600f 100644
--- a/searx/engines/stackoverflow.py
+++ b/searx/engines/stackoverflow.py
@@ -7,6 +7,8 @@ categories = ['it']
url = 'http://stackoverflow.com/'
search_url = url+'search?'
+result_xpath = './/div[@class="excerpt"]//text()'
+
def request(query, params):
params['url'] = search_url + urlencode({'q': query})
@@ -20,6 +22,6 @@ def response(resp):
link = result.xpath('.//div[@class="result-link"]//a')[0]
href = urljoin(url, link.attrib.get('href'))
title = escape(' '.join(link.xpath('.//text()')))
- content = escape(' '.join(result.xpath('.//div[@class="excerpt"]//text()')))
+ content = escape(' '.join(result.xpath(result_xpath)))
results.append({'url': href, 'title': title, 'content': content})
return results
diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py
index 87c091e2d..d6d7cf44d 100644
--- a/searx/engines/startpage.py
+++ b/searx/engines/startpage.py
@@ -1,11 +1,10 @@
from urllib import urlencode
from lxml import html
-from urlparse import urlparse
-from cgi import escape
base_url = 'https://startpage.com/'
search_url = base_url+'do/search'
+
def request(query, params):
global search_url
query = urlencode({'q': query})[2:]
@@ -20,11 +19,10 @@ def response(resp):
results = []
dom = html.fromstring(resp.content)
# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
- # not ads : div[@class="result"] are the direct childs of div[@id="results"]
+ # not ads: div[@class="result"] are the direct childs of div[@id="results"]
for result in dom.xpath('//div[@id="results"]/div[@class="result"]'):
link = result.xpath('.//h3/a')[0]
url = link.attrib.get('href')
- parsed_url = urlparse(url)
title = link.text_content()
content = result.xpath('./p[@class="desc"]')[0].text_content()
results.append({'url': url, 'title': title, 'content': content})
diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py
index f9d9e26ad..23393ac4d 100644
--- a/searx/engines/twitter.py
+++ b/searx/engines/twitter.py
@@ -7,6 +7,9 @@ categories = ['social media']
base_url = 'https://twitter.com/'
search_url = base_url+'search?'
+title_xpath = './/span[@class="username js-action-profile-name"]//text()'
+content_xpath = './/p[@class="js-tweet-text tweet-text"]//text()'
+
def request(query, params):
global search_url
@@ -21,7 +24,9 @@ def response(resp):
for tweet in dom.xpath('//li[@data-item-type="tweet"]'):
link = tweet.xpath('.//small[@class="time"]//a')[0]
url = urljoin(base_url, link.attrib.get('href'))
- title = ''.join(tweet.xpath('.//span[@class="username js-action-profile-name"]//text()'))
- content = escape(''.join(tweet.xpath('.//p[@class="js-tweet-text tweet-text"]//text()')))
- results.append({'url': url, 'title': title, 'content': content})
+ title = ''.join(tweet.xpath(title_xpath))
+ content = escape(''.join(tweet.xpath(content_xpath)))
+ results.append({'url': url,
+ 'title': title,
+ 'content': content})
return results
diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py
index 35bc3d50a..924497a99 100644
--- a/searx/engines/vimeo.py
+++ b/searx/engines/vimeo.py
@@ -5,27 +5,31 @@ from lxml import html
base_url = 'http://vimeo.com'
search_url = base_url + '/search?{query}'
-url_xpath = None
+url_xpath = None
content_xpath = None
-title_xpath = None
+title_xpath = None
results_xpath = ''
+content_tpl = '<a href="{0}"> <img src="{2}"/> </a>'
-# the cookie set by vimeo contains all the following values, but only __utma seems to be requiered
+# the cookie set by vimeo contains all the following values,
+# but only __utma seems to be requiered
cookie = {
#'vuid':'918282893.1027205400'
# 'ab_bs':'%7B%223%22%3A279%7D'
- '__utma':'00000000.000#0000000.0000000000.0000000000.0000000000.0'
+ '__utma': '00000000.000#0000000.0000000000.0000000000.0000000000.0'
# '__utmb':'18302654.1.10.1388942090'
#, '__utmc':'18302654'
- #, '__utmz':'18#302654.1388942090.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)'
+ #, '__utmz':'18#302654.1388942090.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)' # noqa
#, '__utml':'search'
}
+
def request(query, params):
- params['url'] = search_url.format(query=urlencode({'q' :query}))
+ params['url'] = search_url.format(query=urlencode({'q': query}))
params['cookies'] = cookie
return params
+
def response(resp):
results = []
dom = html.fromstring(resp.text)
@@ -36,10 +40,9 @@ def response(resp):
url = base_url + result.xpath(url_xpath)[0]
title = p.unescape(extract_text(result.xpath(title_xpath)))
thumbnail = extract_text(result.xpath(content_xpath)[0])
- content = '<a href="{0}"> <img src="{2}"/> </a>'.format(url, title, thumbnail)
- results.append({'url': url
- , 'title': title
- , 'content': content
- , 'template':'videos.html'
- , 'thumbnail': thumbnail})
+ results.append({'url': url,
+ 'title': title,
+ 'content': content_tpl.format(url, title, thumbnail),
+ 'template': 'videos.html',
+ 'thumbnail': thumbnail})
return results
diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py
index 5e2c3c38b..8960b5f21 100644
--- a/searx/engines/xpath.py
+++ b/searx/engines/xpath.py
@@ -1,21 +1,25 @@
from lxml import html
from urllib import urlencode, unquote
from urlparse import urlparse, urljoin
-from cgi import escape
from lxml.etree import _ElementStringResult
+from searx.utils import html_to_text
-search_url = None
-url_xpath = None
+search_url = None
+url_xpath = None
content_xpath = None
-title_xpath = None
+title_xpath = None
suggestion_xpath = ''
results_xpath = ''
+
'''
if xpath_results is list, extract the text from each result and concat the list
-if xpath_results is a xml element, extract all the text node from it ( text_content() method from lxml )
+if xpath_results is a xml element, extract all the text node from it
+ ( text_content() method from lxml )
if xpath_results is a string element, then it's already done
'''
+
+
def extract_text(xpath_results):
if type(xpath_results) == list:
# it's list of result : concat everything using recursive call
@@ -30,7 +34,7 @@ def extract_text(xpath_results):
return ''.join(xpath_results)
else:
# it's a element
- return xpath_results.text_content()
+ return html_to_text(xpath_results.text_content())
def extract_url(xpath_results):
@@ -60,7 +64,8 @@ def normalize_url(url):
url += '/'
# FIXME : hack for yahoo
- if parsed_url.hostname == 'search.yahoo.com' and parsed_url.path.startswith('/r'):
+ if parsed_url.hostname == 'search.yahoo.com'\
+ and parsed_url.path.startswith('/r'):
p = parsed_url.path
mark = p.find('/**')
if mark != -1:
@@ -82,15 +87,15 @@ def response(resp):
if results_xpath:
for result in dom.xpath(results_xpath):
url = extract_url(result.xpath(url_xpath))
- title = extract_text(result.xpath(title_xpath)[0 ])
+ title = extract_text(result.xpath(title_xpath)[0])
content = extract_text(result.xpath(content_xpath)[0])
results.append({'url': url, 'title': title, 'content': content})
else:
for url, title, content in zip(
- map(extract_url, dom.xpath(url_xpath)), \
- map(extract_text, dom.xpath(title_xpath)), \
- map(extract_text, dom.xpath(content_xpath)), \
- ):
+ map(extract_url, dom.xpath(url_xpath)),
+ map(extract_text, dom.xpath(title_xpath)),
+ map(extract_text, dom.xpath(content_xpath))
+ ):
results.append({'url': url, 'title': title, 'content': content})
if not suggestion_xpath:
diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py
index c93ac522f..a4a41ac3b 100644
--- a/searx/engines/yacy.py
+++ b/searx/engines/yacy.py
@@ -4,10 +4,12 @@ from urllib import urlencode
url = 'http://localhost:8090'
search_url = '/yacysearch.json?{query}&maximumRecords=10'
+
def request(query, params):
- params['url'] = url + search_url.format(query=urlencode({'query':query}))
+ params['url'] = url + search_url.format(query=urlencode({'query': query}))
return params
+
def response(resp):
raw_search_results = loads(resp.text)
@@ -25,7 +27,7 @@ def response(resp):
tmp_result['content'] = ''
if len(result['description']):
- tmp_result['content'] += result['description'] +"<br/>"
+ tmp_result['content'] += result['description'] + "<br/>"
if len(result['pubDate']):
tmp_result['content'] += result['pubDate'] + "<br/>"
diff --git a/searx/engines/youtube.py b/searx/engines/youtube.py
index cefdb6536..62884702f 100644
--- a/searx/engines/youtube.py
+++ b/searx/engines/youtube.py
@@ -5,6 +5,7 @@ categories = ['videos']
search_url = 'https://gdata.youtube.com/feeds/api/videos?alt=json&{query}'
+
def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}))
return params
@@ -30,17 +31,16 @@ def response(resp):
thumbnail = ''
if len(result['media$group']['media$thumbnail']):
thumbnail = result['media$group']['media$thumbnail'][0]['url']
- content += '<a href="{0}" title="{0}" ><img src="{1}" /></a>'.format(url, thumbnail)
+ content += '<a href="{0}" title="{0}" ><img src="{1}" /></a>'.format(url, thumbnail) # noqa
if len(content):
content += '<br />' + result['content']['$t']
else:
content = result['content']['$t']
- results.append({'url': url
- , 'title': title
- , 'content': content
- , 'template':'videos.html'
- , 'thumbnail':thumbnail})
+ results.append({'url': url,
+ 'title': title,
+ 'content': content,
+ 'template': 'videos.html',
+ 'thumbnail': thumbnail})
return results
-
diff --git a/searx/settings.yml b/searx/settings.yml
new file mode 100644
index 000000000..c207f3f57
--- /dev/null
+++ b/searx/settings.yml
@@ -0,0 +1,111 @@
+server:
+ port : 8888
+ secret_key : "ultrasecretkey" # change this!
+ debug : True
+ request_timeout : 3.0 # seconds
+ base_url: False
+
+engines:
+ - name : wikipedia
+ engine : mediawiki
+ url : https://en.wikipedia.org/
+ number_of_results : 1
+
+ - name : bing
+ engine : bing
+ locale : en-US
+
+ - name : currency
+ engine : currency_convert
+ categories : general
+
+ - name : deviantart
+ engine : deviantart
+ categories : images
+
+ - name : ddg definitions
+ engine : duckduckgo_definitions
+
+ - name : duckduckgo
+ engine : duckduckgo
+ locale : en-us
+
+ - name : filecrop
+ engine : filecrop
+ categories : files
+
+ - name : flickr
+ engine : flickr
+ categories : images
+
+ - name : github
+ engine : github
+ categories : it
+
+ - name : google
+ engine : json_engine
+ search_url : https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
+ categories : general
+ url_query : /responseData/results/unescapedUrl
+ content_query : /responseData/results/content
+ title_query : /responseData/results/titleNoFormatting
+
+ - name : google images
+ engine : google_images
+ categories : images
+
+ - name : piratebay
+ engine : piratebay
+ categories : videos, music, files
+
+ - name : soundcloud
+ engine : soundcloud
+ categories : music
+
+ - name : stackoverflow
+ engine : stackoverflow
+ categories : it
+
+ - name : startpage
+ engine : startpage
+
+ - name : twitter
+ engine : twitter
+ categories : social media
+
+ - name : urbandictionary
+ engine : xpath
+ search_url : http://www.urbandictionary.com/define.php?term={query}
+ url_xpath : //div[@class="word"]//a/@href
+ title_xpath : //div[@class="word"]//a
+ content_xpath : //div[@class="definition"]
+
+ - name : yahoo
+ engine : xpath
+ search_url : http://search.yahoo.com/search?p={query}
+ results_xpath : //div[@class="res"]
+ url_xpath : .//h3/a/@href
+ title_xpath : .//h3/a
+ content_xpath : .//div[@class="abstr"]
+ suggestion_xpath : //div[@id="satat"]//a
+
+ - name : youtube
+ engine : youtube
+ categories : videos
+
+ - name : dailymotion
+ engine : dailymotion
+ locale : en_US
+ categories : videos
+
+ - name : vimeo
+ engine : vimeo
+ categories : videos
+ results_xpath : //div[@id="browse_content"]/ol/li
+ url_xpath : ./a/@href
+ title_xpath : ./a/div[@class="data"]/p[@class="title"]/text()
+ content_xpath : ./a/img/@src
+
+locales:
+ en : English
+ hu : Magyar
diff --git a/searx/settings_robot.py b/searx/settings_robot.py
deleted file mode 100644
index 004add2a1..000000000
--- a/searx/settings_robot.py
+++ /dev/null
@@ -1,16 +0,0 @@
-
-port = 11111
-
-secret_key = "ultrasecretkey" # change this!
-
-debug = False
-
-request_timeout = 5.0 # seconds
-
-weights = {} # 'search_engine_name': float(weight) | default is 1.0
-
-blacklist = [] # search engine blacklist
-
-categories = {} # custom search engine categories
-
-base_url = None # "https://your.domain.tld/" or None (to use request parameters)
diff --git a/searx/settings_robot.yml b/searx/settings_robot.yml
new file mode 100644
index 000000000..d60ed3272
--- /dev/null
+++ b/searx/settings_robot.yml
@@ -0,0 +1,107 @@
+server:
+ port : 11111
+ secret_key : "ultrasecretkey" # change this!
+ debug : False
+ request_timeout : 3.0 # seconds
+ base_url: False
+
+engines:
+ - name : wikipedia
+ engine : mediawiki
+ url : https://en.wikipedia.org/
+ number_of_results : 1
+
+ - name : bing
+ engine : bing
+ locale : en-US
+
+ - name : currency
+ engine : currency_convert
+ categories : general
+
+ - name : deviantart
+ engine : deviantart
+ categories : images
+
+ - name : ddg definitions
+ engine : duckduckgo_definitions
+
+ - name : duckduckgo
+ engine : duckduckgo
+ locale : en-us
+
+ - name : filecrop
+ engine : filecrop
+ categories : files
+
+ - name : flickr
+ engine : flickr
+ categories : images
+
+ - name : github
+ engine : github
+ categories : it
+
+ - name : google
+ engine : json_engine
+ search_url : https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
+ categories : general
+ url_query : /responseData/results/unescapedUrl
+ content_query : /responseData/results/content
+ title_query : /responseData/results/titleNoFormatting
+
+ - name : google images
+ engine : google_images
+ categories : images
+
+ - name : piratebay
+ engine : piratebay
+ categories : videos, music, files
+
+ - name : soundcloud
+ engine : soundcloud
+ categories : music
+
+ - name : stackoverflow
+ engine : stackoverflow
+ categories : it
+
+ - name : startpage
+ engine : startpage
+
+ - name : twitter
+ engine : twitter
+ categories : social media
+
+ - name : urbandictionary
+ engine : xpath
+ search_url : http://www.urbandictionary.com/define.php?term={query}
+ url_xpath : //div[@class="word"]//a/@href
+ title_xpath : //div[@class="word"]//a
+ content_xpath : //div[@class="definition"]
+
+ - name : yahoo
+ engine : xpath
+ search_url : http://search.yahoo.com/search?p={query}
+ results_xpath : //div[@class="res"]
+ url_xpath : .//h3/a/@href
+ title_xpath : .//h3/a
+ content_xpath : .//div[@class="abstr"]
+ suggestion_xpath : //div[@id="satat"]//a
+
+ - name : youtube
+ engine : youtube
+ categories : videos
+
+ - name : dailymotion
+ engine : dailymotion
+ locale : en_US
+ categories : videos
+
+ - name : vimeo
+ engine : vimeo
+ categories : videos
+ results_xpath : //div[@id="browse_content"]/ol/li
+ url_xpath : ./a/@href
+ title_xpath : ./a/div[@class="data"]/p[@class="title"]/text()
+ content_xpath : ./a/img/@src
diff --git a/searx/static/css/style.css b/searx/static/css/style.css
index 83d281806..4163e753d 100644
--- a/searx/static/css/style.css
+++ b/searx/static/css/style.css
@@ -49,6 +49,8 @@ input[type="submit"] { border: 1px solid #666666; color: #444444; padding: 4px;
input[type="checkbox"] { visibility: hidden; }
+fieldset { margin: 8px; }
+
#categories { margin: 0 10px; }
.checkbox_container { display: inline-block; position: relative; margin: 0 3px; padding: 0px; }
@@ -79,7 +81,6 @@ a { text-decoration: none; color: #1a11be; }
a:visited { color: #7b11be; }
.result { margin: 19px 0 18px 0; padding: 0; max-width: 55em; clear: both; }
-.result:hover { background: #e8e7e6; }
.result_title { margin-bottom: 0; }
.result h3 { font-size: 1em; word-wrap:break-word; margin: 5px 0 1px 0; padding: 0 }
.result .content { font-size: 0.8em; margin: 0; padding: 0; max-width: 54em; word-wrap:break-word; line-height: 1.24; }
@@ -201,3 +202,5 @@ tr:hover td { background: #DDDDDD; }
.result img { max-width: 90%; width: auto; height: auto }
}
+
+.favicon { float: left; margin-right: 4px; }
diff --git a/searx/templates/about.html b/searx/templates/about.html
index 4e3f4bf4e..bb0a3e882 100644
--- a/searx/templates/about.html
+++ b/searx/templates/about.html
@@ -8,25 +8,25 @@
</p>
<h2>Why use Searx?</h2>
<ul>
- <li>Maybe Searx won’t offer you as personalised results as Google, but it doesn't make a profile about you</li>
- <li>Searx doesn't care about what you search, never shares anything with a third party, and it can't be used to compromise you</li>
- <li>Searx is a free software, the code is 100% open and you can help to make it better. See more on <a href="https://gmail.com/asciimoo/searx">github</a></li>
+ <li>Searx may not offer you as personalised results as Google, but it doesn't generate a profile about you</li>
+ <li>Searx doesn't care about what you search for, never shares anything with a third party, and it can't be used to compromise you</li>
+ <li>Searx is free software, the code is 100% open and you can help to make it better. See more on <a href="https://github.com/asciimoo/searx">github</a></li>
</ul>
- <p>If you do care about privacy, want to be a conscious user, moreover believe
+ <p>If you do care about privacy, want to be a conscious user, or otherwise believe
in digital freedom, make Searx your default search engine or run it on your own server</p>
<h2>Technical details - How does it work?</h2>
<p>Searx is a <a href="https://en.wikipedia.org/wiki/Metasearch_engine">metasearch engine</a>,
inspired by the <a href="http://seeks-project.info/">seeks project</a>.<br />
-It provides basic privacy by mixing your queries with searches on other platforms without storing search data. Queries are made using a POST request on every browser (except chrome*). Therefore they don't show up in our logs, neither in your url history. In case of Chrome* users there is an exception, Searx uses the search bar to perform GET requests.<br />
-Searx can be added to your browser's search bar, moreover it can be set as the default search engine.
+It provides basic privacy by mixing your queries with searches on other platforms without storing search data. Queries are made using a POST request on every browser (except chrome*). Therefore they show up in neither our logs, nor your url history. In case of Chrome* users there is an exception, Searx uses the search bar to perform GET requests.<br />
+Searx can be added to your browser's search bar; moreover, it can be set as the default search engine.
</p>
-<h2>How can I have my own?</h2>
+<h2>How can I make it my own?</h2>
-<p>Searx appreciates your suspicion regarding logs, so take the <a href="https://github.com/asciimoo/searx">code</a> and run it yourself! <br />Add your Searx to this <a href="https://github.com/asciimoo/searx/wiki/Searx-instances">list</a> to help other people to have privacy and make the Internet freer!
-<br />The more decentralized the Internet is the more freedom we have!</p>
+<p>Searx appreciates your concern regarding logs, so take the <a href="https://github.com/asciimoo/searx">code</a> and run it yourself! <br />Add your Searx to this <a href="https://github.com/asciimoo/searx/wiki/Searx-instances">list</a> to help other people reclaim their privacy and make the Internet freer!
+<br />The more decentralized the Internet, is the more freedom we have!</p>
<hr />
@@ -39,7 +39,7 @@ Searx can be added to your browser's search bar, moreover it can be set as the d
<h3>New engines?</h3>
<ul>
- <li>Edit your engines.cfg, see <a href="https://raw.github.com/asciimoo/searx/master/engines.cfg_sample">sample config</a></li>
+ <li>Edit your <a href="https://raw.github.com/asciimoo/searx/master/searx/settings.yml">settings.yml</a></li>
<li>Create your custom engine module, check the <a href="https://github.com/asciimoo/searx/blob/master/examples/basic_engine.py">example engine</a></li>
</ul>
<p>Don't forget to restart searx after config edit!</p>
@@ -48,7 +48,7 @@ Searx can be added to your browser's search bar, moreover it can be set as the d
<p>See the <a href="https://github.com/asciimoo/searx/wiki/Installation">installation and setup</a> wiki page</p>
<h3>How to debug engines?</h3>
-<p><a href="/stats">Stats page</a> contains some useful data about the used engines.</p>
+<p><a href="/stats">Stats page</a> contains some useful data about the engines used.</p>
</div>
{% endblock %}
diff --git a/searx/templates/categories.html b/searx/templates/categories.html
index b1fd3d1fc..57e63c85d 100644
--- a/searx/templates/categories.html
+++ b/searx/templates/categories.html
@@ -1,7 +1,7 @@
<div id="categories">
{% for category in categories %}
<div class="checkbox_container">
- <input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} /><label for="checkbox_{{ category|replace(' ', '_') }}">{{ category }}</label>
+ <input type="checkbox" id="checkbox_{{ category|replace(' ', '_') }}" name="category_{{ category }}" {% if category in selected_categories %}checked="checked"{% endif %} /><label for="checkbox_{{ category|replace(' ', '_') }}">{{ _(category) }}</label>
</div>
{% endfor %}
</div>
diff --git a/searx/templates/engines.html b/searx/templates/engines.html
index 1f52dc09f..008e860f1 100644
--- a/searx/templates/engines.html
+++ b/searx/templates/engines.html
@@ -1,12 +1,12 @@
{% extends 'base.html' %}
{% block content %}
<div class="row">
-<h2>Currently used search engines</h2>
+ <h2>{{ _('Currently used search engines') }}</h2>
<table style="width: 80%;">
<tr>
- <th>Engine name</th>
- <th>Category</th>
+ <th>{{ _('Engine name') }}</th>
+ <th>{{ _('Category') }}</th>
</tr>
{% for (categ,search_engines) in categs %}
{% for search_engine in search_engines %}
@@ -20,7 +20,6 @@
{% endfor %}
{% endfor %}
</table>
-<p>Please add more engines to this list, pull requests are welcome!</p>
-<p class="right"><a href="/">back</a></p>
+<p class="right"><a href="/">{{ _('back') }}</a></p>
</div>
{% endblock %}
diff --git a/searx/templates/index.html b/searx/templates/index.html
index d9fa3b502..18320ae09 100644
--- a/searx/templates/index.html
+++ b/searx/templates/index.html
@@ -4,8 +4,8 @@
<div class="title"><h1>searx</h1></div>
{% include 'search.html' %}
<p class="top_margin">
- <a href="/about" class="hmarg">about</a>
- <a href="/preferences" class="hmarg">preferences</a>
+ <a href="/about" class="hmarg">{{ _('about') }}</a>
+ <a href="/preferences" class="hmarg">{{ _('preferences') }}</a>
</p>
</div>
{% endblock %}
diff --git a/searx/templates/preferences.html b/searx/templates/preferences.html
index 705139e58..3c2afef21 100644
--- a/searx/templates/preferences.html
+++ b/searx/templates/preferences.html
@@ -2,18 +2,28 @@
{% block head %} {% endblock %}
{% block content %}
<div class="row">
- <h2>Preferences</h2>
+ <h2>{{ _('Preferences') }}</h2>
+ <form method="post" action="/preferences" id="search_form">
<fieldset>
- <legend>Default categories</legend>
- <form method="post" action="/preferences" id="search_form">
+ <legend>{{ _('Default categories') }}</legend>
<p>
{% include 'categories.html' %}
</p>
- <input type="submit" value="save" />
- </form>
</fieldset>
- <div class="right"><a href="/">back</a></div>
+ <fieldset>
+ <legend>{{ _('Interface language') }}</legend>
+ <p>
+ <select name='locale'>
+ {% for locale_id,locale_name in locales.items() %}
+ <option value={{ locale_id }} {% if locale_id == current_locale %}selected="selected"{% endif %}>{{ locale_name}}</option>
+ {% endfor %}
+ </select>
+ </p>
+ </fieldset>
+ <input type="submit" value="{{ _('save') }}" />
+ </form>
+ <div class="right"><a href="/">{{ _('back') }}</a></div>
</div>
{% endblock %}
diff --git a/searx/templates/result_templates/default.html b/searx/templates/result_templates/default.html
index ab6d469b4..d06a4598a 100644
--- a/searx/templates/result_templates/default.html
+++ b/searx/templates/result_templates/default.html
@@ -1,13 +1,11 @@
<div class="result {{ result.class }}">
{% if result['favicon'] %}
- <div style="float:left; margin:2px;">
- <img width="18" height="18" src="static/img/icon_{{result['favicon']}}.ico" alt="{{result['favicon']}}.ico" title="{{result['favicon']}}.ico" />
- </div>
+ <img width="14" height="14" class="favicon" src="static/img/icon_{{result['favicon']}}.ico" />
{% endif %}
<div>
- <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3></br>
+ <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3>
<p class="content">{% if result.content %}{{ result.content|safe }}<br />{% endif %}</p>
<p class="url">{{ result.pretty_url }}</p>
</div>
diff --git a/searx/templates/result_templates/videos.html b/searx/templates/result_templates/videos.html
index ae6d8f16c..d3391f0d3 100644
--- a/searx/templates/result_templates/videos.html
+++ b/searx/templates/result_templates/videos.html
@@ -1,13 +1,11 @@
<div class="result">
{% if result['favicon'] %}
- <div style="float:left; margin:2px;">
- <img width="18" height="18" src="static/img/icon_{{result['favicon']}}.ico" alt="{{result['favicon']}}.ico" title="{{result['favicon']}}.ico" />
- </div>
+ <img width="14" height="14" class="favicon" src="static/img/icon_{{result['favicon']}}.ico" />
{% endif %}
<p>
<h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3>
- <a href="{{ result.url }}"><img width="300" height="170" src="{{ result.thumbnail }}" title={{ result.title }} alt=" {{ result.title }}"/></a>
+ <a href="{{ result.url }}"><img width="400px" src="{{ result.thumbnail }}" title={{ result.title }} alt=" {{ result.title }}"/></a>
<p class="url">{{ result.url }}</p>
</p>
</div>
diff --git a/searx/templates/results.html b/searx/templates/results.html
index be40900c3..238671878 100644
--- a/searx/templates/results.html
+++ b/searx/templates/results.html
@@ -7,12 +7,12 @@
</div>
<div id="results">
{% if suggestions %}
- <div id="suggestions"><span>Suggestions: </span>{% for suggestion in suggestions %}<form method="post" action="/"><input type="hidden" name="q" value="{{suggestion}}"><input type="submit" value="{{ suggestion }}" /></form>{% endfor %}</div>
+ <div id="suggestions"><span>{{ _('Suggestions') }}:</span>{% for suggestion in suggestions %}<form method="post" action="/"><input type="hidden" name="q" value="{{suggestion}}"><input type="submit" value="{{ suggestion }}" /></form>{% endfor %}</div>
{% endif %}
-
+
<div id ="result_count">
- Number of results: {{ number_of_results }}
+ {{ _('Number of results') }}: {{ number_of_results }}
</div>
{% for result in results %}
@@ -23,7 +23,7 @@
{% endif %}
{% endfor %}
<div id="apis">
- Download results
+ {{ _('Download results') }}
<form method="post" action="/">
<div class="left">
<input type="hidden" name="q" value="{{ q }}" />
diff --git a/searx/templates/stats.html b/searx/templates/stats.html
index 933616e32..cb5757b31 100644
--- a/searx/templates/stats.html
+++ b/searx/templates/stats.html
@@ -1,7 +1,7 @@
{% extends "base.html" %}
{% block head %} {% endblock %}
{% block content %}
-<h2>Engine stats</h2>
+<h2>{{ _('Engine stats') }}</h2>
{% for stat_name,stat_category in stats %}
<div class="left">
diff --git a/searx/testing.py b/searx/testing.py
index 4b1810d6a..51c44d826 100644
--- a/searx/testing.py
+++ b/searx/testing.py
@@ -7,10 +7,10 @@ from unittest2 import TestCase
import os
import subprocess
-import sys
class SearxTestLayer:
+ """Base layer for non-robot tests."""
__name__ = u'SearxTestLayer'
@@ -36,24 +36,37 @@ class SearxRobotLayer(Layer):
def setUp(self):
os.setpgrp() # create new process group, become its leader
+
+ # get program paths
webapp = os.path.join(
os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
'webapp.py'
)
exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py')
+
+ # set robot settings path
+ os.environ['SEARX_SETTINGS_PATH'] = os.path.abspath(
+ os.path.dirname(__file__) + '/settings_robot.yml')
+
+ # run the server
self.server = subprocess.Popen(
- [exe, webapp, 'settings_robot'],
+ [exe, webapp],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
def tearDown(self):
- # TERM all processes in my group
+ # send TERM signal to all processes in my group, to stop subprocesses
os.killpg(os.getpgid(self.server.pid), 15)
+ # remove previously set environment variable
+ del os.environ['SEARX_SETTINGS_PATH']
+
SEARXROBOTLAYER = SearxRobotLayer()
class SearxTestCase(TestCase):
+ """Base test case for non-robot tests."""
+
layer = SearxTestLayer
diff --git a/searx/translations/hu/LC_MESSAGES/messages.mo b/searx/translations/hu/LC_MESSAGES/messages.mo
new file mode 100644
index 000000000..b0a1f6827
--- /dev/null
+++ b/searx/translations/hu/LC_MESSAGES/messages.mo
Binary files differ
diff --git a/searx/translations/hu/LC_MESSAGES/messages.po b/searx/translations/hu/LC_MESSAGES/messages.po
new file mode 100644
index 000000000..1c3525c20
--- /dev/null
+++ b/searx/translations/hu/LC_MESSAGES/messages.po
@@ -0,0 +1,115 @@
+# Hungarian translations for PROJECT.
+# Copyright (C) 2014 ORGANIZATION
+# This file is distributed under the same license as the PROJECT project.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PROJECT VERSION\n"
+"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
+"POT-Creation-Date: 2014-01-22 00:55+0100\n"
+"PO-Revision-Date: 2014-01-21 23:33+0100\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: hu <LL@li.org>\n"
+"Plural-Forms: nplurals=1; plural=0\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 1.3\n"
+
+#: searx/engines/__init__.py:274
+msgid "Page loads (sec)"
+msgstr "Válaszidők (sec)"
+
+#: searx/engines/__init__.py:278 searx/templates/results.html:15
+msgid "Number of results"
+msgstr "Találatok száma"
+
+#: searx/engines/__init__.py:282
+msgid "Scores"
+msgstr "Pontszámok"
+
+#: searx/engines/__init__.py:286
+msgid "Scores per result"
+msgstr "Pontszámok találatonként"
+
+#: searx/engines/__init__.py:290
+msgid "Errors"
+msgstr "Hibák"
+
+#: searx/templates/engines.html:4
+msgid "Currently used search engines"
+msgstr "Jelenleg használt keresők"
+
+#: searx/templates/engines.html:8
+msgid "Engine name"
+msgstr "Kereső neve"
+
+#: searx/templates/engines.html:9
+msgid "Category"
+msgstr "Kategória"
+
+#: searx/templates/engines.html:23 searx/templates/preferences.html:27
+msgid "back"
+msgstr "vissza"
+
+#: searx/templates/index.html:7
+msgid "about"
+msgstr "rólunk"
+
+#: searx/templates/index.html:8
+msgid "preferences"
+msgstr "beállítások"
+
+#: searx/templates/preferences.html:5
+msgid "Preferences"
+msgstr "Beállítások"
+
+#: searx/templates/preferences.html:10
+msgid "Default categories"
+msgstr "Alapértelmezett kategóriák"
+
+#: searx/templates/preferences.html:16
+msgid "Interface language"
+msgstr "Nyelv"
+
+#: searx/templates/preferences.html:25
+msgid "save"
+msgstr "mentés"
+
+#: searx/templates/results.html:10
+msgid "Suggestions"
+msgstr "Javaslatok"
+
+#: searx/templates/results.html:26
+msgid "Download results"
+msgstr "Találatok letöltése"
+
+#: searx/templates/stats.html:4
+msgid "Engine stats"
+msgstr "Kereső statisztikák"
+
+# categories - manually added
+# TODO - automatically add
+
+msgid "files"
+msgstr "fájlok"
+
+msgid "general"
+msgstr "általános"
+
+msgid "music"
+msgstr "zene"
+
+msgid "social media"
+msgstr "közösségi média"
+
+msgid "images"
+msgstr "képek"
+
+msgid "videos"
+msgstr "videók"
+
+msgid "it"
+msgstr "it"
+
diff --git a/searx/utils.py b/searx/utils.py
index 416055dfa..af8ce952e 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -1,13 +1,16 @@
from HTMLParser import HTMLParser
#import htmlentitydefs
import csv
-import codecs
+from codecs import getincrementalencoder
import cStringIO
import re
+
def gen_useragent():
# TODO
- return "Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"
+ ua = "Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"
+ return ua
+
def highlight_content(content, query):
@@ -34,16 +37,20 @@ def highlight_content(content, query):
return content
+
class HTMLTextExtractor(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
- self.result = [ ]
+ self.result = []
def handle_data(self, d):
self.result.append(d)
def handle_charref(self, number):
- codepoint = int(number[1:], 16) if number[0] in (u'x', u'X') else int(number)
+ if number[0] in (u'x', u'X'):
+ codepoint = int(number[1:], 16)
+ else:
+ codepoint = int(number)
self.result.append(unichr(codepoint))
def handle_entityref(self, name):
@@ -54,6 +61,7 @@ class HTMLTextExtractor(HTMLParser):
def get_text(self):
return u''.join(self.result)
+
def html_to_text(html):
s = HTMLTextExtractor()
s.feed(html)
@@ -71,10 +79,16 @@ class UnicodeWriter:
self.queue = cStringIO.StringIO()
self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
self.stream = f
- self.encoder = codecs.getincrementalencoder(encoding)()
+ self.encoder = getincrementalencoder(encoding)()
def writerow(self, row):
- self.writer.writerow([(s.encode("utf-8").strip() if type(s) == str or type(s) == unicode else str(s)) for s in row])
+ unicode_row = []
+ for col in row:
+ if type(col) == str or type(col) == unicode:
+ unicode_row.append(col.encode('utf-8').strip())
+ else:
+ unicode_row.append(col)
+ self.writer.writerow(unicode_row)
# Fetch UTF-8 output from the queue ...
data = self.queue.getvalue()
data = data.decode("utf-8")
diff --git a/searx/webapp.py b/searx/webapp.py
index 52398801e..b6d3e2b08 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -17,26 +17,36 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
+import json
+import cStringIO
import os
-import sys
-if __name__ == "__main__":
- sys.path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../'))
-from searx import settings
+from flask import Flask, request, render_template
+from flask import url_for, Response, make_response, redirect
+from flask import send_from_directory
-from flask import Flask, request, render_template, url_for, Response, make_response, redirect
+from searx import settings
from searx.engines import search, categories, engines, get_engines_stats
-import json
-import cStringIO
from searx.utils import UnicodeWriter
-from flask import send_from_directory
from searx.utils import highlight_content, html_to_text
+from flask.ext.babel import Babel
+
+app = Flask(
+ __name__,
+ static_folder=os.path.join(os.path.dirname(__file__), 'static'),
+ template_folder=os.path.join(os.path.dirname(__file__), 'templates')
+)
-app = Flask(__name__)
app.secret_key = settings['server']['secret_key']
+babel = Babel(app)
+
+#TODO configurable via settings.yml
+favicons = ['wikipedia', 'youtube', 'vimeo', 'soundcloud',
+ 'twitter', 'stackoverflow', 'github']
+
opensearch_xml = '''<?xml version="1.0" encoding="utf-8"?>
<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
@@ -51,6 +61,24 @@ opensearch_xml = '''<?xml version="1.0" encoding="utf-8"?>
'''
+@babel.localeselector
+def get_locale():
+ locale = request.accept_languages.best_match(settings['locales'].keys())
+
+ if request.cookies.get('locale', '') in settings['locales']:
+ locale = request.cookies.get('locale', '')
+
+ if 'locale' in request.args\
+ and request.args['locale'] in settings['locales']:
+ locale = request.args['locale']
+
+ if 'locale' in request.form\
+ and request.form['locale'] in settings['locales']:
+ locale = request.form['locale']
+
+ return locale
+
+
def get_base_url():
if settings['server']['base_url']:
hostname = settings['server']['base_url']
@@ -65,7 +93,8 @@ def get_base_url():
def render(template_name, **kwargs):
global categories
kwargs['categories'] = ['general']
- kwargs['categories'].extend(x for x in sorted(categories.keys()) if x != 'general')
+ kwargs['categories'].extend(x for x in
+ sorted(categories.keys()) if x != 'general')
if not 'selected_categories' in kwargs:
kwargs['selected_categories'] = []
cookie_categories = request.cookies.get('categories', '').split(',')
@@ -76,6 +105,7 @@ def render(template_name, **kwargs):
kwargs['selected_categories'] = ['general']
return render_template(template_name, **kwargs)
+
def parse_query(query):
query_engines = []
query_parts = query.split()
@@ -89,7 +119,7 @@ def parse_query(query):
def index():
global categories
- if request.method=='POST':
+ if request.method == 'POST':
request_data = request.form
else:
request_data = request.args
@@ -101,14 +131,15 @@ def index():
query, selected_engines = parse_query(request_data['q'].encode('utf-8'))
if not len(selected_engines):
- for pd_name,pd in request_data.items():
+ for pd_name, pd in request_data.items():
if pd_name.startswith('category_'):
category = pd_name[9:]
if not category in categories:
continue
selected_categories.append(category)
if not len(selected_categories):
- cookie_categories = request.cookies.get('categories', '').split(',')
+ cookie_categories = request.cookies.get('categories', '')
+ cookie_categories = cookie_categories.split(',')
for ccateg in cookie_categories:
if ccateg in categories:
selected_categories.append(ccateg)
@@ -116,7 +147,9 @@ def index():
selected_categories = ['general']
for categ in selected_categories:
- selected_engines.extend({'category': categ, 'name': x.name} for x in categories[categ])
+ selected_engines.extend({'category': categ,
+ 'name': x.name}
+ for x in categories[categ])
results, suggestions = search(query, request, selected_engines)
@@ -131,16 +164,18 @@ def index():
result['content'] = html_to_text(result['content']).strip()
result['title'] = html_to_text(result['title']).strip()
if len(result['url']) > 74:
- result['pretty_url'] = result['url'][:35] + '[..]' + result['url'][-35:]
+ url_parts = result['url'][:35], result['url'][-35:]
+ result['pretty_url'] = '{0}[...]{1}'.format(*url_parts)
else:
result['pretty_url'] = result['url']
for engine in result['engines']:
- if engine in ['wikipedia', 'youtube', 'vimeo', 'soundcloud', 'twitter', 'stackoverflow', 'github']:
+ if engine in favicons:
result['favicon'] = engine
if request_data.get('format') == 'json':
- return Response(json.dumps({'query': query, 'results': results}), mimetype='application/json')
+ return Response(json.dumps({'query': query, 'results': results}),
+ mimetype='application/json')
elif request_data.get('format') == 'csv':
csv = UnicodeWriter(cStringIO.StringIO())
keys = ('title', 'url', 'content', 'host', 'engine', 'score')
@@ -151,26 +186,28 @@ def index():
csv.writerow([row.get(key, '') for key in keys])
csv.stream.seek(0)
response = Response(csv.stream.read(), mimetype='application/csv')
- response.headers.add('Content-Disposition', 'attachment;Filename=searx_-_{0}.csv'.format('_'.join(query.split())))
+ content_disp = 'attachment;Filename=searx_-_{0}.csv'.format(query)
+ response.headers.add('Content-Disposition', content_disp)
return response
elif request_data.get('format') == 'rss':
- response_rss = render('opensearch_response_rss.xml'
- ,results=results
- ,q=request_data['q']
- ,number_of_results=len(results)
- ,base_url=get_base_url()
- )
+ response_rss = render(
+ 'opensearch_response_rss.xml',
+ results=results,
+ q=request_data['q'],
+ number_of_results=len(results),
+ base_url=get_base_url()
+ )
return Response(response_rss, mimetype='text/xml')
-
- return render('results.html'
- ,results=results
- ,q=request_data['q']
- ,selected_categories=selected_categories
- ,number_of_results=len(results)+len(featured_results)
- ,featured_results=featured_results
- ,suggestions=suggestions
- )
+ return render(
+ 'results.html',
+ results=results,
+ q=request_data['q'],
+ selected_categories=selected_categories,
+ number_of_results=len(results) + len(featured_results),
+ featured_results=featured_results,
+ suggestions=suggestions
+ )
@app.route('/about', methods=['GET'])
@@ -187,20 +224,37 @@ def list_engines():
@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
- if request.method=='POST':
+ if request.method == 'POST':
selected_categories = []
- for pd_name,pd in request.form.items():
+ locale = None
+ for pd_name, pd in request.form.items():
if pd_name.startswith('category_'):
category = pd_name[9:]
if not category in categories:
continue
selected_categories.append(category)
+ elif pd_name == 'locale' and pd in settings['locales']:
+ locale = pd
+
+ resp = make_response(redirect('/'))
+
+ if locale:
+ # cookie max age: 4 weeks
+ resp.set_cookie(
+ 'locale', locale,
+ max_age=60 * 60 * 24 * 7 * 4
+ )
+
if selected_categories:
- resp = make_response(redirect('/'))
# cookie max age: 4 weeks
- resp.set_cookie('categories', ','.join(selected_categories), max_age=60*60*24*7*4)
- return resp
- return render('preferences.html')
+ resp.set_cookie(
+ 'categories', ','.join(selected_categories),
+ max_age=60 * 60 * 24 * 7 * 4
+ )
+ return resp
+ return render('preferences.html',
+ locales=settings['locales'],
+ current_locale=get_locale())
@app.route('/stats', methods=['GET'])
@@ -216,6 +270,7 @@ def robots():
Allow: /
Allow: /about
Disallow: /stats
+Disallow: /engines
""", mimetype='text/plain')
@@ -229,24 +284,27 @@ def opensearch():
base_url = get_base_url()
ret = opensearch_xml.format(method=method, host=base_url)
resp = Response(response=ret,
- status=200,
- mimetype="application/xml")
+ status=200,
+ mimetype="application/xml")
return resp
+
@app.route('/favicon.ico')
def favicon():
return send_from_directory(os.path.join(app.root_path, 'static/img'),
- 'favicon.png', mimetype='image/vnd.microsoft.icon')
+ 'favicon.png',
+ mimetype='image/vnd.microsoft.icon')
def run():
from gevent import monkey
monkey.patch_all()
- app.run(debug = settings['server']['debug']
- ,use_debugger = settings['server']['debug']
- ,port = settings['server']['port']
- )
+ app.run(
+ debug=settings['server']['debug'],
+ use_debugger=settings['server']['debug'],
+ port=settings['server']['port']
+ )
if __name__ == "__main__":