summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.travis.yml5
-rw-r--r--requirements-dev.txt6
-rw-r--r--searx/engines/kickass.py46
-rw-r--r--searx/settings.yml12
-rw-r--r--searx/templates/courgette/base.html2
-rw-r--r--searx/templates/legacy/base.html2
-rw-r--r--searx/templates/oscar/macros.html6
-rw-r--r--searx/utils.py15
-rw-r--r--searx/webapp.py26
-rw-r--r--tests/robot/test_basic.robot44
-rw-r--r--tests/unit/engines/test_kickass.py26
11 files changed, 108 insertions, 82 deletions
diff --git a/.travis.yml b/.travis.yml
index 65f8ef235..0a174ff66 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,8 @@ cache:
- npm
- directories:
- $HOME/.cache/pip
+addons:
+ firefox: "latest"
language: python
python:
- "2.7"
@@ -12,6 +14,9 @@ before_install:
- "sh -e /etc/init.d/xvfb start"
- npm install less grunt-cli
- ( cd searx/static/themes/oscar;npm install; cd - )
+ - mkdir -p ~/drivers; export PATH=~/drivers:$PATH;
+ - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-linux64.tar.gz";
+ - FILE=`mktemp`; wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C ~/drivers -f $FILE geckodriver; rm $FILE; chmod 777 ~/drivers/geckodriver;
install:
- ./manage.sh update_dev_packages
- pip install coveralls
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 580ef63e5..543521895 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -3,8 +3,8 @@ mock==2.0.0
nose2[coverage-plugin]
pep8==1.7.0
plone.testing==5.0.0
-robotframework-selenium2library==1.7.4
+robotframework-selenium2library==1.8.0
robotsuite==1.7.0
-transifex-client==0.11
+transifex-client==0.12.2
unittest2==1.1.0
-zope.testrunner==4.4.10
+zope.testrunner==4.5.1
diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py
index 4c5d24008..9cd8284da 100644
--- a/searx/engines/kickass.py
+++ b/searx/engines/kickass.py
@@ -16,13 +16,14 @@ from urllib import quote
from lxml import html
from operator import itemgetter
from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size, convert_str_to_int
# engine dependent config
categories = ['videos', 'music', 'files']
paging = True
# search-url
-url = 'https://kickass.to/'
+url = 'https://kickass.cd/'
search_url = url + 'search/{search_term}/{pageno}/'
# specific xpath variables
@@ -57,41 +58,16 @@ def response(resp):
href = urljoin(url, link.attrib['href'])
title = extract_text(link)
content = escape(extract_text(result.xpath(content_xpath)))
- seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
- leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
- filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
- filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0]
- files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0]
-
- # convert seed to int if possible
- if seed.isdigit():
- seed = int(seed)
- else:
- seed = 0
+ seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
+ leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
+ filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))
+ files = extract_text(result.xpath('.//td[contains(@class, "center")][2]'))
- # convert leech to int if possible
- if leech.isdigit():
- leech = int(leech)
- else:
- leech = 0
-
- # convert filesize to byte if possible
- try:
- filesize = float(filesize)
-
- # convert filesize to byte
- if filesize_multiplier == 'TB':
- filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
- elif filesize_multiplier == 'GB':
- filesize = int(filesize * 1024 * 1024 * 1024)
- elif filesize_multiplier == 'MB':
- filesize = int(filesize * 1024 * 1024)
- elif filesize_multiplier == 'KB':
- filesize = int(filesize * 1024)
- except:
- filesize = None
-
- # convert files to int if possible
+ seed = convert_str_to_int(seed)
+ leech = convert_str_to_int(leech)
+
+ filesize, filesize_multiplier = filesize_info.split()
+ filesize = get_torrent_size(filesize, filesize_multiplier)
if files.isdigit():
files = int(files)
else:
diff --git a/searx/settings.yml b/searx/settings.yml
index e83f5aea7..ba7ae428b 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -18,6 +18,12 @@ ui:
default_theme : oscar # ui theme
default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
+# searx supports result proxification using an external service: https://github.com/asciimoo/morty
+# uncomment below section if you have running morty proxy
+#result_proxy:
+# url : http://127.0.0.1:3000/
+# key : your_morty_proxy_key
+
outgoing: # communication with search engines
request_timeout : 2.0 # seconds
useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
@@ -301,6 +307,12 @@ engines:
timeout : 6.0
disabled : True
+ - name: kickass
+ engine : kickass
+ shortcut : kc
+ timeout : 4.0
+ disabled : True
+
- name : microsoft academic
engine : json_engine
paging : True
diff --git a/searx/templates/courgette/base.html b/searx/templates/courgette/base.html
index b2c70a3b7..8e272585c 100644
--- a/searx/templates/courgette/base.html
+++ b/searx/templates/courgette/base.html
@@ -22,7 +22,7 @@
{% endblock %}
{% block meta %}{% endblock %}
{% block head %}
- <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+ <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
{% endblock %}
<script type="text/javascript">
searx = {};
diff --git a/searx/templates/legacy/base.html b/searx/templates/legacy/base.html
index a2c38fef7..da19741cb 100644
--- a/searx/templates/legacy/base.html
+++ b/searx/templates/legacy/base.html
@@ -17,7 +17,7 @@
{% endblock %}
{% block meta %}{% endblock %}
{% block head %}
- <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+ <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
{% endblock %}
</head>
<body>
diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html
index 06881db02..221300fe4 100644
--- a/searx/templates/oscar/macros.html
+++ b/searx/templates/oscar/macros.html
@@ -33,6 +33,9 @@
<span class="label label-default">{{ engine }}</span>
{% endfor %}
<small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+ {% if proxify %}
+ <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+ {% endif %}
</div>
<div class="text-muted"><small>{{ result.pretty_url }}</small></div>
{%- endmacro %}
@@ -44,6 +47,9 @@
<span class="label label-default">{{ engine }}</span>
{% endfor %}
<small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+ {% if proxify %}
+ <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+ {% endif %}
<div class="text-muted"><small>{{ result.pretty_url }}</small></div>
{%- endmacro %}
diff --git a/searx/utils.py b/searx/utils.py
index b3806d3fd..5039fa975 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -252,12 +252,27 @@ def get_torrent_size(filesize, filesize_multiplier):
filesize = int(filesize * 1024 * 1024)
elif filesize_multiplier == 'KB':
filesize = int(filesize * 1024)
+ elif filesize_multiplier == 'TiB':
+ filesize = int(filesize * 1000 * 1000 * 1000 * 1000)
+ elif filesize_multiplier == 'GiB':
+ filesize = int(filesize * 1000 * 1000 * 1000)
+ elif filesize_multiplier == 'MiB':
+ filesize = int(filesize * 1000 * 1000)
+ elif filesize_multiplier == 'KiB':
+ filesize = int(filesize * 1000)
except:
filesize = None
return filesize
+def convert_str_to_int(number_str):
+ if number_str.isdigit():
+ return int(number_str)
+ else:
+ return 0
+
+
def is_valid_lang(lang):
is_abbr = (len(lang) == 2)
if is_abbr:
diff --git a/searx/webapp.py b/searx/webapp.py
index 5bdbc71a6..6c2b98c9c 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -22,10 +22,11 @@ if __name__ == '__main__':
from os.path import realpath, dirname
path.append(realpath(dirname(realpath(__file__)) + '/../'))
-import json
import cStringIO
-import os
import hashlib
+import hmac
+import json
+import os
import requests
from searx import logger
@@ -242,6 +243,20 @@ def url_for_theme(endpoint, override_theme=None, **values):
return url_for(endpoint, **values)
+def proxify(url):
+ if url.startswith('//'):
+ url = 'https:' + url
+
+ if not settings.get('result_proxy'):
+ return url
+
+ h = hmac.new(settings['result_proxy']['key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
+
+ return '{0}?{1}'.format(settings['result_proxy']['url'],
+ urlencode(dict(mortyurl=url.encode('utf-8'),
+ mortyhash=h)))
+
+
def image_proxify(url):
if url.startswith('//'):
@@ -250,8 +265,7 @@ def image_proxify(url):
if not request.preferences.get_value('image_proxy'):
return url
- hash_string = url + settings['server']['secret_key']
- h = hashlib.sha256(hash_string.encode('utf-8')).hexdigest()
+ h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
return '{0}?{1}'.format(url_for('image_proxy'),
urlencode(dict(url=url.encode('utf-8'), h=h)))
@@ -310,6 +324,8 @@ def render(template_name, override_theme=None, **kwargs):
kwargs['image_proxify'] = image_proxify
+ kwargs['proxify'] = proxify if settings.get('result_proxy') else None
+
kwargs['get_result_template'] = get_result_template
kwargs['theme'] = get_current_theme_name(override=override_theme)
@@ -599,7 +615,7 @@ def image_proxy():
if not url:
return '', 400
- h = hashlib.sha256(url + settings['server']['secret_key'].encode('utf-8')).hexdigest()
+ h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest()
if h != request.args.get('h'):
return '', 400
diff --git a/tests/robot/test_basic.robot b/tests/robot/test_basic.robot
index ab41265a0..a466a095c 100644
--- a/tests/robot/test_basic.robot
+++ b/tests/robot/test_basic.robot
@@ -4,6 +4,14 @@ Test Setup Open Browser http://localhost:11111/
Test Teardown Close All Browsers
+*** Keywords ***
+Submit Preferences
+ Set Selenium Speed 2 seconds
+ Submit Form id=search_form
+ Location Should Be http://localhost:11111/
+ Set Selenium Speed 0 seconds
+
+
*** Test Cases ***
Front page
Page Should Contain about
@@ -33,8 +41,7 @@ Switch category
Page Should Contain Checkbox category_dummy
Click Element xpath=//*[.="general"]
Click Element xpath=//*[.="dummy"]
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Checkbox Should Not Be Selected category_general
Checkbox Should Be Selected category_dummy
@@ -43,8 +50,7 @@ Change language
Page Should Contain preferences
Go To http://localhost:11111/preferences
Select From List locale hu
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Page Should Contain rólunk
Page Should Contain beállítások
@@ -53,13 +59,11 @@ Change method
Page Should Contain preferences
Go To http://localhost:11111/preferences
Select From List method GET
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be method GET
Select From List method POST
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be method POST
@@ -69,8 +73,7 @@ Change theme
Go To http://localhost:11111/preferences
List Selection Should Be theme legacy
Select From List theme oscar
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be theme oscar
@@ -80,8 +83,7 @@ Change safesearch
Go To http://localhost:11111/preferences
List Selection Should Be safesearch None
Select From List safesearch Strict
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be safesearch Strict
@@ -91,8 +93,7 @@ Change image proxy
Go To http://localhost:11111/preferences
List Selection Should Be image_proxy Disabled
Select From List image_proxy Enabled
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be image_proxy Enabled
@@ -102,8 +103,7 @@ Change search language
Go To http://localhost:11111/preferences
List Selection Should Be language Automatic
Select From List language Turkish (Turkey) - tr_TR
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be language Turkish (Turkey) - tr_TR
@@ -113,8 +113,7 @@ Change autocomplete
Go To http://localhost:11111/preferences
List Selection Should Be autocomplete -
Select From List autocomplete google
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be autocomplete google
@@ -126,8 +125,7 @@ Change allowed/disabled engines
Element Should Contain xpath=//label[@class="deny"][@for='engine_dummy_dummy_dummy'] Block
Element Should Contain xpath=//label[@class="deny"][@for='engine_general_general_dummy'] Block
Click Element xpath=//label[@class="deny"][@for='engine_general_general_dummy']
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Page Should Contain about
Page Should Contain preferences
Go To http://localhost:11111/preferences
@@ -141,16 +139,14 @@ Block a plugin
Go To http://localhost:11111/preferences
List Selection Should Be theme legacy
Select From List theme oscar
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
List Selection Should Be theme oscar
Page Should Contain Plugins
Click Link Plugins
Checkbox Should Not Be Selected id=plugin_HTTPS_rewrite
Click Element xpath=//label[@for='plugin_HTTPS_rewrite']
- Submit Form id=search_form
- Location Should Be http://localhost:11111/
+ Submit Preferences
Go To http://localhost:11111/preferences
Page Should Contain Plugins
Click Link Plugins
diff --git a/tests/unit/engines/test_kickass.py b/tests/unit/engines/test_kickass.py
index 4cfcaa63c..96c17911c 100644
--- a/tests/unit/engines/test_kickass.py
+++ b/tests/unit/engines/test_kickass.py
@@ -14,7 +14,7 @@ class TestKickassEngine(SearxTestCase):
params = kickass.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
- self.assertIn('kickass.to', params['url'])
+ self.assertIn('kickass.cd', params['url'])
self.assertFalse(params['verify'])
def test_response(self):
@@ -84,7 +84,7 @@ class TestKickassEngine(SearxTestCase):
</span>
</div>
</td>
- <td class="nobr center">449 <span>bytes</span></td>
+ <td class="nobr center">449 bytes</td>
<td class="center">4</td>
<td class="center">2&nbsp;years</td>
<td class="green center">10</td>
@@ -97,7 +97,7 @@ class TestKickassEngine(SearxTestCase):
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This should be the title')
- self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+ self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
self.assertEqual(results[0]['seed'], 10)
self.assertEqual(results[0]['leech'], 1)
@@ -191,7 +191,7 @@ class TestKickassEngine(SearxTestCase):
</span>
</div>
</td>
- <td class="nobr center">1 <span>KB</span></td>
+ <td class="nobr center">1 KiB</td>
<td class="center">4</td>
<td class="center">2&nbsp;years</td>
<td class="green center">10</td>
@@ -235,7 +235,7 @@ class TestKickassEngine(SearxTestCase):
</span>
</div>
</td>
- <td class="nobr center">1 <span>MB</span></td>
+ <td class="nobr center">1 MiB</td>
<td class="center">4</td>
<td class="center">2&nbsp;years</td>
<td class="green center">9</td>
@@ -279,7 +279,7 @@ class TestKickassEngine(SearxTestCase):
</span>
</div>
</td>
- <td class="nobr center">1 <span>GB</span></td>
+ <td class="nobr center">1 GiB</td>
<td class="center">4</td>
<td class="center">2&nbsp;years</td>
<td class="green center">8</td>
@@ -323,7 +323,7 @@ class TestKickassEngine(SearxTestCase):
</span>
</div>
</td>
- <td class="nobr center">1 <span>TB</span></td>
+ <td class="nobr center">1 TiB</td>
<td class="center">4</td>
<td class="center">2&nbsp;years</td>
<td class="green center">7</td>
@@ -367,7 +367,7 @@ class TestKickassEngine(SearxTestCase):
</span>
</div>
</td>
- <td class="nobr center">z <span>bytes</span></td>
+ <td class="nobr center">z bytes</td>
<td class="center">r</td>
<td class="center">2&nbsp;years</td>
<td class="green center">a</td>
@@ -380,17 +380,17 @@ class TestKickassEngine(SearxTestCase):
self.assertEqual(type(results), list)
self.assertEqual(len(results), 5)
self.assertEqual(results[0]['title'], 'This should be the title')
- self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+ self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
self.assertEqual(results[0]['seed'], 10)
self.assertEqual(results[0]['leech'], 1)
self.assertEqual(results[0]['files'], 4)
self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
- self.assertEqual(results[0]['filesize'], 1024)
- self.assertEqual(results[1]['filesize'], 1048576)
- self.assertEqual(results[2]['filesize'], 1073741824)
- self.assertEqual(results[3]['filesize'], 1099511627776)
+ self.assertEqual(results[0]['filesize'], 1000)
+ self.assertEqual(results[1]['filesize'], 1000000)
+ self.assertEqual(results[2]['filesize'], 1000000000)
+ self.assertEqual(results[3]['filesize'], 1000000000000)
self.assertEqual(results[4]['seed'], 0)
self.assertEqual(results[4]['leech'], 0)
self.assertEqual(results[4]['files'], None)
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-12 06:44:04 +0000